From d996c2c10ba5b71eb4f7685ffe2509ff5759da17 Mon Sep 17 00:00:00 2001 From: Knuti_in_Paese Date: Wed, 1 Feb 2023 21:42:51 +0100 Subject: [PATCH 01/32] RPI:using pyRF24 on Debian 11 bullseye environment known RF24 lib can not installed on Debian 11 bullseye 64 bit operating system now, system try to import RF24 nor pyrf24 --- tools/rpi/README.md | 10 ++++++++++ tools/rpi/ahoy.yml.example | 2 +- tools/rpi/hoymiles/__init__.py | 14 +++++++++++++- 3 files changed, 24 insertions(+), 2 deletions(-) diff --git a/tools/rpi/README.md b/tools/rpi/README.md index bc8b23c0..75b5c1e2 100644 --- a/tools/rpi/README.md +++ b/tools/rpi/README.md @@ -78,6 +78,16 @@ cd examples_linux/ python3 getting_started.py # to test and see whether RF24 class can be loaded as module in python correctly ``` + +``` for bullseye - Debian 11 on 64 bit operating system +[ $(lscpu | grep Architecture | awk '{print $2}') != "aarch64" ]] && echo "Not a 64 bit architecture for this step!" + +git clone --recurse-submodules https://github.com/nRF24/pyRF24.git +cd pyRF24 +python -m pip install . -v # this step takes about 5 minutes! +``` + + If there are no error messages on the last step, then the NRF24 Wrapper has been installed successfully. Required python modules diff --git a/tools/rpi/ahoy.yml.example b/tools/rpi/ahoy.yml.example index fb033f80..928374de 100644 --- a/tools/rpi/ahoy.yml.example +++ b/tools/rpi/ahoy.yml.example @@ -31,7 +31,7 @@ ahoy: QoS: 0 Retain: True last_will: - topic: Appelweg_PV/114181807700 # defaults to 'hoymiles/{serial}' + topic: hoymiles/114172220003 # defaults to 'hoymiles/{serial}' payload: "LAST-WILL-MESSAGE: Please check my HOST and Process!" # Influx2 output diff --git a/tools/rpi/hoymiles/__init__.py b/tools/rpi/hoymiles/__init__.py index 74ec9fd8..eeb7e7de 100644 --- a/tools/rpi/hoymiles/__init__.py +++ b/tools/rpi/hoymiles/__init__.py @@ -11,9 +11,21 @@ import re from datetime import datetime import logging import crcmod -from RF24 import RF24, RF24_PA_MIN, RF24_PA_LOW, RF24_PA_HIGH, RF24_PA_MAX, RF24_250KBPS, RF24_CRC_DISABLED, RF24_CRC_8, RF24_CRC_16 from .decoders import * +try: + # OSI Layer 2 driver for nRF24L01 on Arduino & Raspberry Pi/Linux Devices + # https://github.com/nRF24/RF24.git + from RF24 import RF24, RF24_PA_MIN, RF24_PA_LOW, RF24_PA_HIGH, RF24_PA_MAX, RF24_250KBPS, RF24_CRC_DISABLED, RF24_CRC_8, RF24_CRC_16 +except ModuleNotFoundError: + try: + # Repo for pyRF24 package + # https://github.com/nRF24/pyRF24.git + from pyrf24 import RF24, RF24_PA_MIN, RF24_PA_LOW, RF24_PA_HIGH, RF24_PA_MAX, RF24_250KBPS, RF24_CRC_DISABLED, RF24_CRC_8, RF24_CRC_16 + except ModuleNotFoundError: + print("Module for RF24 not found - exit") + exit() + f_crc_m = crcmod.predefined.mkPredefinedCrcFun('modbus') f_crc8 = crcmod.mkCrcFun(0x101, initCrc=0, xorOut=0) From 0d552e3007976689463aa5cbed86822736c93cd3 Mon Sep 17 00:00:00 2001 From: Knuti_in_Paese Date: Thu, 2 Feb 2023 14:21:30 +0100 Subject: [PATCH 02/32] RPI:error handling while getting corruppted data extended error handling while getting corruppted data on 64 bit operating system (bullseye) lots of currupted data are reseived on Debian 11 OS. So we have to check the data length before using strict.unpack --- tools/rpi/README.md | 13 +++++++------ tools/rpi/hoymiles/__init__.py | 8 ++++---- tools/rpi/hoymiles/__main__.py | 3 ++- tools/rpi/hoymiles/decoders/__init__.py | 18 +++++++++++++----- 4 files changed, 26 insertions(+), 16 deletions(-) diff --git a/tools/rpi/README.md b/tools/rpi/README.md index 75b5c1e2..36b08f79 100644 --- a/tools/rpi/README.md +++ b/tools/rpi/README.md @@ -78,8 +78,12 @@ cd examples_linux/ python3 getting_started.py # to test and see whether RF24 class can be loaded as module in python correctly ``` +If there are no error messages on the last step, then the NRF24 Wrapper has been installed successfully. + -``` for bullseye - Debian 11 on 64 bit operating system +for Debian 11 (bullseye) 64 bit operating system +------------------------------------------------- +```code [ $(lscpu | grep Architecture | awk '{print $2}') != "aarch64" ]] && echo "Not a 64 bit architecture for this step!" git clone --recurse-submodules https://github.com/nRF24/pyRF24.git @@ -87,15 +91,12 @@ cd pyRF24 python -m pip install . -v # this step takes about 5 minutes! ``` - -If there are no error messages on the last step, then the NRF24 Wrapper has been installed successfully. - Required python modules ----------------------- Some modules are not installed by default on a RaspberryPi, therefore add them manually: -``` +```code pip install crcmod pyyaml paho-mqtt SunTimes ``` @@ -122,7 +123,7 @@ Python parameters The application describes itself -``` +```code python3 -m hoymiles --help usage: hoymiles [-h] -c [CONFIG_FILE] [--log-transactions] [--verbose] diff --git a/tools/rpi/hoymiles/__init__.py b/tools/rpi/hoymiles/__init__.py index eeb7e7de..a754ff6d 100644 --- a/tools/rpi/hoymiles/__init__.py +++ b/tools/rpi/hoymiles/__init__.py @@ -170,15 +170,15 @@ class ResponseDecoder(ResponseDecoderFactory): model = self.inverter_model command = self.request_command - c_datetime = self.time_rx.strftime("%Y-%m-%d %H:%M:%S.%f") - logging.info(f'{c_datetime} model_decoder: {model}Decode{command.upper()}') + if HOYMILES_DEBUG_LOGGING: + c_datetime = self.time_rx.strftime("%Y-%m-%d %H:%M:%S.%f") + logging.info(f'{c_datetime} model_decoder: {model}Decode{command.upper()}') model_decoders = __import__('hoymiles.decoders') if hasattr(model_decoders, f'{model}Decode{command.upper()}'): device = getattr(model_decoders, f'{model}Decode{command.upper()}') else: - if HOYMILES_DEBUG_LOGGING: - device = getattr(model_decoders, 'DebugDecodeAny') + device = getattr(model_decoders, 'DebugDecodeAny') return device(self.response, time_rx=self.time_rx, diff --git a/tools/rpi/hoymiles/__main__.py b/tools/rpi/hoymiles/__main__.py index 97773ed1..35943eb4 100644 --- a/tools/rpi/hoymiles/__main__.py +++ b/tools/rpi/hoymiles/__main__.py @@ -179,8 +179,8 @@ def poll_inverter(inverter, dtu_ser, do_init, retries): # Handle the response data if any if response: - c_datetime = datetime.now() if hoymiles.HOYMILES_DEBUG_LOGGING: + c_datetime = datetime.now() logging.debug(f'{c_datetime} Payload: ' + hoymiles.hexify_payload(response)) # prepare decoder object @@ -195,6 +195,7 @@ def poll_inverter(inverter, dtu_ser, do_init, retries): # get decoder object result = decoder.decode() if hoymiles.HOYMILES_DEBUG_LOGGING: + c_datetime = datetime.now() logging.info(f'{c_datetime} Decoded: {result.__dict__()}') # check decoder object for output diff --git a/tools/rpi/hoymiles/decoders/__init__.py b/tools/rpi/hoymiles/decoders/__init__.py index b46dbe48..e27b502d 100644 --- a/tools/rpi/hoymiles/decoders/__init__.py +++ b/tools/rpi/hoymiles/decoders/__init__.py @@ -110,6 +110,9 @@ class StatusResponse(Response): :rtype: tuple """ size = struct.calcsize(fmt) + if (len(self.response) < base+size): + logging.error(f'base: {base} size: {size} len: {len(self.response)} fmt: {fmt} rep: {self.response}') + return [0] return struct.unpack(fmt, self.response[base:base+size]) @property @@ -331,10 +334,12 @@ class EventsResponse(UnknownResponse): logging.debug(' '.join([f'{byte:02x}' for byte in chunk]) + ': ') - opcode, a_code, a_count, uptime_sec = struct.unpack('>BBHH', chunk[0:6]) - a_text = self.alarm_codes.get(a_code, 'N/A') - - logging.debug(f' uptime={timedelta(seconds=uptime_sec)} a_count={a_count} opcode={opcode} a_code={a_code} a_text={a_text}') + if (len(chunk[0:6]) == 6): + opcode, a_code, a_count, uptime_sec = struct.unpack('>BBHH', chunk[0:6]) + a_text = self.alarm_codes.get(a_code, 'N/A') + logging.debug(f' uptime={timedelta(seconds=uptime_sec)} a_count={a_count} opcode={opcode} a_code={a_code} a_text={a_text}') + else: + logging.error(f'length of chunk must be greater or equal 6 bytes: {chunk}') dbg = '' for fmt in ['BBHHHHH']: @@ -362,7 +367,10 @@ class HardwareInfoResponse(UnknownResponse): """ Base values, availabe in each __dict__ call """ responce_info = self.response - logging.info(f'HardwareInfoResponse: {struct.unpack(">HHHHHHHH", responce_info)}') + if (len(responce_info) >= 16): + logging.info(f'HardwareInfoResponse: {struct.unpack(">HHHHHHHH", responce_info)}') + else: + logging.error(f'wrong length of HardwareInfoResponse: {responce_info}') fw_version, fw_build_yyyy, fw_build_mmdd, fw_build_hhmm, hw_id = struct.unpack('>HHHHH', self.response[0:10]) From 892f554ff54025af530219ec17e2260f19660af4 Mon Sep 17 00:00:00 2001 From: Knuti_in_Paese Date: Sat, 4 Feb 2023 11:41:29 +0100 Subject: [PATCH 03/32] RPI:finer tuned debug logging Description for prep RF24 and pyrf24 on debian 11 (bullseye) 64 bit OS --- tools/rpi/README.md | 52 +++++++++++++++++++++++-- tools/rpi/ahoy.service | 7 ++-- tools/rpi/hoymiles/__init__.py | 29 +++++++++++--- tools/rpi/hoymiles/__main__.py | 22 +++++++---- tools/rpi/hoymiles/decoders/__init__.py | 28 +++++++------ tools/rpi/hoymiles/outputs.py | 15 +++++-- 6 files changed, 118 insertions(+), 35 deletions(-) diff --git a/tools/rpi/README.md b/tools/rpi/README.md index 36b08f79..12b6f5ef 100644 --- a/tools/rpi/README.md +++ b/tools/rpi/README.md @@ -81,14 +81,58 @@ python3 getting_started.py # to test and see whether RF24 class can be loaded as If there are no error messages on the last step, then the NRF24 Wrapper has been installed successfully. -for Debian 11 (bullseye) 64 bit operating system -------------------------------------------------- +Building RF24 Wrapper for Debian 11 (bullseye) 64 bit operating system +---------------------------------------------------------------------- +The description above does not work on Debian 11 (bullseye) 64 bit operating system. +There are 2 possible sollutions to install the RF24 Wrapper. + + * `1. solution:` +```code +sudo apt install cmake git python3-dev libboost-python-dev python3-pip python3-rpi.gpio + +sudo ln -s $(ls /usr/lib/$(ls /usr/lib/gcc | \ + head -1)/libboost_python3*.so | \ + tail -1) /usr/lib/$(ls /usr/lib/gcc | \ + head -1)/libboost_python3.so + +git clone https://github.com/nRF24/RF24.git +cd RF24 + +rm -rf build Makefile.inc +./configure --driver=SPIDEV +``` + * edit `Makefile.inc` with your prefered editor e.g. nano or vi + old: +```code + CPUFLAGS=-marm -march=armv6zk -mtune=arm1176jzf-s -mfpu=vfp -mfloat-abi=hard + CFLAGS=-marm -march=armv6zk -mtune=arm1176jzf-s -mfpu=vfp -mfloat-abi=hard -Ofast -Wall -pthread +``` + new: +```code + CPUFLAGS= + CFLAGS=-Ofast -Wall -pthread +``` + * continue with +```code +make +sudo make install + +cd pyRF24 +rm -r ./build/ ./dist/ ./RF24.egg-info/ ./__pycache__/ #just to make sure there is no old stuff +python3 -m pip install --upgrade pip +python3 -m pip install . +python3 -m pip list #watch for RF24 module - if its there its installed +``` + + + + * `2. solution:` ```code -[ $(lscpu | grep Architecture | awk '{print $2}') != "aarch64" ]] && echo "Not a 64 bit architecture for this step!" +sudo apt install git python3-dev libboost-python-dev python3-pip python3-rpi.gpio git clone --recurse-submodules https://github.com/nRF24/pyRF24.git cd pyRF24 -python -m pip install . -v # this step takes about 5 minutes! +python3 -m pip install . -v # this step takes about 5 minutes on my RPI-4 ! ``` Required python modules diff --git a/tools/rpi/ahoy.service b/tools/rpi/ahoy.service index 394bc09e..4af9ea89 100644 --- a/tools/rpi/ahoy.service +++ b/tools/rpi/ahoy.service @@ -6,11 +6,10 @@ # WorkingDirectory (absolute path to your private ahoy dir) # To change other config parameter, please consult systemd documentation # -# To activate this service, create a link, enable and start the ahoy.service -# $ mkdir -p $HOME/.config/systemd/user -# $ ln -sf $(pwd)/ahoy/tools/rpi/ahoy.service -t $HOME/.config/systemd/user +# To activate this service, create a link with enable and start the ahoy.service +# $ mkdir -p $HOME/.config/systemd/user +# $ systemctl --user enable $(pwd)/ahoy/tools/rpi/ahoy.service # $ systemctl --user status ahoy -# $ systemctl --user enable ahoy # $ systemctl --user start ahoy # $ systemctl --user status ahoy # diff --git a/tools/rpi/hoymiles/__init__.py b/tools/rpi/hoymiles/__init__.py index a754ff6d..210bed65 100644 --- a/tools/rpi/hoymiles/__init__.py +++ b/tools/rpi/hoymiles/__init__.py @@ -12,18 +12,26 @@ from datetime import datetime import logging import crcmod from .decoders import * +from os import environ try: # OSI Layer 2 driver for nRF24L01 on Arduino & Raspberry Pi/Linux Devices # https://github.com/nRF24/RF24.git from RF24 import RF24, RF24_PA_MIN, RF24_PA_LOW, RF24_PA_HIGH, RF24_PA_MAX, RF24_250KBPS, RF24_CRC_DISABLED, RF24_CRC_8, RF24_CRC_16 -except ModuleNotFoundError: + if environ.get('TERM') is not None: + print('Using python Module: RF24') +except ModuleNotFoundError as e: + if environ.get('TERM') is not None: + print(f'{e} - try to use module: RF24') try: # Repo for pyRF24 package # https://github.com/nRF24/pyRF24.git from pyrf24 import RF24, RF24_PA_MIN, RF24_PA_LOW, RF24_PA_HIGH, RF24_PA_MAX, RF24_250KBPS, RF24_CRC_DISABLED, RF24_CRC_8, RF24_CRC_16 - except ModuleNotFoundError: - print("Module for RF24 not found - exit") + if environ.get('TERM') is not None: + print(f'{e} - Using python Module: pyrf24') + except ModuleNotFoundError as e: + if environ.get('TERM') is not None: + print(f'{e} - exit') exit() f_crc_m = crcmod.predefined.mkPredefinedCrcFun('modbus') @@ -171,8 +179,19 @@ class ResponseDecoder(ResponseDecoderFactory): command = self.request_command if HOYMILES_DEBUG_LOGGING: - c_datetime = self.time_rx.strftime("%Y-%m-%d %H:%M:%S.%f") - logging.info(f'{c_datetime} model_decoder: {model}Decode{command.upper()}') + if command.upper() == '01': + model_desc = "Firmware version / date" + elif command.upper() == '02': + model_desc = "Inverter generic events log" + elif command.upper() == '0B': + model_desc = "mirco-inverters status data" + elif command.upper() == '0C': + model_desc = "mirco-inverters status data" + elif command.upper() == '11': + model_desc = "Inverter generic events log" + elif command.upper() == '12': + model_desc = "Inverter major events log" + logging.info(f'model_decoder: {model}Decode{command.upper()} - {model_desc}') model_decoders = __import__('hoymiles.decoders') if hasattr(model_decoders, f'{model}Decode{command.upper()}'): diff --git a/tools/rpi/hoymiles/__main__.py b/tools/rpi/hoymiles/__main__.py index 35943eb4..00608a44 100644 --- a/tools/rpi/hoymiles/__main__.py +++ b/tools/rpi/hoymiles/__main__.py @@ -174,12 +174,13 @@ def poll_inverter(inverter, dtu_ser, do_init, retries): response = com.get_payload() payload_ttl = 0 except Exception as e_all: - logging.error(f'Error while retrieving data: {e_all}') + if hoymiles.HOYMILES_TRANSACTION_LOGGING: + logging.error(f'Error while retrieving data: {e_all}') pass # Handle the response data if any if response: - if hoymiles.HOYMILES_DEBUG_LOGGING: + if hoymiles.HOYMILES_TRANSACTION_LOGGING: c_datetime = datetime.now() logging.debug(f'{c_datetime} Payload: ' + hoymiles.hexify_payload(response)) @@ -195,8 +196,7 @@ def poll_inverter(inverter, dtu_ser, do_init, retries): # get decoder object result = decoder.decode() if hoymiles.HOYMILES_DEBUG_LOGGING: - c_datetime = datetime.now() - logging.info(f'{c_datetime} Decoded: {result.__dict__()}') + logging.info(f'Decoded: {result.__dict__()}') # check decoder object for output if isinstance(result, hoymiles.decoders.StatusResponse): @@ -282,6 +282,12 @@ def init_logging(ahoy_config): lvl = logging.WARNING elif level == 'ERROR': lvl = logging.ERROR + elif level == 'FATAL': + lvl = logging.FATAL + if hoymiles.HOYMILES_TRANSACTION_LOGGING and hoymiles.HOYMILES_DEBUG_LOGGING: + lvl = logging.DEBUG + if not hoymiles.HOYMILES_TRANSACTION_LOGGING and not hoymiles.HOYMILES_DEBUG_LOGGING: + lvl = logging.INFO logging.basicConfig(filename=fn, format='%(asctime)s %(levelname)s: %(message)s', datefmt='%Y-%m-%d %H:%M:%S', level=lvl) if __name__ == '__main__': @@ -309,15 +315,15 @@ if __name__ == '__main__': logging.error(f'Failed to load config file {global_config.config_file}: {e_yaml}') sys.exit(1) - # read AHOY configuration file and prepare logging - ahoy_config = dict(cfg.get('ahoy', {})) - init_logging(ahoy_config) - if global_config.log_transactions: hoymiles.HOYMILES_TRANSACTION_LOGGING=True if global_config.verbose: hoymiles.HOYMILES_DEBUG_LOGGING=True + # read AHOY configuration file and prepare logging + ahoy_config = dict(cfg.get('ahoy', {})) + init_logging(ahoy_config) + # Prepare for multiple transceivers, makes them configurable for radio_config in ahoy_config.get('nrf', [{}]): hmradio = hoymiles.HoymilesNRF(**radio_config) diff --git a/tools/rpi/hoymiles/decoders/__init__.py b/tools/rpi/hoymiles/decoders/__init__.py index e27b502d..ff277dbd 100644 --- a/tools/rpi/hoymiles/decoders/__init__.py +++ b/tools/rpi/hoymiles/decoders/__init__.py @@ -99,6 +99,7 @@ class StatusResponse(Response): frequency = None powerfactor = None event_count = None + unpack_error = False def unpack(self, fmt, base): """ @@ -111,6 +112,7 @@ class StatusResponse(Response): """ size = struct.calcsize(fmt) if (len(self.response) < base+size): + self.unpack_error = True logging.error(f'base: {base} size: {size} len: {len(self.response)} fmt: {fmt} rep: {self.response}') return [0] return struct.unpack(fmt, self.response[base:base+size]) @@ -196,7 +198,8 @@ class StatusResponse(Response): data['event_count'] = self.event_count data['time'] = self.time_rx - return data + if not self.unpack_error: + return data class UnknownResponse(Response): """ @@ -334,12 +337,13 @@ class EventsResponse(UnknownResponse): logging.debug(' '.join([f'{byte:02x}' for byte in chunk]) + ': ') - if (len(chunk[0:6]) == 6): - opcode, a_code, a_count, uptime_sec = struct.unpack('>BBHH', chunk[0:6]) - a_text = self.alarm_codes.get(a_code, 'N/A') - logging.debug(f' uptime={timedelta(seconds=uptime_sec)} a_count={a_count} opcode={opcode} a_code={a_code} a_text={a_text}') - else: + if (len(chunk[0:6]) < 6): logging.error(f'length of chunk must be greater or equal 6 bytes: {chunk}') + return + + opcode, a_code, a_count, uptime_sec = struct.unpack('>BBHH', chunk[0:6]) + a_text = self.alarm_codes.get(a_code, 'N/A') + logging.debug(f' uptime={timedelta(seconds=uptime_sec)} a_count={a_count} opcode={opcode} a_code={a_code} a_text={a_text}') dbg = '' for fmt in ['BBHHHHH']: @@ -366,12 +370,15 @@ class HardwareInfoResponse(UnknownResponse): def __dict__(self): """ Base values, availabe in each __dict__ call """ + data = super().__dict__() responce_info = self.response - if (len(responce_info) >= 16): - logging.info(f'HardwareInfoResponse: {struct.unpack(">HHHHHHHH", responce_info)}') - else: - logging.error(f'wrong length of HardwareInfoResponse: {responce_info}') + if (len(self.response) != 16): + logging.error(f'HardwareInfoResponse: data length should be 16 bytes - measured {len(self.response)} bytes') + logging.error(f'HardwareInfoResponse: data: {self.response}') + return data + + logging.info(f'HardwareInfoResponse: {struct.unpack(">HHHHHHHH", self.response[0:16])}') fw_version, fw_build_yyyy, fw_build_mmdd, fw_build_hhmm, hw_id = struct.unpack('>HHHHH', self.response[0:10]) fw_version_maj = int((fw_version / 10000)) @@ -385,7 +392,6 @@ class HardwareInfoResponse(UnknownResponse): f'build at {fw_build_dd:>02}/{fw_build_mm:>02}/{fw_build_yyyy}T{fw_build_HH:>02}:{fw_build_MM:>02}, '\ f'HW revision {hw_id}') - data = super().__dict__() data['FW_ver_maj'] = fw_version_maj data['FW_ver_min'] = fw_version_min data['FW_ver_pat'] = fw_version_pat diff --git a/tools/rpi/hoymiles/outputs.py b/tools/rpi/hoymiles/outputs.py index 8fb55f3e..e4754fbc 100644 --- a/tools/rpi/hoymiles/outputs.py +++ b/tools/rpi/hoymiles/outputs.py @@ -9,6 +9,7 @@ import socket import logging from datetime import datetime, timezone from hoymiles.decoders import StatusResponse, HardwareInfoResponse +from hoymiles import HOYMILES_TRANSACTION_LOGGING, HOYMILES_DEBUG_LOGGING class OutputPluginFactory: def __init__(self, **params): @@ -277,9 +278,10 @@ class VzInverterOutput: self.channels = dict() for channel in config.get('channels', []): - uid = channel.get('uid') + uid = channel.get('uid', None) ctype = channel.get('type') - if uid and ctype: + # if uid and ctype: + if ctype: self.channels[ctype] = uid def store_status(self, data, session): @@ -330,10 +332,17 @@ class VzInverterOutput: def try_publish(self, ts, ctype, value): if not ctype in self.channels: - logging.warning(f'ctype \"{ctype}\" not found in ahoy.yml') + if HOYMILES_DEBUG_LOGGING: + logging.warning(f'ctype \"{ctype}\" not found in ahoy.yml') return + uid = self.channels[ctype] url = f'{self.baseurl}/data/{uid}.json?operation=add&ts={ts}&value={value}' + if uid == None: + if HOYMILES_DEBUG_LOGGING: + logging.warning(f'ctype \"{ctype}\" has no configured uid-value in ahoy.yml') + return + try: r = self.session.get(url) if r.status_code == 404: From 9a0bee831d33aaecd7c707fa1f8d08f672f76c14 Mon Sep 17 00:00:00 2001 From: Knuti_in_Paese Date: Sat, 4 Feb 2023 16:40:50 +0100 Subject: [PATCH 04/32] RPi:specify README.md and collect data from EventsResponse --- tools/rpi/README.md | 26 ++++++++++++++++++------- tools/rpi/hoymiles/decoders/__init__.py | 15 ++++++++++---- 2 files changed, 30 insertions(+), 11 deletions(-) diff --git a/tools/rpi/README.md b/tools/rpi/README.md index 12b6f5ef..13758b08 100644 --- a/tools/rpi/README.md +++ b/tools/rpi/README.md @@ -84,9 +84,14 @@ If there are no error messages on the last step, then the NRF24 Wrapper has been Building RF24 Wrapper for Debian 11 (bullseye) 64 bit operating system ---------------------------------------------------------------------- The description above does not work on Debian 11 (bullseye) 64 bit operating system. -There are 2 possible sollutions to install the RF24 Wrapper. +Please check first, if you have Debian 11 (bullseye) 64 bit operating system installed: + - `uname -a` search for aarch64 + - `lsb_release -d` + - `cat /etc/debian_version` - * `1. solution:` +There are 2 possible solutions to install the RF24 wrapper: + + * `1. Solution:` ```code sudo apt install cmake git python3-dev libboost-python-dev python3-pip python3-rpi.gpio @@ -101,13 +106,13 @@ cd RF24 rm -rf build Makefile.inc ./configure --driver=SPIDEV ``` - * edit `Makefile.inc` with your prefered editor e.g. nano or vi - old: + # edit `Makefile.inc` with your prefered editor e.g. nano or vi + - old: ```code CPUFLAGS=-marm -march=armv6zk -mtune=arm1176jzf-s -mfpu=vfp -mfloat-abi=hard CFLAGS=-marm -march=armv6zk -mtune=arm1176jzf-s -mfpu=vfp -mfloat-abi=hard -Ofast -Wall -pthread ``` - new: + - new: ```code CPUFLAGS= CFLAGS=-Ofast -Wall -pthread @@ -125,8 +130,7 @@ python3 -m pip list #watch for RF24 module - if its there its installed ``` - - * `2. solution:` + * `2. Solution:` ```code sudo apt install git python3-dev libboost-python-dev python3-pip python3-rpi.gpio @@ -135,6 +139,14 @@ cd pyRF24 python3 -m pip install . -v # this step takes about 5 minutes on my RPI-4 ! ``` +If you have problems with your radio module from ahoi, +e.g.: cannot interpret received data, +please try to reduce the speed of the radio module! +Add the following line to your ahoy.yml configuration file in "nrf" section: +* `spispeed: 600000` + + + Required python modules ----------------------- diff --git a/tools/rpi/hoymiles/decoders/__init__.py b/tools/rpi/hoymiles/decoders/__init__.py index ff277dbd..fa80a3b4 100644 --- a/tools/rpi/hoymiles/decoders/__init__.py +++ b/tools/rpi/hoymiles/decoders/__init__.py @@ -327,9 +327,9 @@ class EventsResponse(UnknownResponse): #logging.debug(' payload has valid modbus crc') self.response = self.response[:-2] - status = struct.unpack('>H', self.response[:2])[0] - a_text = self.alarm_codes.get(status, 'N/A') - logging.info (f' Inverter status: {a_text} ({status})') + self.status = struct.unpack('>H', self.response[:2])[0] + self.a_text = self.alarm_codes.get(self.status, 'N/A') + logging.info (f' Inverter status: {self.a_text} ({self.status})') chunk_size = 12 for i_chunk in range(2, len(self.response), chunk_size): @@ -350,6 +350,14 @@ class EventsResponse(UnknownResponse): dbg += f' {fmt:7}: ' + str(struct.unpack('>' + fmt, chunk)) logging.debug(dbg) + def __dict__(self): + """ Base values, availabe in each __dict__ call """ + + data = super().__dict__() + data['inv_stat_num'] = self.status + data['inv_stat_txt'] = self.a_text + return data + class HardwareInfoResponse(UnknownResponse): def __init__(self, *args, **params): super().__init__(*args, **params) @@ -371,7 +379,6 @@ class HardwareInfoResponse(UnknownResponse): """ Base values, availabe in each __dict__ call """ data = super().__dict__() - responce_info = self.response if (len(self.response) != 16): logging.error(f'HardwareInfoResponse: data length should be 16 bytes - measured {len(self.response)} bytes') From 57bc46191c4d93d02c9bec3d223d20f1a9acc2e9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Knuti=5Fin=5FP=C3=A4se?= <122045840+PaeserBastelstube@users.noreply.github.com> Date: Sat, 4 Feb 2023 17:25:58 +0100 Subject: [PATCH 05/32] RPi: README.md format one new section --- tools/rpi/README.md | 40 ++++++++++++++++++++-------------------- 1 file changed, 20 insertions(+), 20 deletions(-) diff --git a/tools/rpi/README.md b/tools/rpi/README.md index 13758b08..d1829706 100644 --- a/tools/rpi/README.md +++ b/tools/rpi/README.md @@ -91,7 +91,7 @@ Please check first, if you have Debian 11 (bullseye) 64 bit operating system ins There are 2 possible solutions to install the RF24 wrapper: - * `1. Solution:` +**__1. Solution:__** ```code sudo apt install cmake git python3-dev libboost-python-dev python3-pip python3-rpi.gpio @@ -106,18 +106,19 @@ cd RF24 rm -rf build Makefile.inc ./configure --driver=SPIDEV ``` - # edit `Makefile.inc` with your prefered editor e.g. nano or vi - - old: -```code - CPUFLAGS=-marm -march=armv6zk -mtune=arm1176jzf-s -mfpu=vfp -mfloat-abi=hard - CFLAGS=-marm -march=armv6zk -mtune=arm1176jzf-s -mfpu=vfp -mfloat-abi=hard -Ofast -Wall -pthread -``` - - new: -```code - CPUFLAGS= - CFLAGS=-Ofast -Wall -pthread -``` - * continue with +> _edit `Makefile.inc` with your prefered editor e.g. nano or vi_ +> +> old: +>```code +> CPUFLAGS=-marm -march=armv6zk -mtune=arm1176jzf-s -mfpu=vfp -mfloat-abi=hard +> CFLAGS=-marm -march=armv6zk -mtune=arm1176jzf-s -mfpu=vfp -mfloat-abi=hard -Ofast -Wall -pthread +>``` +> new: +>```code +> CPUFLAGS= +> CFLAGS=-Ofast -Wall -pthread +>``` +_continue now_ ```code make sudo make install @@ -130,7 +131,7 @@ python3 -m pip list #watch for RF24 module - if its there its installed ``` - * `2. Solution:` +**__2. Solution:__** ```code sudo apt install git python3-dev libboost-python-dev python3-pip python3-rpi.gpio @@ -139,11 +140,10 @@ cd pyRF24 python3 -m pip install . -v # this step takes about 5 minutes on my RPI-4 ! ``` -If you have problems with your radio module from ahoi, -e.g.: cannot interpret received data, -please try to reduce the speed of the radio module! -Add the following line to your ahoy.yml configuration file in "nrf" section: -* `spispeed: 600000` +If you have problems with your radio module from ahoi, e.g.: cannot interpret received data, +please try to reduce the speed of your radio module! +Add the following parameter to your ahoy.yml configuration file in "nrf" section: +`spispeed: 600000` (0.6 MHz) @@ -247,7 +247,7 @@ Todo - Ability to talk to multiple inverters - MQTT gateway - understand channel hopping -- configurable polling interval +- ~~configurable polling interval~~ done: interval ist configurable in ahoy.yml - commands - picture of setup! - python module From 6b3af717fbb3b42239f6de7d04930c8698321eb5 Mon Sep 17 00:00:00 2001 From: Knuti_in_Paese Date: Tue, 14 Feb 2023 10:34:22 +0100 Subject: [PATCH 06/32] RPi:(new)DTU-name,Disco-handler,ext.Error-handling,sun2mqtt Add disconnect handler for influx and volkszaehler. Change spec. Informations on ahoy.service and ahoy.yml.example. Extented Error handling. Send sun-rise and sun-set information to MQTT. --- tools/rpi/ahoy.service | 3 +- tools/rpi/ahoy.yml.example | 13 ++--- tools/rpi/hoymiles/__main__.py | 47 ++++++++++++++---- tools/rpi/hoymiles/decoders/__init__.py | 17 ++++++- tools/rpi/hoymiles/outputs.py | 63 ++++++++++++++++++++----- 5 files changed, 112 insertions(+), 31 deletions(-) diff --git a/tools/rpi/ahoy.service b/tools/rpi/ahoy.service index 4af9ea89..c7be5bb2 100644 --- a/tools/rpi/ahoy.service +++ b/tools/rpi/ahoy.service @@ -6,8 +6,7 @@ # WorkingDirectory (absolute path to your private ahoy dir) # To change other config parameter, please consult systemd documentation # -# To activate this service, create a link with enable and start the ahoy.service -# $ mkdir -p $HOME/.config/systemd/user +# To activate this service, enable and start ahoy.service # $ systemctl --user enable $(pwd)/ahoy/tools/rpi/ahoy.service # $ systemctl --user status ahoy # $ systemctl --user start ahoy diff --git a/tools/rpi/ahoy.yml.example b/tools/rpi/ahoy.yml.example index 928374de..9301067f 100644 --- a/tools/rpi/ahoy.yml.example +++ b/tools/rpi/ahoy.yml.example @@ -31,7 +31,7 @@ ahoy: QoS: 0 Retain: True last_will: - topic: hoymiles/114172220003 # defaults to 'hoymiles/{serial}' + topic: my_DTU_name # Name of DTU - default: hoymiles/{DTU-serial} payload: "LAST-WILL-MESSAGE: Please check my HOST and Process!" # Influx2 output @@ -96,6 +96,7 @@ ahoy: dtu: serial: 99978563001 + name: my_DTU_name inverters: - name: 'balkon' @@ -103,14 +104,14 @@ ahoy: txpower: 'low' # txpower per inverter (min,low,high,max) mqtt: send_raw_enabled: false # allow inject debug data via mqtt - topic: 'hoymiles/114172221234' # defaults to '{inverter-name}/{serial}' + topic: 'hoymiles/114172220003' # defaults to '{inverter-name}/{serial}' strings: # list all available strings - s_name: 'String 1 left' # String 1 name - s_maxpower: 395 # String 1 max power in Wp + s_maxpower: 395 # String 1 max power in inverter - s_name: 'String 2 right' # String 2 name - s_maxpower: 400 # String 2 max power in Wp + s_maxpower: 400 # String 2 max power in inverter - s_name: 'String 3 up' # String 3 name - s_maxpower: 405 # String 3 max power in Wp + s_maxpower: 405 # String 3 max power in inverter - s_name: 'String 4 down' # String 4 name - s_maxpower: 410 # String 4 max power in Wp + s_maxpower: 410 # String 4 max power in inverter diff --git a/tools/rpi/hoymiles/__main__.py b/tools/rpi/hoymiles/__main__.py index 00608a44..7de4a1a2 100644 --- a/tools/rpi/hoymiles/__main__.py +++ b/tools/rpi/hoymiles/__main__.py @@ -33,6 +33,12 @@ def signal_handler(sig_num, frame): if mqtt_client: mqtt_client.disco() + if influx_client: + influx_client.disco() + + if volkszaehler_client: + volkszaehler_client.disco() + sys.exit(0) signal(SIGINT, signal_handler) # Interrupt from keyboard (CTRL + C) @@ -75,7 +81,6 @@ class SunsetHandler: else: logging.info('Sunset disabled.') - def checkWaitForSunrise(self): if not self.suntimes: return @@ -94,6 +99,23 @@ class SunsetHandler: time.sleep(time_to_sleep) logging.info (f'Woke up...') + def sun_status2mqtt(self, dtu_ser, dtu_name): + if not mqtt_client: + return + local_sunrise = self.suntimes.riselocal(datetime.now()).strftime("%d.%m.%YT%H:%M") + local_sunset = self.suntimes.setlocal(datetime.now()).strftime("%d.%m.%YT%H:%M") + local_zone = self.suntimes.setlocal(datetime.now()).tzinfo._key + if self.suntimes: + mqtt_client.info2mqtt({'topic' : f'{dtu_name}/{dtu_ser}'}, \ + {'dis_night_comm' : 'True', \ + 'local_sunrise' : local_sunrise, \ + 'local_sunset' : local_sunset, + 'local_zone' : local_zone}) + else: + mqtt_client.sun_info2mqtt({'sun_topic': f'{dtu_name}/{dtu_ser}'}, \ + {'dis_night_comm': 'False'}) + + def main_loop(ahoy_config): """Main loop""" inverters = [ @@ -101,7 +123,9 @@ def main_loop(ahoy_config): if not inverter.get('disabled', False)] sunset = SunsetHandler(ahoy_config.get('sunset')) - dtu_ser = ahoy_config.get('dtu', {}).get('serial') + dtu_ser = ahoy_config.get('dtu', {}).get('serial', None) + dtu_name = ahoy_config.get('dtu', {}).get('name', 'hoymiles-dtu') + sunset.sun_status2mqtt(dtu_ser, dtu_name) loop_interval = ahoy_config.get('interval', 1) try: @@ -112,6 +136,11 @@ def main_loop(ahoy_config): t_loop_start = time.time() for inverter in inverters: + if not 'name' in inverter: + inverter['name'] = 'hoymiles' + if not 'serial' in inverter: + logging.error("No inverter serial number found in ahoy.yml - exit") + sys.exit(999) if hoymiles.HOYMILES_DEBUG_LOGGING: logging.info(f'Poll inverter name={inverter["name"]} ser={inverter["serial"]}') poll_inverter(inverter, dtu_ser, do_init, 3) @@ -122,8 +151,6 @@ def main_loop(ahoy_config): if time_to_sleep > 0: time.sleep(time_to_sleep) - except KeyboardInterrupt: - sys.exit() except Exception as e: logging.fatal('Exception catched: %s' % e) logging.fatal(traceback.print_exc()) @@ -284,11 +311,11 @@ def init_logging(ahoy_config): lvl = logging.ERROR elif level == 'FATAL': lvl = logging.FATAL - if hoymiles.HOYMILES_TRANSACTION_LOGGING and hoymiles.HOYMILES_DEBUG_LOGGING: - lvl = logging.DEBUG - if not hoymiles.HOYMILES_TRANSACTION_LOGGING and not hoymiles.HOYMILES_DEBUG_LOGGING: - lvl = logging.INFO + if hoymiles.HOYMILES_TRANSACTION_LOGGING: + lvl = logging.DEBUG logging.basicConfig(filename=fn, format='%(asctime)s %(levelname)s: %(message)s', datefmt='%Y-%m-%d %H:%M:%S', level=lvl) + dtu_name = ahoy_config.get('dtu',{}).get('name','hoymiles-dtu') + logging.info(f'start logging for {dtu_name} with level: {logging.root.level}') if __name__ == '__main__': parser = argparse.ArgumentParser(description='Ahoy - Hoymiles solar inverter gateway', prog="hoymiles") @@ -330,14 +357,14 @@ if __name__ == '__main__': # create MQTT - client object mqtt_client = None - mqtt_config = ahoy_config.get('mqtt', {}) + mqtt_config = ahoy_config.get('mqtt', None) if mqtt_config and not mqtt_config.get('disabled', False): from .outputs import MqttOutputPlugin mqtt_client = MqttOutputPlugin(mqtt_config) # create INFLUX - client object influx_client = None - influx_config = ahoy_config.get('influxdb', {}) + influx_config = ahoy_config.get('influxdb', None) if influx_config and not influx_config.get('disabled', False): from .outputs import InfluxOutputPlugin influx_client = InfluxOutputPlugin( diff --git a/tools/rpi/hoymiles/decoders/__init__.py b/tools/rpi/hoymiles/decoders/__init__.py index fa80a3b4..bb32fb07 100644 --- a/tools/rpi/hoymiles/decoders/__init__.py +++ b/tools/rpi/hoymiles/decoders/__init__.py @@ -155,6 +155,7 @@ class StatusResponse(Response): s_exists = False string_id = len(strings) string = {} + string['name'] = self.inv_strings[string_id]['s_name'] for key in self.string_keys: prop = f'dc_{key}_{string_id}' if hasattr(self, prop): @@ -329,7 +330,7 @@ class EventsResponse(UnknownResponse): self.status = struct.unpack('>H', self.response[:2])[0] self.a_text = self.alarm_codes.get(self.status, 'N/A') - logging.info (f' Inverter status: {self.a_text} ({self.status})') + logging.info (f'Inverter status: {self.a_text} ({self.status})') chunk_size = 12 for i_chunk in range(2, len(self.response), chunk_size): @@ -489,6 +490,8 @@ class Hm300Decode0B(StatusResponse): """ String 1 irratiation in percent """ if self.inv_strings is None: return None + if self.inv_strings[0]['s_maxpower'] == 0: + return 0.00 return round(self.unpack('>H', 6)[0]/10/self.inv_strings[0]['s_maxpower']*100, 3) @property @@ -561,6 +564,8 @@ class Hm600Decode0B(StatusResponse): """ String 1 irratiation in percent """ if self.inv_strings is None: return None + if self.inv_strings[0]['s_maxpower'] == 0: + return 0.00 return round(self.unpack('>H', 6)[0]/10/self.inv_strings[0]['s_maxpower']*100, 3) @property @@ -588,6 +593,8 @@ class Hm600Decode0B(StatusResponse): """ String 2 irratiation in percent """ if self.inv_strings is None: return None + if self.inv_strings[1]['s_maxpower'] == 0: + return 0.00 return round(self.unpack('>H', 12)[0]/10/self.inv_strings[1]['s_maxpower']*100, 3) @property @@ -668,6 +675,8 @@ class Hm1200Decode0B(StatusResponse): """ String 1 irratiation in percent """ if self.inv_strings is None: return None + if self.inv_strings[0]['s_maxpower'] == 0: + return 0.00 return round(self.unpack('>H', 8)[0]/10/self.inv_strings[0]['s_maxpower']*100, 3) @property @@ -695,6 +704,8 @@ class Hm1200Decode0B(StatusResponse): """ String 2 irratiation in percent """ if self.inv_strings is None: return None + if self.inv_strings[1]['s_maxpower'] == 0: + return 0.00 return round(self.unpack('>H', 10)[0]/10/self.inv_strings[1]['s_maxpower']*100, 3) @property @@ -722,6 +733,8 @@ class Hm1200Decode0B(StatusResponse): """ String 3 irratiation in percent """ if self.inv_strings is None: return None + if self.inv_strings[2]['s_maxpower'] == 0: + return 0.00 return round(self.unpack('>H', 30)[0]/10/self.inv_strings[2]['s_maxpower']*100, 3) @property @@ -749,6 +762,8 @@ class Hm1200Decode0B(StatusResponse): """ String 4 irratiation in percent """ if self.inv_strings is None: return None + if self.inv_strings[3]['s_maxpower'] == 0: + return 0.00 return round(self.unpack('>H', 32)[0]/10/self.inv_strings[3]['s_maxpower']*100, 3) @property diff --git a/tools/rpi/hoymiles/outputs.py b/tools/rpi/hoymiles/outputs.py index e4754fbc..11971a85 100644 --- a/tools/rpi/hoymiles/outputs.py +++ b/tools/rpi/hoymiles/outputs.py @@ -40,6 +40,7 @@ class InfluxOutputPlugin(OutputPluginFactory): def __init__(self, url, token, **params): """ Initialize InfluxOutputPlugin + https://influxdb-client.readthedocs.io/en/stable/api.html#influxdbclient The following targets must be present in your InfluxDB. This does not automatically create anything for You. @@ -69,8 +70,12 @@ class InfluxOutputPlugin(OutputPluginFactory): self._org = params.get('org', '') self._measurement = params.get('measurement', f'inverter,host={socket.gethostname()}') - client = InfluxDBClient(url, token, bucket=self._bucket) - self.api = client.write_api() + with InfluxDBClient(url, token, bucket=self._bucket) as self.client: + self.api = self.client.write_api() + + def disco(self, **params): + self.client.close() # Shutdown the client + return def store_status(self, response, **params): """ @@ -103,6 +108,9 @@ class InfluxOutputPlugin(OutputPluginFactory): # InfluxDB requires nanoseconds ctime = int(utctime.timestamp() * 1e9) + if HOYMILES_DEBUG_LOGGING: + logging.info(f'InfluxDB: utctime: {utctime}') + # AC Data phase_id = 0 for phase in data['phases']: @@ -136,6 +144,9 @@ class InfluxOutputPlugin(OutputPluginFactory): data_stack.append(f'{measurement},type=YieldToday value={data["yield_today"]/1000:.3f} {ctime}') data_stack.append(f'{measurement},type=Efficiency value={data["efficiency"]:.2f} {ctime}') + if HOYMILES_DEBUG_LOGGING: + #logging.debug(f'INFLUX data to DB: {data_stack}') + pass self.api.write(self._bucket, self._org, data_stack) class MqttOutputPlugin(OutputPluginFactory): @@ -197,6 +208,12 @@ class MqttOutputPlugin(OutputPluginFactory): def disco(self, **params): self.client.loop_stop() # Stop loop self.client.disconnect() # disconnect + return + + def info2mqtt(self, mqtt_topic, mqtt_data): + for mqtt_key in mqtt_data: + self.client.publish(f'{mqtt_topic["topic"]}/{mqtt_key}', mqtt_data[mqtt_key], self.qos, self.ret) + return def store_status(self, response, **params): """ @@ -210,13 +227,18 @@ class MqttOutputPlugin(OutputPluginFactory): """ data = response.__dict__() - topic = f'{data.get("inverter_name", "hoymiles")}/{data.get("inverter_ser", None)}' + topic = params.get('topic', None) + if not topic: + topic = f'{data.get("inverter_name", "hoymiles")}/{data.get("inverter_ser", None)}' + + if HOYMILES_DEBUG_LOGGING: + logging.info(f'MQTT-topic: {topic} data-type: {type(response)}') if isinstance(response, StatusResponse): # Global Head if data['time'] is not None: - self.client.publish(f'{topic}/time', data['time'].strftime("%d.%m.%y - %H:%M:%S"), self.qos, self.ret) + self.client.publish(f'{topic}/time', data['time'].strftime("%d.%m.%YT%H:%M:%S"), self.qos, self.ret) # AC Data phase_id = 0 @@ -234,12 +256,16 @@ class MqttOutputPlugin(OutputPluginFactory): string_id = 0 string_sum_power = 0 for string in data['strings']: - self.client.publish(f'{topic}/emeter-dc/{string_id}/voltage', string['voltage'], self.qos, self.ret) - self.client.publish(f'{topic}/emeter-dc/{string_id}/current', string['current'], self.qos, self.ret) - self.client.publish(f'{topic}/emeter-dc/{string_id}/power', string['power'], self.qos, self.ret) - self.client.publish(f'{topic}/emeter-dc/{string_id}/YieldDay', string['energy_daily'], self.qos, self.ret) - self.client.publish(f'{topic}/emeter-dc/{string_id}/YieldTotal', string['energy_total']/1000, self.qos, self.ret) - self.client.publish(f'{topic}/emeter-dc/{string_id}/Irradiation', string['irradiation'], self.qos, self.ret) + if 'name' in string: + string_name = string['name'].replace(" ","_") + else: + string_name = string_id + self.client.publish(f'{topic}/emeter-dc/{string_name}/voltage', string['voltage'], self.qos, self.ret) + self.client.publish(f'{topic}/emeter-dc/{string_name}/current', string['current'], self.qos, self.ret) + self.client.publish(f'{topic}/emeter-dc/{string_name}/power', string['power'], self.qos, self.ret) + self.client.publish(f'{topic}/emeter-dc/{string_name}/YieldDay', string['energy_daily'], self.qos, self.ret) + self.client.publish(f'{topic}/emeter-dc/{string_name}/YieldTotal', string['energy_total']/1000, self.qos, self.ret) + self.client.publish(f'{topic}/emeter-dc/{string_name}/Irradiation', string['irradiation'], self.qos, self.ret) string_id = string_id + 1 string_sum_power += string['power'] @@ -297,6 +323,9 @@ class VzInverterOutput: ts = int(round(data['time'].timestamp() * 1000)) + if HOYMILES_DEBUG_LOGGING: + logging.info(f'Volkszaehler-Timestamp: {ts}') + # AC Data phase_id = 0 for phase in data['phases']: @@ -329,6 +358,7 @@ class VzInverterOutput: if data['yield_today'] is not None: self.try_publish(ts, f'yield_today', data['yield_today']) self.try_publish(ts, f'efficiency', data['efficiency']) + return def try_publish(self, ts, ctype, value): if not ctype in self.channels: @@ -340,9 +370,12 @@ class VzInverterOutput: url = f'{self.baseurl}/data/{uid}.json?operation=add&ts={ts}&value={value}' if uid == None: if HOYMILES_DEBUG_LOGGING: - logging.warning(f'ctype \"{ctype}\" has no configured uid-value in ahoy.yml') + logging.debug(f'ctype \"{ctype}\" has no configured uid-value in ahoy.yml') return + if HOYMILES_DEBUG_LOGGING: + logging.debug(f'VZ-url: {url}') + try: r = self.session.get(url) if r.status_code == 404: @@ -353,6 +386,7 @@ class VzInverterOutput: raise ValueError(f'Transmit result {url}') except ConnectionError as e: raise ValueError(f'Could not connect VZ-DB {type(e)} {e.keys()}') + return class VolkszaehlerOutputPlugin(OutputPluginFactory): def __init__(self, config, **params): @@ -373,13 +407,17 @@ class VolkszaehlerOutputPlugin(OutputPluginFactory): exit(1) self.session = requests.Session() - self.inverters = dict() + self.inverters = dict() for inverterconfig in config.get('inverters', []): serial = inverterconfig.get('serial') output = VzInverterOutput(inverterconfig, self.session) self.inverters[serial] = output + def disco(self, **params): + self.session.close() # closing the connection + return + def store_status(self, response, **params): """ Publish StatusResponse object @@ -404,3 +442,4 @@ class VolkszaehlerOutputPlugin(OutputPluginFactory): output.store_status(data, self.session) except ValueError as e: logging.warning('Could not send data to volkszaehler instance: %s' % e) + return From 4f0d365211af8d8b8afc7b83bb1158e6e14e4cd2 Mon Sep 17 00:00:00 2001 From: lumapu Date: Sat, 25 Feb 2023 01:40:25 +0100 Subject: [PATCH 07/32] improved html and navi, navi is visible even when API dies #660 reduced maximum allowed JSON size for API to 6000Bytes #660 small fix: output command at `prepareDevInformCmd` #692 improved inverter handling for MQTT #671 --- .gitignore | 1 + src/CHANGES.md | 6 ++ src/app.cpp | 15 ++-- src/config/settings.h | 2 +- src/defines.h | 2 +- src/hm/hmPayload.h | 3 +- src/publisher/pubMqtt.h | 99 +++++++++++++++------------ src/web/RestApi.h | 61 ++--------------- src/web/html/api.js | 27 ++++---- src/web/html/convert.py | 110 ++++++++++++++++++++++++++---- src/web/html/includes/footer.html | 16 +++++ src/web/html/includes/header.html | 3 + src/web/html/includes/nav.html | 27 ++++++++ src/web/html/index.html | 61 +++-------------- src/web/html/login.html | 25 +------ src/web/html/serial.html | 35 ++-------- src/web/html/setup.html | 39 ++--------- src/web/html/style.css | 8 ++- src/web/html/system.html | 35 ++-------- src/web/html/update.html | 43 ++---------- src/web/html/visualization.html | 36 ++-------- 21 files changed, 278 insertions(+), 376 deletions(-) create mode 100644 src/web/html/includes/footer.html create mode 100644 src/web/html/includes/header.html create mode 100644 src/web/html/includes/nav.html diff --git a/.gitignore b/.gitignore index ed5e9538..d6a35860 100644 --- a/.gitignore +++ b/.gitignore @@ -6,6 +6,7 @@ .vscode/extensions.json src/config/config_override.h src/web/html/h/* +src/web/html/tmp/* /**/Debug /**/v16/* *.db diff --git a/src/CHANGES.md b/src/CHANGES.md index 38f7a57e..678e872c 100644 --- a/src/CHANGES.md +++ b/src/CHANGES.md @@ -2,6 +2,12 @@ (starting from release version `0.5.66`) +## 0.5.91 +* improved html and navi, navi is visible even when API dies #660 +* reduced maximum allowed JSON size for API to 6000Bytes #660 +* small fix: output command at `prepareDevInformCmd` #692 +* improved inverter handling #671 + ## 0.5.90 * merged PR #684, #698, #705 * webserial minor overflow fix #660 diff --git a/src/app.cpp b/src/app.cpp index f3b175b6..4847b512 100644 --- a/src/app.cpp +++ b/src/app.cpp @@ -21,12 +21,6 @@ void app::setup() { resetSystem(); - /*DBGPRINTLN("--- start"); - DBGPRINTLN(String(ESP.getFreeHeap())); - DBGPRINTLN(String(ESP.getHeapFragmentation())); - DBGPRINTLN(String(ESP.getMaxFreeBlockSize()));*/ - - mSettings.setup(); mSettings.getPtr(mConfig); DPRINT(DBG_INFO, F("Settings valid: ")); @@ -50,6 +44,7 @@ void app::setup() { #endif mSys.addInverters(&mConfig->inst); + mPayload.setup(this, &mSys, &mStat, mConfig->nrf.maxRetransPerPyld, &mTimestamp); mPayload.enableSerialDebug(mConfig->serial.debug); mPayload.addPayloadListener(std::bind(&app::payloadEventListener, this, std::placeholders::_1)); @@ -57,10 +52,10 @@ void app::setup() { mMiPayload.setup(this, &mSys, &mStat, mConfig->nrf.maxRetransPerPyld, &mTimestamp); mMiPayload.enableSerialDebug(mConfig->serial.debug); - /*DBGPRINTLN("--- after payload"); + DBGPRINTLN("--- after payload"); DBGPRINTLN(String(ESP.getFreeHeap())); DBGPRINTLN(String(ESP.getHeapFragmentation())); - DBGPRINTLN(String(ESP.getMaxFreeBlockSize()));*/ + DBGPRINTLN(String(ESP.getMaxFreeBlockSize())); if(!mSys.Radio.isChipConnected()) DPRINTLN(DBG_WARN, F("WARNING! your NRF24 module can't be reached, check the wiring")); @@ -90,10 +85,10 @@ void app::setup() { regularTickers(); - /*DBGPRINTLN("--- end setup"); + DBGPRINTLN("--- end setup"); DBGPRINTLN(String(ESP.getFreeHeap())); DBGPRINTLN(String(ESP.getHeapFragmentation())); - DBGPRINTLN(String(ESP.getMaxFreeBlockSize()));*/ + DBGPRINTLN(String(ESP.getMaxFreeBlockSize())); } //----------------------------------------------------------------------------- diff --git a/src/config/settings.h b/src/config/settings.h index 527dea72..0528a902 100644 --- a/src/config/settings.h +++ b/src/config/settings.h @@ -250,7 +250,7 @@ class settings { return false; } - DynamicJsonDocument json(4500); + DynamicJsonDocument json(5500); JsonObject root = json.to(); jsonWifi(root.createNestedObject(F("wifi")), true); jsonNrf(root.createNestedObject(F("nrf")), true); diff --git a/src/defines.h b/src/defines.h index 36471ae0..8bd4a04a 100644 --- a/src/defines.h +++ b/src/defines.h @@ -13,7 +13,7 @@ //------------------------------------- #define VERSION_MAJOR 0 #define VERSION_MINOR 5 -#define VERSION_PATCH 90 +#define VERSION_PATCH 91 //------------------------------------- typedef struct { diff --git a/src/hm/hmPayload.h b/src/hm/hmPayload.h index 7a05bee3..245a769e 100644 --- a/src/hm/hmPayload.h +++ b/src/hm/hmPayload.h @@ -157,7 +157,8 @@ class HmPayload { uint8_t cmd = iv->getQueuedCmd(); DPRINT(DBG_INFO, F("(#")); DBGPRINT(String(iv->id)); - DBGPRINT(F(") prepareDevInformCmd")); // + String(cmd, HEX)); + DBGPRINT(F(") prepareDevInformCmd 0x")); + DBGPRINTLN(String(cmd, HEX)); mSys->Radio.prepareDevInformCmd(iv->radioId.u64, cmd, mPayload[iv->id].ts, iv->alarmMesIndex, false); mPayload[iv->id].txCmd = cmd; } diff --git a/src/publisher/pubMqtt.h b/src/publisher/pubMqtt.h index 113292b8..78478536 100644 --- a/src/publisher/pubMqtt.h +++ b/src/publisher/pubMqtt.h @@ -406,7 +406,7 @@ class PubMqtt { return (pos >= DEVICE_CLS_ASSIGN_LIST_LEN) ? NULL : stateClasses[deviceFieldAssignment[pos].stateClsId]; } - bool processIvStatus() { + bool processIvStatus() { // returns true if any inverter is available bool allAvail = true; // shows if all enabled inverters are available bool anyAvail = false; // shows if at least one enabled inverter is available @@ -419,17 +419,19 @@ class PubMqtt { iv = mSys->getInverterByPos(id); if (NULL == iv) continue; // skip to next inverter + if (!iv->config->enabled) + continue; // skip to next inverter rec = iv->getRecordStruct(RealTimeRunData_Debug); // inverter status uint8_t status = MQTT_STATUS_NOT_AVAIL_NOT_PROD; - if (iv->config->enabled) { - if (iv->isAvailable(*mUtcTimestamp)) - status = (iv->isProducing(*mUtcTimestamp)) ? MQTT_STATUS_AVAIL_PROD : MQTT_STATUS_AVAIL_NOT_PROD; - else // inverter is enabled but not available - allAvail = false; + if (iv->isAvailable(*mUtcTimestamp)) { + anyAvail = true; + status = (iv->isProducing(*mUtcTimestamp)) ? MQTT_STATUS_AVAIL_PROD : MQTT_STATUS_AVAIL_NOT_PROD; } + else // inverter is enabled but not available + allAvail = false; if(mLastIvState[id] != status) { // if status changed from producing to not producing send last data immediately @@ -439,11 +441,11 @@ class PubMqtt { mLastIvState[id] = status; changed = true; - snprintf(topic, 32 + MAX_NAME_LENGTH, "%s/%s", iv->config->name, mqttStr[MQTT_STR_AVAILABLE]); + snprintf(topic, 32 + MAX_NAME_LENGTH, "%s/available", iv->config->name); snprintf(val, 40, "%d", status); publish(topic, val, true); - snprintf(topic, 32 + MAX_NAME_LENGTH, "%s/%s", iv->config->name, mqttStr[MQTT_STR_LAST_SUCCESS]); + snprintf(topic, 32 + MAX_NAME_LENGTH, "%s/last_success", iv->config->name); snprintf(val, 40, "%d", iv->getLastTs(rec)); publish(topic, val, true); } @@ -451,7 +453,7 @@ class PubMqtt { if(changed) { snprintf(val, 32, "%d", ((allAvail) ? MQTT_STATUS_ONLINE : ((anyAvail) ? MQTT_STATUS_PARTIAL : MQTT_STATUS_OFFLINE))); - publish(subtopics[MQTT_STATUS], val, true); + publish("status", val, true); } return anyAvail; @@ -474,24 +476,26 @@ class PubMqtt { char topic[7 + MQTT_TOPIC_LEN], val[40]; record_t<> *rec = iv->getRecordStruct(curInfoCmd); - for (uint8_t i = 0; i < rec->length; i++) { - bool retained = false; - if (curInfoCmd == RealTimeRunData_Debug) { - switch (rec->assign[i].fieldId) { - case FLD_YT: - case FLD_YD: - if ((rec->assign[i].ch == CH0) && (!iv->isProducing(*mUtcTimestamp))) // avoids returns to 0 on restart - continue; - retained = true; - break; + if (iv->getLastTs(rec) > 0) { + for (uint8_t i = 0; i < rec->length; i++) { + bool retained = false; + if (curInfoCmd == RealTimeRunData_Debug) { + switch (rec->assign[i].fieldId) { + case FLD_YT: + case FLD_YD: + if ((rec->assign[i].ch == CH0) && (!iv->isProducing(*mUtcTimestamp))) // avoids returns to 0 on restart + continue; + retained = true; + break; + } } - } - snprintf(topic, 32 + MAX_NAME_LENGTH, "%s/ch%d/%s", iv->config->name, rec->assign[i].ch, fields[rec->assign[i].fieldId]); - snprintf(val, 40, "%g", ah::round3(iv->getValue(i, rec))); - publish(topic, val, retained); + snprintf(topic, 32 + MAX_NAME_LENGTH, "%s/ch%d/%s", iv->config->name, rec->assign[i].ch, fields[rec->assign[i].fieldId]); + snprintf(val, 40, "%g", ah::round3(iv->getValue(i, rec))); + publish(topic, val, retained); - yield(); + yield(); + } } } @@ -512,42 +516,49 @@ class PubMqtt { uint8_t curInfoCmd = mSendList.front(); if ((curInfoCmd != RealTimeRunData_Debug) || !RTRDataHasBeenSent) { // send RTR Data only once + bool sendTotals = (curInfoCmd == RealTimeRunData_Debug); + for (uint8_t id = 0; id < mSys->getNumInverters(); id++) { Inverter<> *iv = mSys->getInverterByPos(id); if (NULL == iv) continue; // skip to next inverter + if (!iv->config->enabled) + continue; // skip to next inverter // send RTR Data only if status is available - if ((curInfoCmd != RealTimeRunData_Debug) || (MQTT_STATUS_AVAIL_PROD == mLastIvState[id])) + if ((curInfoCmd != RealTimeRunData_Debug) || (MQTT_STATUS_NOT_AVAIL_NOT_PROD != mLastIvState[id])) sendData(iv, curInfoCmd); // calculate total values for RealTimeRunData_Debug - if (curInfoCmd == RealTimeRunData_Debug) { + if (sendTotals) { record_t<> *rec = iv->getRecordStruct(curInfoCmd); - for (uint8_t i = 0; i < rec->length; i++) { - if (CH0 == rec->assign[i].ch) { - switch (rec->assign[i].fieldId) { - case FLD_PAC: - total[0] += iv->getValue(i, rec); - break; - case FLD_YT: - total[1] += iv->getValue(i, rec); - break; - case FLD_YD: - total[2] += iv->getValue(i, rec); - break; - case FLD_PDC: - total[3] += iv->getValue(i, rec); - break; + sendTotals &= (iv->getLastTs(rec) > 0); + if (sendTotals) { + for (uint8_t i = 0; i < rec->length; i++) { + if (CH0 == rec->assign[i].ch) { + switch (rec->assign[i].fieldId) { + case FLD_PAC: + total[0] += iv->getValue(i, rec); + break; + case FLD_YT: + total[1] += iv->getValue(i, rec); + break; + case FLD_YD: + total[2] += iv->getValue(i, rec); + break; + case FLD_PDC: + total[3] += iv->getValue(i, rec); + break; + } } } } - yield(); } + yield(); } - if (curInfoCmd == RealTimeRunData_Debug) { + if (sendTotals) { uint8_t fieldId; for (uint8_t i = 0; i < 4; i++) { switch (i) { @@ -565,7 +576,7 @@ class PubMqtt { fieldId = FLD_PDC; break; } - snprintf(topic, 32 + MAX_NAME_LENGTH, "%s/%s", mqttStr[MQTT_STR_TOTAL], fields[fieldId]); + snprintf(topic, 32 + MAX_NAME_LENGTH, "total/%s", fields[fieldId]); snprintf(val, 40, "%g", ah::round3(total[i])); publish(topic, val, true); } diff --git a/src/web/RestApi.h b/src/web/RestApi.h index 3dcbc481..1193116f 100644 --- a/src/web/RestApi.h +++ b/src/web/RestApi.h @@ -71,7 +71,7 @@ class RestApi { mHeapFrag = ESP.getHeapFragmentation(); #endif - AsyncJsonResponse* response = new AsyncJsonResponse(false, 8192); + AsyncJsonResponse* response = new AsyncJsonResponse(false, 6000); JsonObject root = response->getRoot(); String path = request->url().substring(5); @@ -83,7 +83,6 @@ class RestApi { else if(path == "reboot") getReboot(root); else if(path == "statistics") getStatistics(root); else if(path == "inverter/list") getInverterList(root); - else if(path == "menu") getMenu(root); else if(path == "index") getIndex(root); else if(path == "setup") getSetup(root); else if(path == "setup/networks") getNetworks(root); @@ -183,10 +182,13 @@ class RestApi { } void getGeneric(JsonObject obj) { - obj[F("version")] = String(mApp->getVersion()); obj[F("build")] = String(AUTO_GIT_HASH); obj[F("wifi_rssi")] = (WiFi.status() != WL_CONNECTED) ? 0 : WiFi.RSSI(); obj[F("ts_uptime")] = mApp->getUptime(); + obj[F("menu_prot")] = mApp->getProtection(); + obj[F("menu_maskH")] = ((mConfig->sys.protectionMask >> 8) & 0xff); + obj[F("menu_maskL")] = ((mConfig->sys.protectionMask ) & 0xff); + obj[F("menu_protEn")] = (bool) (strlen(mConfig->sys.adminPwd) > 0); #if defined(ESP32) obj[F("esp_type")] = F("ESP32"); @@ -244,7 +246,6 @@ class RestApi { } void getHtmlSystem(JsonObject obj) { - getMenu(obj.createNestedObject(F("menu"))); getSysInfo(obj.createNestedObject(F("system"))); getGeneric(obj.createNestedObject(F("generic"))); obj[F("html")] = F("Factory Reset

Reboot"); @@ -252,7 +253,6 @@ class RestApi { } void getHtmlLogout(JsonObject obj) { - getMenu(obj.createNestedObject(F("menu"))); getGeneric(obj.createNestedObject(F("generic"))); obj[F("refresh")] = 3; obj[F("refresh_url")] = "/"; @@ -260,7 +260,6 @@ class RestApi { } void getHtmlSave(JsonObject obj) { - getMenu(obj.createNestedObject(F("menu"))); getGeneric(obj.createNestedObject(F("generic"))); obj[F("refresh")] = 2; obj[F("refresh_url")] = "/setup"; @@ -268,7 +267,6 @@ class RestApi { } void getReboot(JsonObject obj) { - getMenu(obj.createNestedObject(F("menu"))); getGeneric(obj.createNestedObject(F("generic"))); obj[F("refresh")] = 10; obj[F("refresh_url")] = "/"; @@ -377,54 +375,9 @@ class RestApi { obj[F("pinDisp1")] = mConfig->plugin.display.pin1; } - void getMenu(JsonObject obj) { - uint8_t i = 0; - uint16_t mask = (mApp->getProtection()) ? mConfig->sys.protectionMask : 0; - if(!CHECK_MASK(mask, PROT_MASK_LIVE)) { - obj[F("name")][i] = "Live"; - obj[F("link")][i++] = "/live"; - } - if(!CHECK_MASK(mask, PROT_MASK_SERIAL)) { - obj[F("name")][i] = "Serial / Control"; - obj[F("link")][i++] = "/serial"; - } - if(!CHECK_MASK(mask, PROT_MASK_SETUP)) { - obj[F("name")][i] = "Settings"; - obj[F("link")][i++] = "/setup"; - } - obj[F("name")][i++] = "-"; - obj[F("name")][i] = "REST API"; - obj[F("link")][i] = "/api"; - obj[F("trgt")][i++] = "_blank"; - obj[F("name")][i++] = "-"; - if(!CHECK_MASK(mask, PROT_MASK_UPDATE)) { - obj[F("name")][i] = "Update"; - obj[F("link")][i++] = "/update"; - } - if(!CHECK_MASK(mask, PROT_MASK_SYSTEM)) { - obj[F("name")][i] = "System"; - obj[F("link")][i++] = "/system"; - } - obj[F("name")][i++] = "-"; - obj[F("name")][i] = "Documentation"; - obj[F("link")][i] = "https://ahoydtu.de"; - obj[F("trgt")][i++] = "_blank"; - if(strlen(mConfig->sys.adminPwd) > 0) { - obj[F("name")][i++] = "-"; - if(mApp->getProtection()) { - obj[F("name")][i] = "Login"; - obj[F("link")][i++] = "/login"; - } else { - obj[F("name")][i] = "Logout"; - obj[F("link")][i++] = "/logout"; - } - } - } void getIndex(JsonObject obj) { - getMenu(obj.createNestedObject(F("menu"))); getGeneric(obj.createNestedObject(F("generic"))); - obj[F("ts_now")] = mApp->getTimestamp(); obj[F("ts_sunrise")] = mApp->getSunrise(); obj[F("ts_sunset")] = mApp->getSunset(); @@ -473,10 +426,9 @@ class RestApi { } void getSetup(JsonObject obj) { - getMenu(obj.createNestedObject(F("menu"))); getGeneric(obj.createNestedObject(F("generic"))); getSysInfo(obj.createNestedObject(F("system"))); - getInverterList(obj.createNestedObject(F("inverter"))); + //getInverterList(obj.createNestedObject(F("inverter"))); getMqtt(obj.createNestedObject(F("mqtt"))); getNtp(obj.createNestedObject(F("ntp"))); getSun(obj.createNestedObject(F("sun"))); @@ -492,7 +444,6 @@ class RestApi { } void getLive(JsonObject obj) { - getMenu(obj.createNestedObject(F("menu"))); getGeneric(obj.createNestedObject(F("generic"))); JsonArray invArr = obj.createNestedArray(F("inverter")); obj["refresh_interval"] = mConfig->nrf.sendInterval; diff --git a/src/web/html/api.js b/src/web/html/api.js index 2e1ffb2b..0da4e094 100644 --- a/src/web/html/api.js +++ b/src/web/html/api.js @@ -38,18 +38,21 @@ function topnav() { toggle("topnav"); } -function parseMenu(obj) { - var e = document.getElementById("topnav"); - e.innerHTML = ""; - for(var i = 0; i < obj["name"].length; i ++) { - if(obj["name"][i] == "-") - e.appendChild(span("", ["seperator"])); - else { - var l = link(obj["link"][i], obj["name"][i], obj["trgt"][i]); - if(obj["link"][i] == window.location.pathname) - l.classList.add("active"); - e.appendChild(l); - } +function parseNav(obj) { + for(i = 0; i < 7; i++) { + var l = document.getElementById("nav"+i); + if(window.location.pathname == "/" + l.href.split('/').pop()) + l.classList.add("active"); + + if(obj["menu_protEn"]) { + if(obj["menu_prot"]) { + if((((obj["menu_mask"] >> i) & 0x01) == 0x01) || (1 == i)) + l.classList.remove("hide"); + + } else if(0 == i) + l.classList.remove("hide"); + } else if(i > 1) + l.classList.remove("hide"); } } diff --git a/src/web/html/convert.py b/src/web/html/convert.py index 4a8f1f32..22a600ce 100644 --- a/src/web/html/convert.py +++ b/src/web/html/convert.py @@ -2,10 +2,82 @@ import re import os import gzip import glob - +import shutil +import pkg_resources +from datetime import date from pathlib import Path +from dulwich import porcelain + +required_pkgs = {'dulwich'} +installed_pkgs = {pkg.key for pkg in pkg_resources.working_set} +missing_pkgs = required_pkgs - installed_pkgs + +if missing_pkgs: + env.Execute('"$PYTHONEXE" -m pip install dulwich') + + +def get_git_sha(): + try: + build_version = porcelain.describe('../../../') # refers to the repository root dir + except: + build_version = "g0000000" + + build_flag = "-D AUTO_GIT_HASH=\\\"" + build_version[1:] + "\\\"" + #print ("Firmware Revision: " + build_version) + return (build_flag) + +def readVersion(path): + f = open(path, "r") + lines = f.readlines() + f.close() + + today = date.today() + search = ["_MAJOR", "_MINOR", "_PATCH"] + version = today.strftime("%y%m%d") + "_ahoy_" + ver = "" + for line in lines: + if(line.find("VERSION_") != -1): + for s in search: + p = line.find(s) + if(p != -1): + version += line[p+13:].rstrip() + "." + ver += line[p+13:].rstrip() + "." + return ver[:-1] + +def htmlParts(file, header, nav, footer, version): + p = ""; + f = open(file, "r") + lines = f.readlines() + f.close(); -def convert2Header(inFile): + f = open(header, "r") + h = f.read().strip() + f.close() + + f = open(nav, "r") + n = f.read().strip() + f.close() + + f = open(footer, "r") + fo = f.read().strip() + f.close() + + for line in lines: + line = line.replace("{#HTML_HEADER}", h) + line = line.replace("{#HTML_NAV}", n) + line = line.replace("{#HTML_FOOTER}", fo) + p += line + + #placeholders + link = 'GIT SHA: ' + get_git_sha() + ' :: ' + version + '' + p = p.replace("{#VERSION}", version) + p = p.replace("{#VERSION_GIT}", link) + f = open("tmp/" + file, "w") + f.write(p); + f.close(); + return p + +def convert2Header(inFile, version): fileType = inFile.split(".")[1] define = inFile.split(".")[0].upper() define2 = inFile.split(".")[1].upper() @@ -17,14 +89,19 @@ def convert2Header(inFile): Path("html/h").mkdir(exist_ok=True) else: outName = "h/" + inFileVarName + ".h" - Path("h").mkdir(exist_ok=True) + data = "" if fileType == "ico": f = open(inFile, "rb") + data = f.read() + f.close() else: - f = open(inFile, "r") - data = f.read() - f.close() + if fileType == "html": + data = htmlParts(inFile, "includes/header.html", "includes/nav.html", "includes/footer.html", version) + else: + f = open(inFile, "r") + data = f.read() + f.close() if fileType == "css": data = data.replace('\n', '') @@ -53,13 +130,17 @@ def convert2Header(inFile): f.close() # delete all files in the 'h' dir -dir = 'h' +wd = 'h' if os.getcwd()[-4:] != "html": - dir = "web/html/" + dir + wd = "web/html/" + wd -if os.path.exists(dir): - for f in os.listdir(dir): - os.remove(os.path.join(dir, f)) +if os.path.exists(wd): + for f in os.listdir(wd): + os.remove(os.path.join(wd, f)) +wd += "/tmp" +if os.path.exists(wd): + for f in os.listdir(wd): + os.remove(os.path.join(wd, f)) # grab all files with following extensions if os.getcwd()[-4:] != "html": @@ -69,6 +150,11 @@ files_grabbed = [] for files in types: files_grabbed.extend(glob.glob(files)) +Path("h").mkdir(exist_ok=True) +Path("tmp").mkdir(exist_ok=True) # created to check if webpages are valid with all replacements +shutil.copyfile("style.css", "tmp/style.css") +version = readVersion("../../defines.h") + # go throw the array for val in files_grabbed: - convert2Header(val) + convert2Header(val, version) diff --git a/src/web/html/includes/footer.html b/src/web/html/includes/footer.html new file mode 100644 index 00000000..9aa1cf3c --- /dev/null +++ b/src/web/html/includes/footer.html @@ -0,0 +1,16 @@ + diff --git a/src/web/html/includes/header.html b/src/web/html/includes/header.html new file mode 100644 index 00000000..0531efb6 --- /dev/null +++ b/src/web/html/includes/header.html @@ -0,0 +1,3 @@ + + + diff --git a/src/web/html/includes/nav.html b/src/web/html/includes/nav.html new file mode 100644 index 00000000..97d05e74 --- /dev/null +++ b/src/web/html/includes/nav.html @@ -0,0 +1,27 @@ +
+ AhoyDTU + + + + + + +
+
+ + diff --git a/src/web/html/index.html b/src/web/html/index.html index 6bb58432..1c997340 100644 --- a/src/web/html/index.html +++ b/src/web/html/index.html @@ -2,36 +2,12 @@ Index - - - + {#HTML_HEADER} -
- AhoyDTU - - - - - -
-
-
+ {#HTML_NAV}
-

Uptime:
ESP-Time: @@ -60,22 +36,7 @@

- + {#HTML_FOOTER} + {#HTML_HEADER}
@@ -18,25 +16,6 @@
- - + {#HTML_FOOTER} diff --git a/src/web/html/serial.html b/src/web/html/serial.html index 71388c00..b158d170 100644 --- a/src/web/html/serial.html +++ b/src/web/html/serial.html @@ -2,21 +2,10 @@ Serial Console - - - + {#HTML_HEADER} -
- AhoyDTU - - - - - -
-
-
+ {#HTML_NAV}
@@ -53,22 +42,7 @@
- + {#HTML_FOOTER} + {#HTML_HEADER} -
- AhoyDTU - - - - - -
-
-
+ {#HTML_NAV}
@@ -224,22 +213,7 @@
- + {#HTML_FOOTER} + {#HTML_HEADER} -
- AhoyDTU - - - - - -
-
-
+ {#HTML_NAV} 

@@ -26,25 +15,10 @@
                 

             

         

-        
+        {#HTML_FOOTER}
         
+        {#HTML_HEADER}
     
     
-        

-            AhoyDTU
-            
-                
-                
-                
-            
-            

-            

-        

+        {#HTML_NAV}
         

             

                 

@@ -25,43 +14,21 @@
                 

             

         

-        
+        {#HTML_FOOTER}
         
     
 
diff --git a/src/web/html/visualization.html b/src/web/html/visualization.html
index c278dd25..a1d11bcc 100644
--- a/src/web/html/visualization.html
+++ b/src/web/html/visualization.html
@@ -2,50 +2,24 @@
 
     
         Live
-        
-        
+        {#HTML_HEADER}
         
-        
     
     
-        

-            AhoyDTU
-            
-                
-                
-                
-            
-            

-            

-        

+        {#HTML_NAV}
         

             

                 

                 
Every  seconds the values are updated

             

         

-        
+        {#HTML_FOOTER}
         
diff --git a/src/web/html/includes/nav.html b/src/web/html/includes/nav.html
index 95ba005d..f1e1a2bd 100644
--- a/src/web/html/includes/nav.html
+++ b/src/web/html/includes/nav.html
@@ -6,15 +6,15 @@
         
     
     

-        Live
-        Serial / Control
-        Settings
+        Live
+        Serial / Control
+        Settings
         
-        Update
-        System
+        Update
+        System
         
-        REST API
-        Documentation
+        REST API
+        Documentation
         
         Login
         Logout
diff --git a/src/web/html/index.html b/src/web/html/index.html
index 1c997340..27a46a9c 100644
--- a/src/web/html/index.html
+++ b/src/web/html/index.html
@@ -67,12 +67,8 @@
             }
 
             function parseGeneric(obj) {
-                // Disclaimer
-                //if(obj["disclaimer"] == false) sessionStorage.setItem("gDisclaimer", promptFunction());
-                /*if(exeOnce){
-                    parseVersion(obj);
+                if(exeOnce)
                     parseESP(obj);
-                }*/
                 parseRssi(obj);
             }
 
@@ -124,14 +120,14 @@
                 var p = div(["none"]);
                 for(var i of obj) {
                     var icon = iconWarn;
-                    var color = "#F70";
+                    var cl = "icon-warn";
                     avail = "";
                     if(false == i["enabled"]) {
                         avail = "disabled";
                     }
                     else if(false == i["is_avail"]) {
                         icon = iconInfo;
-                        color = "#00d";
+                        cl = "icon-info";
                         avail = "not yet available";
                     }
                     else if(0 == i["ts_last_success"]) {
@@ -144,12 +140,12 @@
                         if(false == i["is_producing"])
                             avail += "not ";
                         else
-                            color = "#090";
+                            cl = "icon-success";
                         avail += "producing";
                     }
 
                     p.append(
-                        svg(icon, 20, 20, color, "icon"),
+                        svg(icon, 30, 30, "icon " + cl),
                         span("Inverter #" + i["id"] + ": " + i["name"] + " (v" + i["version"] + ") is " + avail),
                         br()
                     );
@@ -167,22 +163,22 @@
             function parseWarnInfo(warn, success) {
                 var p = div(["none"]);
                 for(var w of warn) {
-                    p.append(svg(iconWarn, 20, 20, "#F70", "icon"), span(w), br());
+                    p.append(svg(iconWarn, 30, 30, "icon icon-warn"), span(w), br());
                 }
                 for(var i of success) {
-                    p.append(svg(iconSuccess, 20, 20, "#090", "icon"), span(i), br());
+                    p.append(svg(iconSuccess, 30, 30, "icon icon-success"), span(i), br());
                 }
 
                 if(commInfo.length > 0)
-                    p.append(svg(iconInfo, 20, 20, "#00d", "icon"), span(commInfo), br());
+                    p.append(svg(iconInfo, 30, 30, "icon icon-info"), span(commInfo), br());
 
                 if(null != release) {
                     if(getVerInt("{#VERSION}") < getVerInt(release))
-                        p.append(svg(iconInfo, 20, 20, "#00d", "icon"), span("Update available, current released version: " + release), br());
+                        p.append(svg(iconInfo, 30, 30, "icon icon-info"), span("Update available, current released version: " + release), br());
                     else if(getVerInt("{#VERSION}") > getVerInt(release))
-                        p.append(svg(iconInfo, 20, 20, "#00d", "icon"), span("You are using development version {#VERSION}. In case of issues you may want to try the current stable release: " + release), br());
+                        p.append(svg(iconInfo, 30, 30, "icon icon-info"), span("You are using development version {#VERSION}. In case of issues you may want to try the current stable release: " + release), br());
                                        else 
-                        p.append(svg(iconInfo, 20, 20, "#00d", "icon"), span("You are using the current stable release: " + release), br());
+                        p.append(svg(iconInfo, 30, 30, "icon icon-info"), span("You are using the current stable release: " + release), br());
                 }
 
                 document.getElementById("warn_info").replaceChildren(p);
diff --git a/src/web/html/login.html b/src/web/html/login.html
index 30531e3b..e790f6a4 100644
--- a/src/web/html/login.html
+++ b/src/web/html/login.html
@@ -7,11 +7,13 @@
     
         

             

-                

+                

                     

-                        
AhoyDTU

-                        
-                        
+                        
AhoyDTU

+                        

+                            

+                            

+                        

                     

                 

             

diff --git a/src/web/html/serial.html b/src/web/html/serial.html
index fbb09ed9..da9d2816 100644
--- a/src/web/html/serial.html
+++ b/src/web/html/serial.html
@@ -8,37 +8,56 @@
         {#HTML_NAV}
         

             

-                

-                    

-                    connected: 
-                    Uptime: 
-                    
-                    
-                    

+                

+                    
+                

+                

+                    
connected: 

+                    
Uptime: 

+                    

+                        
+                        
+                    

+                

+                

+                

                     
Commands

-                    
-                    
-                    
-                    
-                    
-                    
-                    

-                    
-                    

-                    

+                

+                

+                    
Select Inverter

+                    

+                

+                

+                    
Power Limit Command

+                    

+                        
+                    

+                

+                

+                    
Power Limit Value

+                    

+                

+                

+                    

+                    

+                

+                

+                    
Control Inverter

+                    

                         
                         
                         
                     

-                    

-                    
Ctrl result: n/a

+                

+                

+                    
Ctrl result

+                    
n/a

                 

             

         

diff --git a/src/web/html/setup.html b/src/web/html/setup.html
index 9eb9a8bd..a9fa695a 100644
--- a/src/web/html/setup.html
+++ b/src/web/html/setup.html
@@ -20,54 +20,108 @@
         

             

                 

-                    

+                    
+                    

+                    

                         Device Host Name
-                        
-                        
+                        

+                            
Device Name

+                            

+                        

+                        

+                            
Dark Mode

+                            

+                        

+                    

+                    

+                        System Config
+                        
Pinout

+                        

+
+                        
Radio (NRF24L01+)

+                        

+
+                        
Serial Console

+                        

+                            
print inverter data

+                            

+                        

+                        

+                            
Serial Debug

+                            

+                        

+                        

+                            
Interval [s]

+                            

+                        

                     

+                    

 
                     
                     

-                        

+                        

                             WiFi
                             
Enter the credentials to your prefered WiFi station. After rebooting the device tries to connect with this information.

-                            
-                            

-                            
-                            
-                            
-                            
-                            
-                            
+
+                            

+                                
Search Networks

+                                

+                            

+
+                            

+                                
Avail Networks

+                                

+                                    
+                                

+                            

+                            

+                                
SSID

+                                

+                            

+                            

+                                
Password

+                                

+                            

                         

-                        

+                        

                             Static IP (optional)
                             

                                 Leave fields blank for DHCP

-                                The following fields are parsed in this format: 192.168.1.1
+                                The following fields are parsed in this format: 192.168.4.1
                             

-                            
-                            
-                            
-                            
-                            
-                            
-                            
-                            
-                            
-                            
+                            

+                                
IP Address

+                                

+                            

+                            

+                                
Submask

+                                

+                            

+                            

+                                
DNS 1

+                                

+                            

+                            

+                                
DNS 2

+                                

+                            

+                            

+                                
Gateway

+                                

+                            

                         

                     

 
                     
                     

-                        

-                            Protection
-                            
-                            
-                            
+                        

+                            Protection
+                            

+                                
Admin Password

+                                

+                            

                             
Select pages which should be protected by password

                             

                         

@@ -75,140 +129,178 @@
 
                     
                     

-                    

+                    

                     Inverter
-                        


-                        
-                        
General

-                        
-                        
-                        
-                        
-
-
-                        
-                        

-                        
-                        

-                        
-                        

+                        

+                        

+                            

+                            

+                        

+                        

+                            
General

+                            

+                        

+                        

+                            
Interval [s]

+                            

+                        

+                        

+                            
Max retries per Payload

+                            

+                        

+                        

+                            
Reset values and YieldDay at midnight

+                            

+                        

+                        

+                            
Reset values when inverter polling stops at sunset

+                            

+                        

+                        

+                            
Reset values when inverter status is 'not available'

+                            

+                        

                     

                     

 
                     
                     

-                    

-                    NTP Server
-                        
-                        
-                        
-                        
-                        
-                        
-                        
-                        
+                    

+                        NTP Server
+                        

+                            
NTP Server / IP

+                            

+                        

+                        

+                            
NTP Port

+                            

+                        

+                        

+                            
set system time

+                            

+                                
+                                
+                                
+                            

+                        

                     

                     

 
                     
                     

-                    

-                    Sunrise & Sunset
-                        

-                            Use a decimal separator: '.' (dot) for Latitude and Longitude
-                        

-                        
-                        
-                        
-                        
-                        
-                        
-                        

-                        
-                        

+                    

+                        Sunrise & Sunset
+                        
Use a decimal separator: '.' (dot) for Latitude and Longitude

+
+                        

+                            
Latitude (decimal)

+                            

+                        

+                        

+                            
Longitude (decimal)

+                            

+                        

+                        

+                            
Offset (pre sunrise, post sunset)

+                            

+                        

+                        

+                            
Stop polling inverters during night

+                            

+                        

                     

                     

 
                     
                     

-                    

-                    MQTT
-                        
-                        
-                        
-                        
-                        
-                        
-                        
-                        
-                        
-                        
+                    

+                        MQTT
+                        

+                            
Broker / Server IP

+                            

+                        

+                        

+                            
Port

+                            

+                        

+                        

+                            
Username (optional)

+                            

+                        

+                        

+                            
Password (optional)

+                            

+                        

+                        

+                            
Topic

+                            

+                        

                         
Send Inverter data in a fixed interval, even if there is no change. A value of '0' disables the fixed interval. The data is published once it was successfully received from inverter. (default: 0)

-                        
-                        
-                        
-                        
-                        
-                    

-                    

-
-                    
-                    

-                    

-                        System Config
-                        
Pinout

-                        

-
-                        
Radio (NRF24L01+)

-                        

-
-                        
Serial Console

-                        
-                        

-                        
-                        

-                        
-                        
+                        

+                            
Interval [s]

+                            

+                        

+                        

+                            
Discovery Config (homeassistant)

+                            

+                                
+                                
+                            

+                        

                     

                     

 
                     
                     

-                    

+                    

                         Display Config
                         

-                        
-                        

-                        
-                        

-                        
-                        

-                        
-                        

-
-                        
-                        
+                        

+                            
Show Logo

+                            

+                        

+                        

+                            
Turn off while inverters are offline

+                            

+                        

+                        

+                            
Enable pixel shifting

+                            

+                        

+                        

+                            
Rotate 180 degree

+                            

+                        

+                        

+                            
Contrast

+                            

+                        

+                        
+
                         
Pinout

                         

                     

                     

 
-                    

-                        
-                        
-                        
+                    

+                        
Reboot device after successful save

+                        

+                            
+                            
+                        

                     

+
                     

-                    

+                    

                         ERASE SETTINGS (not WiFi)
-                        

+                        

                             Upload / Store JSON Settings
                             
                                 
                                 
                             
                         

-                        Download settings (JSON file) (only saved values, passwords will be removed!)
+                        Download settings (JSON file) (only saved values, passwords will be removed!)
                     

                 
             

@@ -343,23 +435,38 @@
                 document.getElementsByName(id + "Name")[0].value = "";
             }
 
+            function mlCb(id, des, chk=false) {
+                var cb = ml("input", {type: "checkbox", id: id, name: id}, "");
+                if(chk)
+                    cb.checked = true;
+                return ml("div", {class: "row mb-3"}, [
+                    ml("div", {class: "col-8 col-sm-3"}, des),
+                    ml("div", {class: "col-4 col-sm-9"}, cb)
+                ]);
+            }
+
+            function mlE(des, e) {
+                return ml("div", {class: "row mb-3"}, [
+                    ml("div", {class: "col-12 col-sm-3 my-2"}, des),
+                    ml("div", {class: "col-12 col-sm-9"}, e)
+                ]);
+            }
+
             function ivHtml(obj, id) {
                 highestId = id + 1;
                 if(highestId == maxInv)
                     setHide("btnAdd", true);
-                iv = document.getElementById("inverter");
+
+                var iv = document.getElementById("inverter");
                 iv.appendChild(des("Inverter " + id));
                 id = "inv" + id;
 
-                iv.appendChild(lbl(id + "Enable", "Communication Enable"));
-                var en = inp(id + "Enable", null, null, ["cb"], id + "Enable", "checkbox");
-                en.checked = obj["enabled"];
-                iv.appendChild(en);
-                iv.appendChild(br());
-
-                iv.appendChild(lbl(id + "Addr", "Serial Number (12 digits)*"));
                 var addr = inp(id + "Addr", obj["serial"], 12, ["text"], null, "text", "[0-9]+", "Invalid input");
-                iv.appendChild(addr);
+                iv.append(
+                    mlCb(id + "Enable", "Communication Enable", obj["enabled"]),
+                    mlE("Serial Number (12 digits)*", addr)
+                );
+
                 ['keyup', 'change'].forEach(function(evt) {
                     addr.addEventListener(evt, (e) => {
                         var serial = addr.value.substring(0,4);
@@ -369,9 +476,9 @@
                             setHide(id+"ModName"+i, true);
                             setHide(id+"YieldCor"+i, true);
                         }
-                        setHide("lbl"+id+"ModPwr", true);
-                        setHide("lbl"+id+"ModName", true);
-                        setHide("lbl"+id+"YieldCor", true);
+                        setHide("row"+id+"ModPwr", true);
+                        setHide("row"+id+"ModName", true);
+                        setHide("row"+id+"YieldCor", true);
 
                         if(serial.charAt(0) == 1) {
                             if((serial.charAt(1) == 0) || (serial.charAt(1) == 1)) {
@@ -391,39 +498,44 @@
                                 setHide(id+"ModName"+i, false);
                                 setHide(id+"YieldCor"+i, false);
                             }
-                            setHide("lbl"+id+"ModPwr", false);
-                            setHide("lbl"+id+"ModName", false);
-                            setHide("lbl"+id+"YieldCor", false);
+                            setHide("row"+id+"ModPwr", false);
+                            setHide("row"+id+"ModName", false);
+                            setHide("row"+id+"YieldCor", false);
                         }
                     })
                 });
 
-                iv.append(
-                    lbl(id + "Name", "Name*"),
-                    inp(id + "Name", obj["name"], 16, ["text"], null, "text", "[A-Za-z0-9./#$%&=+_-]+", "Invalid input")
-                );
+                iv.append(mlE("Name*", inp(id + "Name", obj["name"], 16, ["text"], null, "text", "[A-Za-z0-9./#$%&=+_-]+", "Invalid input")));
 
                 for(var j of [
                     ["ModPwr", "ch_max_power", "Max Module Power (Wp)", 4, "[0-9]+"],
                     ["ModName", "ch_name", "Module Name", 16, null],
                     ["YieldCor", "ch_yield_cor", "Yield Total Correction [kWh]", 16, "[0-9-]+"]]) {
+
                     var cl = (re.test(obj["serial"])) ? null : ["hide"];
-                    iv.appendChild(lbl(null, j[2], cl, "lbl" + id + j[0]));
-                    d = div([j[0]]);
+
                     i = 0;
-                    cl = (re.test(obj["serial"])) ? ["text", "sh"] : ["text", "sh", "hide"];
+                    arrIn = [];
                     for(it of obj[j[1]]) {
-                        d.appendChild(inp(id + j[0] + i, it, j[3], cl, id + j[0] + i, "text", j[4], "Invalid input"));
+                        arrIn.push(ml("div", {class: "col-3 "},
+                            inp(id + j[0] + i, it, j[3], [], id + j[0] + i, "text", j[4], "Invalid input")
+                        ));
                         i++;
                     }
-                    iv.appendChild(d);
+
+                    iv.append(
+                        ml("div", {class: "row mb-2 mb-sm-3", id: "row" + id + j[0]}, [
+                            ml("div", {class: "col-12 col-sm-3 my-2"}, j[2]),
+                            ml("div", {class: "col-12 col-sm-9"},
+                                ml("div", {class: "row"}, arrIn)
+                            )
+                        ])
+                    );
                 }
+
                 var del = inp(id+"del", "X", 0, ["btn", "btnDel"], id+"del", "button");
                 del.addEventListener("click", delIv);
-                iv.append(
-                    lbl(id + "lbldel", "Delete"),
-                    del
-                );
+                iv.append(mlE("Delete", del));
             }
 
             function ivGlob(obj) {
@@ -436,20 +548,18 @@
             function parseSys(obj) {
                 for(var i of [["device", "device_name"], ["ssid", "ssid"]])
                     document.getElementsByName(i[0])[0].value = obj[i[1]];
-                var e = document.getElementsByName("adminpwd")[0];
+                document.getElementsByName("darkMode")[0].checked = obj["dark_mode"];
+                e = document.getElementsByName("adminpwd")[0];
                 if(!obj["pwd_set"])
                     e.value = "";
                 var d = document.getElementById("prot_mask");
-                var a = ["Index", "Live", "Serial / Console", "Settings", "Update", "System"]
+                var a = ["Index", "Live", "Serial / Console", "Settings", "Update", "System"];
+                var el = [];
                 for(var i = 0; i < 6; i++) {
-                    var chkd = ((obj["prot_mask"] & (1 << i)) == (1 << i));
-                    var sp = lbl("protMask" + i, a[i]);
-                    var cb = inp("protMask" + i, null, null, ["cb"], "protMask" + i, "checkbox", null, null, chkd);
-                    if(0 == i)
-                        d.replaceChildren(sp, cb, br());
-                    else
-                        d.append(sp, cb, br());
+                    var chk = ((obj["prot_mask"] & (1 << i)) == (1 << i));
+                    el.push(mlCb("protMask" + i, a[i], chk))
                 }
+                d.append(...el);
             }
 
             function parseGeneric(obj) {
@@ -495,20 +605,31 @@
                 var e = document.getElementById("pinout");
                 pins = [['cs', 'pinCs'], ['ce', 'pinCe'], ['irq', 'pinIrq'], ['led0', 'pinLed0'], ['led1', 'pinLed1']];
                 for(p of pins) {
-                    e.appendChild(lbl(p[1], p[0].toUpperCase()));
-                    e.appendChild(sel(p[1], ("ESP8266" == type) ? esp8266pins : esp32pins, obj[p[0]]));
+                    e.append(
+                        ml("div", {class: "row mb-3"}, [
+                            ml("div", {class: "col-12 col-sm-3 my-2"}, p[0].toUpperCase()),
+                            ml("div", {class: "col-12 col-sm-9"},
+                                sel(p[1], ("ESP8266" == type) ? esp8266pins : esp32pins, obj[p[0]])
+                            )
+                        ])
+                    );
                 }
             }
 
             function parseRadio(obj) {
-                var e = document.getElementById("rf24");
-                e.appendChild(lbl("rf24Power", "Amplifier Power Level"));
-                e.appendChild(sel("rf24Power", [
-                    [0, "MIN"],
-                    [1, "LOW"],
-                    [2, "HIGH"],
-                    [3, "MAX"]
-                ], obj["power_level"]));
+                var e = document.getElementById("rf24").append(
+                    ml("div", {class: "row mb-3"}, [
+                        ml("div", {class: "col-12 col-sm-3 my-2"}, p[0].toUpperCase()),
+                        ml("div", {class: "col-12 col-sm-9"},
+                            sel("rf24Power", [
+                                [0, "MIN"],
+                                [1, "LOW"],
+                                [2, "HIGH"],
+                                [3, "MAX"]
+                            ], obj["power_level"])
+                        )
+                    ])
+                );
             }
 
             function parseSerial(obj) {
@@ -524,14 +645,22 @@
                 var e = document.getElementById("dispPins");
                 pins = [['SCL / CS', 'pinDisp0'], ['SDA / DC', 'pinDisp1']];
                 for(p of pins) {
-                    e.appendChild(lbl(p[1], p[0].toUpperCase()));
-                    e.appendChild(sel(p[1], ("ESP8266" == type) ? esp8266pins : esp32pins, obj[p[1]]));
+                    e.append(
+                        ml("div", {class: "row mb-3"}, [
+                            ml("div", {class: "col-12 col-sm-3 my-2"}, p[0].toUpperCase()),
+                            ml("div", {class: "col-12 col-sm-9"},
+                                sel(p[1], ("ESP8266" == type) ? esp8266pins : esp32pins, obj[p[1]])
+                            )
+                        ])
+                    );
                 }
 
                 var opts = [[0, "None"], [1, "Nokia5110"], [2, "SSD1306 0.96\""], [3, "SH1106 1.3\""]];
                 document.getElementById("dispType").append(
-                    lbl("dispType", "Type"),
-                    sel("dispType", opts, obj["disp_type"])
+                    ml("div", {class: "row mb-3"}, [
+                        ml("div", {class: "col-12 col-sm-3 my-2"}, "Type"),
+                        ml("div", {class: "col-12 col-sm-9"}, sel("dispType", opts, obj["disp_type"]))
+                    ])
                 );
 
                 e = document.getElementById("contrast");
@@ -576,11 +705,7 @@
                     e.value = s.value;
             }
 
-            hiddenInput = document.getElementById("disclaimer")
-            hiddenInput.value = sessionStorage.getItem("gDisclaimer");
-
             getAjax("/api/setup", parse);
-
         
     
 
diff --git a/src/web/html/style.css b/src/web/html/style.css
index 384ff1f7..7a4266ee 100644
--- a/src/web/html/style.css
+++ b/src/web/html/style.css
@@ -4,26 +4,39 @@ html, body {
     padding: 0;
     height: 100%;
     min-height: 100%;
+    background-color: var(--bg);
+    color: var(--fg);
 }
 
 h2 {
     padding-left: 10px;
 }
 
+span, li, h3, label, fieldset {
+    color: var(--fg);
+}
+
+fieldset, input[type=submit], .btn {
+    border-radius: 4px;
+}
+
+#live span {
+    color: var(--fg2);
+}
+
 .topnav {
-    background-color: #333;
+    background-color: var(--nav-bg);
     position: fixed;
     top: 0;
     width: 100%;
 }
 
 .topnav a {
-    color: #fff;
+    color: var(--fg2);
     padding: 14px 14px;
     text-decoration: none;
     font-size: 17px;
     display: block;
-    height: 20px;
 }
 
 #topnav a {
@@ -33,18 +46,17 @@ h2 {
 .topnav a.icon {
     top: 0;
     left: 0;
-    background: #333;
+    background: var(--nav-bg);
     display: block;
     position: absolute;
 }
 
 .topnav a:hover {
-    background-color: #044e86 !important;
-    color: #000;
+    background-color: var(--primary-hover) !important;
 }
 
 .topnav .info {
-    color: #fff;
+    color: var(--fg2);
     position: absolute;
     right: 24px;
     top: 5px;
@@ -61,8 +73,24 @@ svg.icon {
     padding: 5px 7px 5px 0px;
 }
 
+.icon-info {
+    fill: var(--info);
+}
+
+.icon-warn {
+    fill: var(--warn);
+}
+
+.icon-success {
+    fill: var(--success);
+}
+
+.wifi {
+    fill: var(--fg2);
+}
+
 .title {
-    background-color: #006ec0;
+    background-color: var(--primary);
     color: #fff !important;
     padding-left: 80px !important
 }
@@ -78,7 +106,7 @@ svg.icon {
 }
 
 .topnav .active {
-    background-color: #555;
+    background-color: var(--nav-active);
 }
 
 span.seperator {
@@ -89,6 +117,197 @@ span.seperator {
     display: block;
 }
 
+#content {
+    max-width: 1140px;
+}
+
+.total-h {
+    background-color: var(--total-head-title);
+    color: var(--fg2);
+}
+
+.total-bg {
+    background-color: var(--total-bg);
+    color: var(--fg2);
+}
+
+.iv-h {
+    background-color: var(--iv-head-title);
+    color: var(--fg2);
+}
+
+.iv-bg {
+    background-color: var(--iv-head-bg);
+    color: var(--fg2);
+}
+
+.ch-h {
+    background-color: var(--ch-head-title);
+    color: var(--fg2);
+}
+
+.ch-bg {
+    background-color: var(--ch-head-bg);
+    color: var(--fg2);
+}
+
+.ts-h {
+    background-color: var(--ts-head);
+    color: var(--fg2);
+}
+
+.ts-bg {
+    background-color: var(--ts-bg);
+    color: var(--fg2);
+}
+
+.hr {
+    border-top: 1px solid var(--iv-head-title);
+    margin: 1rem 0 1rem;
+}
+
+p {
+    text-align: justify;
+    font-size: 13pt;
+    color: var(--fg);
+}
+
+#footer {
+    background-color: var(--footer-bg);
+}
+
+.row { display: flex; max-width: 100%; flex-wrap: wrap; }
+.col { flex: 1 0 0%; }
+
+.col-1, .col-2, .col-3, .col-4,
+.col-5, .col-6, .col-7, .col-8,
+.col-9, .col-10, .col-11, .col-12 { flex: 0 0 auto; }
+
+.col-1 { width: 8.333333333%; }
+.col-2 { width: 16.66666667%; }
+.col-3 { width: 25%; }
+.col-4 { width: 33.33333333%; }
+.col-5 { width: 41.66666667%; }
+.col-6 { width: 50%; }
+.col-7 { width: 58.33333333%; }
+.col-8 { width: 66.66666667%; }
+.col-9 { width: 75%; }
+.col-10 { width: 83.33333333%; }
+.col-11 { width: 91.66666667%; }
+.col-12 { width: 100%; }
+
+.p-1 { padding: 0.25rem; }
+.p-2 { padding: 0.5rem; }
+.p-3 { padding: 1rem; }
+.p-4 { padding: 1.5rem; }
+.p-5 { padding: 3rem; }
+
+.px-1 { padding: 0 0.25rem 0 0.25rem; }
+.px-2 { padding: 0 0.5rem 0 0.5rem; }
+.px-3 { padding: 0 1rem 0 1rem; }
+.px-4 { padding: 0 1.5rem 0 1.5rem; }
+.px-5 { padding: 0 3rem 0 3rem; }
+
+.py-1 { padding: 0.25rem 0 0.25rem; }
+.py-2 { padding: 0.5rem 0 0.5rem; }
+.py-3 { padding: 1rem 0 1rem; }
+.py-4 { padding: 1.5rem 0 1.5rem; }
+.py-5 { padding: 3rem 0 3rem; }
+
+.mx-1  { margin: 0 0.25rem 0 0.25rem; }
+.mx-2  { margin: 0 0.5rem 0 0.5rem; }
+.mx-3  { margin: 0 1rem 0 1rem; }
+.mx-4  { margin: 0 1.5rem 0 1.5rem; }
+.mx-5  { margin: 0 3rem 0 3rem; }
+
+.my-1  { margin: 0.25rem 0 0.25rem; }
+.my-2  { margin: 0.5rem 0 0.5rem; }
+.my-3  { margin: 1rem 0 1rem; }
+.my-4  { margin: 1.5rem 0 1.5rem; }
+.my-5  { margin: 3rem 0 3rem; }
+
+.mt-1  { margin-top: 0.25rem }
+.mt-2  { margin-top: 0.5rem }
+.mt-3  { margin-top: 1rem }
+.mt-4  { margin-top: 1.5rem }
+.mt-5  { margin-top: 3rem }
+
+.mb-1  { margin-bottom: 0.25rem }
+.mb-2  { margin-bottom: 0.5rem }
+.mb-3  { margin-bottom: 1rem }
+.mb-4  { margin-bottom: 1.5rem }
+.mb-5  { margin-bottom: 3rem }
+
+.fs-1 { font-size: 3.5rem; }
+.fs-2 { font-size: 3rem; }
+.fs-3 { font-size: 2.5rem; }
+.fs-4 { font-size: 2rem; }
+.fs-5 { font-size: 1.75rem; }
+.fs-6 { font-size: 1.5rem; }
+.fs-7 { font-size: 1.25rem; }
+.fs-8 { font-size: 1rem; }
+.fs-9 { font-size: 0.75rem; }
+.fs-10 { font-size: 0.5rem; }
+
+.a-r { text-align: right; }
+.a-c { text-align: center; }
+
+.row > * {
+    padding-left: 0.5rem;
+    padding-right: 0.5rem;
+}
+
+*, ::after, ::before {
+    box-sizing: border-box;
+}
+
+/* sm */
+@media(min-width: 768px) {
+    .col-sm-1 { width: 8.333333333%; }
+    .col-sm-2 { width: 16.66666667%; }
+    .col-sm-3 { width: 25%; }
+    .col-sm-4 { width: 33.33333333%; }
+    .col-sm-5 { width: 41.66666667%; }
+    .col-sm-6 { width: 50%; }
+    .col-sm-7 { width: 58.33333333%; }
+    .col-sm-8 { width: 66.66666667%; }
+    .col-sm-9 { width: 75%; }
+    .col-sm-10 { width: 83.33333333%; }
+    .col-sm-11 { width: 91.66666667%; }
+    .col-sm-12 { width: 100%; }
+
+    .mb-sm-1  { margin-bottom: 0.25rem }
+    .mb-sm-2  { margin-bottom: 0.5rem }
+    .mb-sm-3  { margin-bottom: 1rem }
+    .mb-sm-4  { margin-bottom: 1.5rem }
+    .mb-sm-5  { margin-bottom: 3rem }
+
+    .fs-sm-1 { font-size: 3.5rem; }
+    .fs-sm-2 { font-size: 3rem; }
+    .fs-sm-3 { font-size: 2.5rem; }
+    .fs-sm-4 { font-size: 2rem; }
+    .fs-sm-5 { font-size: 1.75rem; }
+    .fs-sm-6 { font-size: 1.5rem; }
+    .fs-sm-7 { font-size: 1.25rem; }
+    .fs-sm-8 { font-size: 1rem; }
+}
+
+/* md */
+@media(min-width: 992px) {
+    .col-md-1 { width: 8.333333333%; }
+    .col-md-2 { width: 16.66666667%; }
+    .col-md-3 { width: 25%; }
+    .col-md-4 { width: 33.33333333%; }
+    .col-md-5 { width: 41.66666667%; }
+    .col-md-6 { width: 50%; }
+    .col-md-7 { width: 58.33333333%; }
+    .col-md-8 { width: 66.66666667%; }
+    .col-md-9 { width: 75%; }
+    .col-md-10 { width: 83.33333333%; }
+    .col-md-11 { width: 91.66666667%; }
+    .col-md-12 { width: 100%; }
+}
+
 #wrapper {
     min-height: 100%;
 }
@@ -101,7 +320,6 @@ span.seperator {
 #footer {
     height: 121px;
     margin-top: -121px;
-    background-color: #555;
     width: 100%;
     font-size: 13px;
 }
@@ -176,13 +394,6 @@ span.seperator {
     }
 }
 
-/** old CSS below **/
-
-p {
-    text-align: justify;
-    font-size: 13pt;
-}
-
 p.lic, p.lic a {
     font-size: 8pt;
     color: #999;
@@ -191,11 +402,11 @@ p.lic, p.lic a {
 .des {
     margin-top: 20px;
     font-size: 13pt;
-    color: #006ec0;
+    color: var(--secondary);
 }
 
 .s_active, .s_collapsible:hover {
-    background-color: #044e86;
+    background-color: var(--primary-hover);
     color: #fff;
 }
 
@@ -205,34 +416,34 @@ p.lic, p.lic a {
 }
 
 .s_collapsible {
-    background-color: #006ec0;
+    background-color: var(--primary);
     color: white;
     cursor: pointer;
-    padding: 18px;
+    padding: 12px;
     width: 100%;
     border: none;
     text-align: left;
     outline: none;
     font-size: 15px;
-    margin-bottom: 4px;
+    margin-bottom: 5px;
 }
 
 .subdes {
     font-size: 12pt;
-    color: #006ec0;
+    color: var(--secondary);
     margin-left: 7px;
 }
 
 .subsubdes {
     font-size:12pt;
-    color:#006ec0;
+    color:var(--secondary);
     margin: 0 0 7px 12px;
 }
 
 a:link, a:visited {
     text-decoration: none;
     font-size: 13pt;
-    color: #006ec0;
+    color: var(--secondary);
 }
 
 a:hover, a:focus {
@@ -240,14 +451,14 @@ a:hover, a:focus {
 }
 
 a.btn {
-    background-color: #006ec0;
+    background-color: var(--primary);
     color: #fff;
     padding: 7px 15px 7px 15px;
     display: inline-block;
 }
 
 a.btn:hover {
-    background-color: #044e86 !important;
+    background-color: var(--primary-hover) !important;
 }
 
 input, select {
@@ -255,11 +466,13 @@ input, select {
     font-size: 13pt;
 }
 
-input.text, select {
-    width: 70%;
+input[type=text], input[type=password], select, input[type=number] {
+    width: 100%;
     box-sizing: border-box;
-    margin-bottom: 10px;
     border: 1px solid #ccc;
+    border-radius: 4px;
+    background-color: var(--input-bg);
+    color: var(--fg);
 }
 
 input.sh {
@@ -272,7 +485,7 @@ input.btnDel {
 }
 
 input.btn {
-    background-color: #006ec0;
+    background-color: var(--primary);
     color: #fff;
     border: 0px;
     padding: 7px 20px 7px 20px;
@@ -303,10 +516,6 @@ pre {
     white-space: pre-wrap;
 }
 
-fieldset {
-    margin-bottom: 15px;
-}
-
 .left {
     float: left;
 }
@@ -315,88 +524,11 @@ fieldset {
     float: right;
 }
 
-div.ch-iv {
-    width: 100%;
-    background-color: #32b004;
-    display: inline-block;
-    margin-bottom: 15px;
-    padding-bottom: 20px;
-    overflow: auto;
-}
-
-div.ch {
-    width: 220px;
-    min-height: 350px;
-    background-color: #006ec0;
-    display: inline-block;
-    margin: 0 20px 10px 0px;
-    overflow: auto;
-    padding-bottom: 20px;
-}
-
-div.ch-all {
-    width: 100%;
-    background-color: #b06e04;
-    display: inline-block;
-    margin-bottom: 15px;
-    padding-bottom: 20px;
-    overflow: auto;
-}
-
-div.ch .value, div.ch .info, div.ch .head, div.ch-iv .value, div.ch-iv .info, div.ch-iv .head, div.ch-all .value, div.ch-all .info, div.ch-all .head {
-    color: #fff;
-    display: block;
-    width: 100%;
-    text-align: center;
-}
-
 .subgrp {
     float: left;
     width: 220px;
 }
 
-div.ch .unit, div.ch-iv .unit, div.ch-all .unit {
-    font-size: 19px;
-    margin-left: 10px;
-}
-
-div.ch .value, div.ch-iv .value, div.ch-all .value {
-    margin-top: 20px;
-    font-size: 24px;
-}
-
-div.ch .info, div.ch-iv .info, div.ch-all .info {
-    margin-top: 3px;
-    font-size: 10px;
-}
-
-div.ch .head {
-    background-color: #003c80;
-    padding: 10px 0 10px 0;
-}
-
-div.ch-all .head {
-    background-color: #8e5903;
-    padding: 10px 0 10px 0;
-}
-
-div.ch-iv .head {
-    background-color: #1c6800;
-    padding: 10px 0 10px 0;
-}
-
-div.iv {
-    max-width: 960px;
-    margin-bottom: 40px;
-}
-
-div.ts {
-    font-size: 13px;
-    background-color: #ddd;
-    border-top: 7px solid #999;
-    padding: 7px;
-}
-
 div.ModPwr, div.ModName, div.YieldCor {
   width:70%;
   display: inline-block;
@@ -447,104 +579,19 @@ div.hr {
 }
 
 #login {
-    width: 300px;
+    width: 450px;
     height: 200px;
     border: 1px solid #ccc;
-    background-color: #eee;
+    background-color: var(--ts-head);
     position: absolute;
     top: 50%;
     left: 50%;
     margin-top: -160px;
-    margin-left: -150px;
-}
-
-#login .pad {
-    padding: 20px;
-}
-
-#login .pad input {
-    width: 100%;
-    padding: 7px 0 7px 0;
-    border: 0px;
-    margin-bottom: 10px;
+    margin-left: -225px;
 }
 
 
 .head {
-    background-color: #006ec0;
+    background-color: var(--primary);
     color: #fff;
 }
-
-.row { display: flex; max-width: 100%; flex-wrap: wrap; }
-.col { flex: 1 0 0%; }
-
-.col-1, .col-2, .col-3, .col-4,
-.col-5, .col-6, .col-7, .col-8,
-.col-9, .col-10, .col-11, .col-12 { flex: 0 0 auto; }
-
-
-.col-1 { width: 8.333333333%; }
-.col-2 { width: 16.66666667%; }
-.col-3 { width: 25%; }
-.col-4 { width: 33.33333333%; }
-.col-5 { width: 41.66666667%; }
-.col-6 { width: 50%; }
-.col-7 { width: 58.33333333%; }
-.col-8 { width: 66.66666667%; }
-.col-9 { width: 75%; }
-.col-10 { width: 83.33333333%; }
-.col-11 { width: 91.66666667%; }
-.col-12 { width: 100%; }
-
-.p-1 { padding: 0.25rem; }
-.p-2 { padding: 0.5rem; }
-.p-3 { padding: 1rem; }
-.p-4 { padding: 1.5rem; }
-.p-5 { padding: 3rem; }
-
-.mt-1  { margin-top: 0.25rem }
-.mt-2  { margin-top: 0.5rem }
-.mt-3  { margin-top: 1rem }
-.mt-4  { margin-top: 1.5rem }
-.mt-5  { margin-top: 3rem }
-
-.mb-1  { margin-bottom: 0.25rem }
-.mb-2  { margin-bottom: 0.5rem }
-.mb-3  { margin-bottom: 1rem }
-.mb-4  { margin-bottom: 1.5rem }
-.mb-5  { margin-bottom: 3rem }
-
-.a-r { text-align: right; }
-.a-c { text-align: center; }
-
-/* sm */
-@media(min-width: 768px) {
-    .col-sm-1 { width: 8.333333333%; }
-    .col-sm-2 { width: 16.66666667%; }
-    .col-sm-3 { width: 25%; }
-    .col-sm-4 { width: 33.33333333%; }
-    .col-sm-5 { width: 41.66666667%; }
-    .col-sm-6 { width: 50%; }
-    .col-sm-7 { width: 58.33333333%; }
-    .col-sm-8 { width: 66.66666667%; }
-    .col-sm-9 { width: 75%; }
-    .col-sm-10 { width: 83.33333333%; }
-    .col-sm-11 { width: 91.66666667%; }
-    .col-sm-12 { width: 100%; }
-}
-
-/* md */
-@media(min-width: 992px) {
-    .col-md-1 { width: 8.333333333%; }
-    .col-md-2 { width: 16.66666667%; }
-    .col-md-3 { width: 25%; }
-    .col-md-4 { width: 33.33333333%; }
-    .col-md-5 { width: 41.66666667%; }
-    .col-md-6 { width: 50%; }
-    .col-md-7 { width: 58.33333333%; }
-    .col-md-8 { width: 66.66666667%; }
-    .col-md-9 { width: 75%; }
-    .col-md-10 { width: 83.33333333%; }
-    .col-md-11 { width: 91.66666667%; }
-    .col-md-12 { width: 100%; }
-}
diff --git a/src/web/html/update.html b/src/web/html/update.html
index dd5f4d71..214bc19f 100644
--- a/src/web/html/update.html
+++ b/src/web/html/update.html
@@ -8,10 +8,13 @@
         {#HTML_NAV}
         

             

-                

-                    
-                    
-                

+                

+                    Select firmware file (*.bin)
+                    

+                        
+                        
+                    

+                

             

         

         {#HTML_FOOTER}
diff --git a/src/web/html/visualization.html b/src/web/html/visualization.html
index a1d11bcc..232eb60a 100644
--- a/src/web/html/visualization.html
+++ b/src/web/html/visualization.html
@@ -16,6 +16,11 @@
         {#HTML_FOOTER}