Merge pull request #27 from Sprinterfreak/pypackage

Pypackage: integrate ahoy.py into module

tools/rpi/README.md

@@ -40,8 +40,27 @@ contact the inverter every second on channel 40, and listen for replies.
 
 Whenever it sees a reply, it will decoded and logged to the given log file.
 
-    $ sudo python3 ahoy.py --config /home/dtu/ahoy.yml | tee -a log2.log
+    $ sudo python3 -um hoymiles --log-transactions --verbose --config /home/dtu/ahoy.yml | tee -a log2.log
 
+Python parameters
+- `-u` enables python's unbuffered mode
+- `-m hoymiles` tells python to load module 'hoymiles' as main app
+
+
+The application describes itself
+```
+python -m hoymiles --help
+usage: hoymiles [-h] -c [CONFIG_FILE] [--log-transactions] [--verbose]
+
+Ahoy - Hoymiles solar inverter gateway
+
+optional arguments:
+  -h, --help            show this help message and exit
+  -c [CONFIG_FILE], --config-file [CONFIG_FILE]
+                        configuration file
+  --log-transactions    Enable transaction logging output
+  --verbose             Enable debug output
+```
 
 
 Inject payloads via MQTT

tools/rpi/ahoy.py

@@ -1,212 +0,0 @@
-#!/usr/bin/env python3
-# -*- coding: utf-8 -*-
-
-import sys
-import struct
-import re
-import time
-from datetime import datetime
-import argparse
-import hoymiles
-from RF24 import RF24, RF24_PA_LOW, RF24_PA_MAX, RF24_250KBPS, RF24_CRC_DISABLED, RF24_CRC_8, RF24_CRC_16
-import paho.mqtt.client
-import yaml
-from yaml.loader import SafeLoader
-
-parser = argparse.ArgumentParser(description='Ahoy - Hoymiles solar inverter gateway')
-parser.add_argument("-c", "--config-file", nargs="?",
-    help="configuration file")
-global_config = parser.parse_args()
-
-if global_config.config_file:
-    with open(global_config.config_file) as yf:
-        cfg = yaml.load(yf, Loader=SafeLoader)
-else:
-    with open(global_config.config_file) as yf:
-        cfg = yaml.load('ahoy.yml', Loader=SafeLoader)
-
-radio = RF24(22, 0, 1000000)
-hmradio = hoymiles.HoymilesNRF(device=radio)
-mqtt_client = None
-
-command_queue = {}
-mqtt_command_topic_subs = []
-
-hoymiles.HOYMILES_TRANSACTION_LOGGING=True
-hoymiles.HOYMILES_DEBUG_LOGGING=True
-
-def main_loop():
-    inverters = [
-            inverter for inverter in ahoy_config.get('inverters', [])
-            if not inverter.get('disabled', False)]
-
-    for inverter in inverters:
-        if hoymiles.HOYMILES_DEBUG_LOGGING:
-            print(f'Poll inverter {inverter["serial"]}')
-        poll_inverter(inverter)
-
-def poll_inverter(inverter):
-    inverter_ser = inverter.get('serial')
-    dtu_ser = ahoy_config.get('dtu', {}).get('serial')
-
-    if len(command_queue[str(inverter_ser)]) > 0:
-        payload = command_queue[str(inverter_ser)].pop(0)
-    else:
-        payload = hoymiles.compose_set_time_payload()
-
-    payload_ttl = 4
-    while payload_ttl > 0:
-        payload_ttl = payload_ttl - 1
-        com = hoymiles.InverterTransaction(
-                radio=hmradio,
-                dtu_ser=dtu_ser,
-                inverter_ser=inverter_ser,
-                request=next(hoymiles.compose_esb_packet(
-                    payload,
-                    seq=b'\x80',
-                    src=dtu_ser,
-                    dst=inverter_ser
-                    )))
-        response = None
-        while com.rxtx():
-            try:
-                response = com.get_payload()
-                payload_ttl = 0
-            except Exception as e:
-                print(f'Error while retrieving data: {e}')
-                pass
-
-    if response:
-        dt = datetime.now()
-        print(f'{dt} Payload: ' + hoymiles.hexify_payload(response))
-        decoder = hoymiles.ResponseDecoder(response,
-                request=com.request,
-                inverter_ser=inverter_ser
-                )
-        result = decoder.decode()
-        if isinstance(result, hoymiles.decoders.StatusResponse):
-            data = result.__dict__()
-            if hoymiles.HOYMILES_DEBUG_LOGGING:
-                print(f'{dt} Decoded: {data["temperature"]}', end='')
-                phase_id = 0
-                for phase in data['phases']:
-                    print(f' phase{phase_id}=voltage:{phase["voltage"]}, current:{phase["current"]}, power:{phase["power"]}, frequency:{data["frequency"]}', end='')
-                    phase_id = phase_id + 1
-                string_id = 0
-                for string in data['strings']:
-                    print(f' string{string_id}=voltage:{string["voltage"]}, current:{string["current"]}, power:{string["power"]}, total:{string["energy_total"]/1000}, daily:{string["energy_daily"]}', end='')
-                    string_id = string_id + 1
-                print()
-
-            if mqtt_client:
-                mqtt_send_status(mqtt_client, inverter_ser, data,
-                        topic=inverter.get('mqtt', {}).get('topic', None))
-
-def mqtt_send_status(broker, inverter_ser, data, topic=None):
-    """ Publish StatusResponse object """
-
-    if not topic:
-        topic = f'hoymiles/{inverter_ser}'
-
-    # AC Data
-    phase_id = 0
-    for phase in data['phases']:
-        broker.publish(f'{topic}/emeter/{phase_id}/power', phase['power'])
-        broker.publish(f'{topic}/emeter/{phase_id}/voltage', phase['voltage'])
-        broker.publish(f'{topic}/emeter/{phase_id}/current', phase['current'])
-        phase_id = phase_id + 1
-
-    # DC Data
-    string_id = 0
-    for string in data['strings']:
-        broker.publish(f'{topic}/emeter-dc/{string_id}/total', string['energy_total']/1000)
-        broker.publish(f'{topic}/emeter-dc/{string_id}/power', string['power'])
-        broker.publish(f'{topic}/emeter-dc/{string_id}/voltage', string['voltage'])
-        broker.publish(f'{topic}/emeter-dc/{string_id}/current', string['current'])
-        string_id = string_id + 1
-    # Global
-    broker.publish(f'{topic}/frequency', data['frequency'])
-    broker.publish(f'{topic}/temperature', data['temperature'])
-
-def mqtt_on_command(client, userdata, message):
-    """
-    Handle commands to topic
-        hoymiles/{inverter_ser}/command
-    frame a payload and put onto command_queue
-
-    Inverters must have mqtt.send_raw_enabled: true configured
-
-    This can be used to inject debug payloads
-    The message must be in hexlified format
-
-    Use of variables:
-    tttttttt gets expanded to a current int(time)
-
-    Example injects exactly the same as we normally use to poll data:
-      mosquitto -h broker -t inverter_topic/command -m 800b00tttttttt0000000500000000
-
-    This allows for even faster hacking during runtime
-    """
-    try:
-        inverter_ser = next(
-                item[0] for item in mqtt_command_topic_subs if item[1] == message.topic)
-    except StopIteration:
-        print('Unexpedtedly received mqtt message for {message.topic}')
-
-    if inverter_ser:
-        p_message = message.payload.decode('utf-8').lower()
-
-        # Expand tttttttt to current time for use in hexlified payload
-        expand_time = ''.join(f'{b:02x}' for b in struct.pack('>L', int(time.time())))
-        p_message = p_message.replace('tttttttt', expand_time)
-
-        if (len(p_message) < 2048 \
-            and len(p_message) % 2 == 0 \
-            and re.match(r'^[a-f0-9]+$', p_message)):
-            payload = bytes.fromhex(p_message)
-            # commands must start with \x80
-            if payload[0] == 0x80:
-                command_queue[str(inverter_ser)].append(
-                    hoymiles.frame_payload(payload[1:]))
-
-if __name__ == '__main__':
-    ahoy_config = dict(cfg.get('ahoy', {}))
-
-    mqtt_config = ahoy_config.get('mqtt', [])
-    if not mqtt_config.get('disabled', False):
-        mqtt_client = paho.mqtt.client.Client()
-        mqtt_client.username_pw_set(mqtt_config.get('user', None), mqtt_config.get('password', None))
-        mqtt_client.connect(mqtt_config.get('host', '127.0.0.1'), mqtt_config.get('port', 1883))
-        mqtt_client.loop_start()
-        mqtt_client.on_message = mqtt_on_command
-
-    if not radio.begin():
-        raise RuntimeError('Can\'t open radio')
-
-    inverters = [inverter.get('serial') for inverter in ahoy_config.get('inverters', [])]
-    for inverter in ahoy_config.get('inverters', []):
-        inverter_ser = inverter.get('serial')
-        command_queue[str(inverter_ser)] = []
-
-        #
-        # Enables and subscribe inverter to mqtt /command-Topic
-        #
-        if inverter.get('mqtt', {}).get('send_raw_enabled', False):
-            topic_item = (
-                    str(inverter_ser),
-                    inverter.get('mqtt', {}).get('topic', f'hoymiles/{inverter_ser}') + '/command'
-                    )
-            mqtt_client.subscribe(topic_item[1])
-            mqtt_command_topic_subs.append(topic_item)
-
-    loop_interval = ahoy_config.get('interval', 1)
-    try:
-        while True:
-            main_loop()
-
-            if loop_interval:
-                time.sleep(time.time() % loop_interval)
-
-    except KeyboardInterrupt:
-        radio.powerDown()
-        sys.exit()

tools/rpi/ahoy.yml.example

@@ -3,6 +3,12 @@
 ahoy:
   interval: 0
   sunset: true
+
+  # List of available NRF24 transceivers
+  nrf:
+    - ce_pin: 22
+      cs_pin: 0
+
   mqtt:
     disabled: false
     host: example-broker.local

tools/rpi/hoymiles/__init__.py

@@ -11,13 +11,17 @@ f_crc_m = crcmod.predefined.mkPredefinedCrcFun('modbus')
 f_crc8 = crcmod.mkCrcFun(0x101, initCrc=0, xorOut=0)
 
 
-HOYMILES_TRANSACTION_LOGGING=True
+HOYMILES_TRANSACTION_LOGGING=False
-HOYMILES_DEBUG_LOGGING=True
+HOYMILES_DEBUG_LOGGING=False
 
 def ser_to_hm_addr(s):
     """
     Calculate the 4 bytes that the HM devices use in their internal messages to 
     address each other.
+
+    :param str s: inverter serial
+    :return: inverter address
+    :rtype: bytes
     """
     bcd = int(str(s)[-8:], base=16)
     return struct.pack('>L', bcd)
@@ -33,11 +37,20 @@ def ser_to_esb_addr(s):
     The inverters use a BCD representation of the last 8
     digits of their serial number, in reverse byte order, 
     followed by \x01.
+
+    :param str s: inverter serial
+    :return: ESB inverter address
+    :rtype: bytes
     """
     air_order = ser_to_hm_addr(s)[::-1] + b'\x01'
     return air_order[::-1]
 
 def print_addr(a):
+    """
+    Debug print addresses
+
+    :param str a: inverter serial
+    """
     print(f"ser# {a} ", end='')
     print(f" -> HM  {' '.join([f'{x:02x}' for x in ser_to_hm_addr(a)])}", end='')
     print(f" -> ESB {' '.join([f'{x:02x}' for x in ser_to_esb_addr(a)])}")
@@ -46,6 +59,15 @@ def print_addr(a):
 t_last_tx = 0
 
 class ResponseDecoderFactory:
+    """
+    Prepare payload decoder
+
+    :param bytes response: ESB response frame to decode
+    :param request: ESB request frame
+    :type request: bytes
+    :param inverter_ser: inverter serial
+    :type inverter_ser: str
+    """
     model = None
     request = None
     response = None
@@ -63,11 +85,27 @@ class ResponseDecoderFactory:
             self.model = self.inverter_model
 
     def unpack(self, fmt, base):
+        """
+        Data unpack helper
+
+        :param str fmt: struct format string
+        :param int base: unpack base position from self.response bytes
+        :return: unpacked values
+        :rtype: tuple
+        """
         size = struct.calcsize(fmt)
         return struct.unpack(fmt, self.response[base:base+size])
 
     @property
     def inverter_model(self):
+        """
+        Find decoder for inverter model
+
+        :return: suitable decoder model string
+        :rtype: str
+        :raises ValueError: on invalid inverter serial
+        :raises NotImplementedError: if inverter model can not be determined
+        """
         if not self.inverter_ser:
             raise ValueError('Inverter serial while decoding response')
 
@@ -90,14 +128,32 @@ class ResponseDecoderFactory:
 
     @property
     def request_command(self):
+        """
+        Return requested command identifier byte
+
+        :return: hexlified command byte string
+        :rtype: str
+        """
         r_code = self.request[10]
         return f'{r_code:02x}'
 
 class ResponseDecoder(ResponseDecoderFactory):
+    """
+    Base response
+
+    :param bytes response: ESB frame response
+    """
     def __init__(self, response, **params):
+        """Initialize ResponseDecoder"""
         ResponseDecoderFactory.__init__(self, response, **params)
 
     def decode(self):
+        """
+        Decode Payload
+
+        :return: payload decoder instance
+        :rtype: object
+        """
         model = self.inverter_model
         command = self.request_command
 
@@ -111,10 +167,19 @@ class ResponseDecoder(ResponseDecoderFactory):
         return device(self.response)
 
 class InverterPacketFragment:
+    """ESB Frame"""
     def __init__(self, time_rx=None, payload=None, ch_rx=None, ch_tx=None, **params):
         """
         Callback: get's invoked whenever a Nordic ESB packet has been received.
-        :param p: Payload of the received packet.
+
+        :param time_rx: datetime when frame was received
+        :type time_rx: datetime
+        :param payload: payload bytes
+        :type payload: bytes
+        :param ch_rx: channel where packet was received
+        :type ch_rx: int
+        :param ch_tx: channel where request was sent
+        :type ch_tx: int
         """
 
         if not time_rx:
@@ -132,39 +197,56 @@ class InverterPacketFragment:
 
     @property
     def mid(self):
-        """
+        """Transaction counter"""
-        Transaction counter
-        """
         return self.frame[0]
+
     @property
     def src(self):
         """
-        Sender dddress
+        Sender adddress
+
+        :return: sender address
+        :rtype: int
         """
         src = struct.unpack('>L', self.frame[1:5])
         return src[0]
     @property
     def dst(self):
         """
-        Receiver address
+        Receiver adddress
+
+        :return: receiver address
+        :rtype: int
         """
         dst = struct.unpack('>L', self.frame[5:8])
         return dst[0]
     @property
     def seq(self):
         """
-        Packet sequence
+        Framne sequence number
+
+        :return: sequence number
+        :rtype: int
         """
         result = struct.unpack('>B', self.frame[9:10])
         return result[0]
     @property
     def data(self):
         """
-        Packet without protocol framing
+        Data without protocol framing
+
+        :return: payload chunk
+        :rtype: bytes
         """
         return self.frame[10:-1]
 
     def __str__(self):
+        """
+        Represent received ESB frame
+
+        :return: log line received frame
+        :rtype: str
+        """
         dt = self.time_rx.strftime("%Y-%m-%d %H:%M:%S.%f")
         size = len(self.frame)
         channel = f' channel {self.ch_rx}' if self.ch_rx else ''
@@ -172,6 +254,7 @@ class InverterPacketFragment:
         return f"{dt} Received {size} bytes{channel}: {raw}"
 
 class HoymilesNRF:
+    """Hoymiles NRF24 Interface"""
     tx_channel_id = 0
     tx_channel_list = [40]
     rx_channel_id = 0
@@ -180,11 +263,20 @@ class HoymilesNRF:
     rx_error = 0
 
     def __init__(self, device):
+        """
+        Claim radio device
+
+        :param NRF24 device: instance of NRF24
+        """
         self.radio = device
 
     def transmit(self, packet):
         """
         Transmit Packet
+
+        :param bytes packet: buffer to send
+        :return: if ACK received of ACK disabled
+        :rtype: bool
         """
 
         #dst_esb_addr = b'\x01' + packet[1:5]
@@ -208,6 +300,10 @@ class HoymilesNRF:
     def receive(self, timeout=None):
         """
         Receive Packets
+
+        :param timeout: receive timeout in nanoseconds (default: 12e8)
+        :type timeout: int
+        :yields: fragment
         """
 
         if not timeout:
@@ -257,6 +353,14 @@ class HoymilesNRF:
             time.sleep(0.005)
 
     def next_rx_channel(self):
+        """
+        Select next channel from hop list
+        - if hopping enabled
+        - if channel has no ack
+
+        :return: if new channel selected
+        :rtype: bool
+        """
         if not self.rx_channel_ack:
             self.rx_channel_id = self.rx_channel_id + 1
             if self.rx_channel_id >= len(self.rx_channel_list):
@@ -266,19 +370,52 @@ class HoymilesNRF:
 
     @property
     def tx_channel(self):
+        """
+        Get current tx channel
+
+        :return: tx_channel
+        :rtype: int
+        """
         return self.tx_channel_list[self.tx_channel_id]
 
     @property
     def rx_channel(self):
+        """
+        Get current rx channel
+
+        :return: rx_channel
+        :rtype: int
+        """
         return self.rx_channel_list[self.rx_channel_id]
 
 def frame_payload(payload):
+    """
+    Prepare payload for transmission, append Modbus CRC16
+
+    :param bytes payload: payload to be prepared
+    :return: payload + crc
+    :rtype: bytes
+    """
     payload_crc = f_crc_m(payload)
     payload = payload + struct.pack('>H', payload_crc)
 
     return payload
 
 def compose_esb_fragment(fragment, seq=b'\80', src=99999999, dst=1, **params):
+    """
+    Build standart ESB request fragment
+
+    :param bytes fragment: up to 16 bytes payload chunk
+    :param seq: frame sequence byte
+    :type seq: bytes
+    :param src: dtu address
+    :type src: int
+    :param dst: inverter address
+    :type dst: int
+    :return: esb frame fragment
+    :rtype: bytes
+    :raises ValueError: if fragment size larger 16 byte
+    """
     if len(fragment) > 17:
         raise ValueError(f'ESB fragment exeeds mtu ({mtu}): Fragment size {len(fragment)} bytes')
 
@@ -296,11 +433,27 @@ def compose_esb_fragment(fragment, seq=b'\80', src=99999999, dst=1, **params):
     return p
 
 def compose_esb_packet(packet, mtu=17, **params):
+    """
+    Build ESB packet, chunk packet
+
+    :param bytes packet: payload data
+    :param mtu: maximum transmission unit per frame (default: 17)
+    :type mtu: int
+    :yields: fragment
+    """
     for i in range(0, len(packet), mtu):
         fragment = compose_esb_fragment(packet[i:i+mtu], **params)
         yield(fragment)
 
 def compose_set_time_payload(timestamp=None):
+    """
+    Build set time request packet
+
+    :param timestamp: time to set (default: int(time.time()) )
+    :type timestamp: int
+    :return: payload
+    :rtype: bytes
+    """
     if not timestamp:
         timestamp = int(time.time())
 
@@ -310,22 +463,11 @@ def compose_set_time_payload(timestamp=None):
 
     return frame_payload(payload)
 
-def compose_02_payload(timestamp=None):
-    payload = b'\x02'
-    if timestamp:
-        payload = payload + b'\x00'
-        payload = payload + struct.pack('>L', timestamp)  # big-endian: msb at low address
-        payload = payload + b'\x00\x00\x00\x05\x00\x00\x00\x00'
-
-    return frame_payload(payload)
-
-def compose_11_payload():
-    payload = b'\x11'
-
-    return frame_payload(payload)
-
-
 class InverterTransaction:
+    """
+    Inverter transaction buffer, implements transport-layer functions while
+    communicating with Hoymiles inverters
+    """
     tx_queue = []
     scratch = []
     inverter_ser = None
@@ -341,6 +483,18 @@ class InverterTransaction:
             dtu_ser=None,
             radio=None,
             **params):
+        """
+        :param request: Transmit ESB packet
+        :type request: bytes
+        :param request_time: datetime of transmission
+        :type request_time: datetime
+        :param inverter_ser: inverter serial
+        :type inverter_ser: str
+        :param dtu_ser: DTU serial
+        :type dtu_ser: str
+        :param radio: HoymilesNRF instance to use
+        :type radio: HoymilesNRF or None
+        """
 
         if radio:
             self.radio = radio
@@ -371,6 +525,9 @@ class InverterTransaction:
         """
         Transmit next packet from tx_queue if available
         and wait for responses
+
+        :return: if we got contact
+        :rtype: bool
         """
         if not self.radio:
             return False
@@ -399,15 +556,22 @@ class InverterTransaction:
 
         return wait
 
-    def frame_append(self, payload_frame):
+    def frame_append(self, frame):
         """
         Append received raw frame to local scratch buffer
+
+        :param bytes frame: Received ESB frame
+        :return None
         """
-        self.scratch.append(payload_frame)
+        self.scratch.append(frame)
 
     def queue_tx(self, frame):
         """
         Enqueue packet for transmission if radio is available
+
+        :param bytes frame: ESB frame for transmit
+        :return: if radio is available and frame scheduled
+        :rtype: bool
         """
         if not self.radio:
             return False
@@ -418,7 +582,14 @@ class InverterTransaction:
 
     def get_payload(self, src=None):
         """
-        Reconstruct Hoymiles payload from scratch
+        Reconstruct Hoymiles payload from scratch buffer
+
+        :param src: filter frames by inverter hm_address (default self.inverter_address)
+        :type src: bytes
+        :return: payload
+        :rtype: bytes
+        :raises BufferError: if one or more frames are missing
+        :raises ValueError: if assambled payload fails CRC check
         """
 
         if not src:
@@ -458,6 +629,10 @@ class InverterTransaction:
     def __retransmit_frame(self, frame_id):
         """
         Build and queue retransmit request
+
+        :param int frame_id: frame id to re-schedule
+        :return: if successful scheduled
+        :rtype: bool
         """
         packet = compose_esb_fragment(b'',
                 seq=int(0x80 + frame_id).to_bytes(1, 'big'),
@@ -467,9 +642,22 @@ class InverterTransaction:
         return self.queue_tx(packet)
 
     def __str__(self):
+        """
+        Represent transmit payload
+
+        :return: log line of payload for transmission
+        :rtype: str
+        """
         dt = self.request_time.strftime("%Y-%m-%d %H:%M:%S.%f")
         size = len(self.request)
         return f'{dt} Transmit | {hexify_payload(self.request)}'
 
 def hexify_payload(byte_var):
+    """
+    Represent bytes
+
+    :param bytes byte_var: bytes to be hexlified
+    :return: two-byte while-space padded byte representation
+    :rtype: str
+    """
     return ' '.join([f"{b:02x}" for b in byte_var])

tools/rpi/hoymiles/__main__.py

@@ -0,0 +1,261 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+
+import sys
+import struct
+import re
+import time
+from datetime import datetime
+import argparse
+import hoymiles
+from RF24 import RF24, RF24_PA_LOW, RF24_PA_MAX, RF24_250KBPS, RF24_CRC_DISABLED, RF24_CRC_8, RF24_CRC_16
+import paho.mqtt.client
+import yaml
+from yaml.loader import SafeLoader
+
+def main_loop():
+    """Main loop"""
+    inverters = [
+            inverter for inverter in ahoy_config.get('inverters', [])
+            if not inverter.get('disabled', False)]
+
+    for inverter in inverters:
+        if hoymiles.HOYMILES_DEBUG_LOGGING:
+            print(f'Poll inverter {inverter["serial"]}')
+        poll_inverter(inverter)
+
+def poll_inverter(inverter, retries=4):
+    """
+    Send/Receive command_queue, initiate status poll on inverter
+
+    :param str inverter: inverter serial
+    :param retries: tx retry count if no inverter contact
+    :type retries: int
+    """
+    inverter_ser = inverter.get('serial')
+    dtu_ser = ahoy_config.get('dtu', {}).get('serial')
+
+    # Queue at least status data request
+    command_queue[str(inverter_ser)].append(hoymiles.compose_set_time_payload())
+
+    # Putt all queued commands for current inverter on air
+    while len(command_queue[str(inverter_ser)]) > 0:
+        payload = command_queue[str(inverter_ser)].pop(0)
+
+        # Send payload {ttl}-times until we get at least one reponse
+        payload_ttl = retries
+        while payload_ttl > 0:
+            payload_ttl = payload_ttl - 1
+            com = hoymiles.InverterTransaction(
+                    radio=hmradio,
+                    dtu_ser=dtu_ser,
+                    inverter_ser=inverter_ser,
+                    request=next(hoymiles.compose_esb_packet(
+                        payload,
+                        seq=b'\x80',
+                        src=dtu_ser,
+                        dst=inverter_ser
+                        )))
+            response = None
+            while com.rxtx():
+                try:
+                    response = com.get_payload()
+                    payload_ttl = 0
+                except Exception as e:
+                    print(f'Error while retrieving data: {e}')
+                    pass
+
+        # Handle the response data if any
+        if response:
+            dt = datetime.now()
+            print(f'{dt} Payload: ' + hoymiles.hexify_payload(response))
+            decoder = hoymiles.ResponseDecoder(response,
+                    request=com.request,
+                    inverter_ser=inverter_ser
+                    )
+            result = decoder.decode()
+            if isinstance(result, hoymiles.decoders.StatusResponse):
+                data = result.__dict__()
+                if hoymiles.HOYMILES_DEBUG_LOGGING:
+                    print(f'{dt} Decoded: {data["temperature"]}', end='')
+                    phase_id = 0
+                    for phase in data['phases']:
+                        print(f' phase{phase_id}=voltage:{phase["voltage"]}, current:{phase["current"]}, power:{phase["power"]}, frequency:{data["frequency"]}', end='')
+                        phase_id = phase_id + 1
+                    string_id = 0
+                    for string in data['strings']:
+                        print(f' string{string_id}=voltage:{string["voltage"]}, current:{string["current"]}, power:{string["power"]}, total:{string["energy_total"]/1000}, daily:{string["energy_daily"]}', end='')
+                        string_id = string_id + 1
+                    print()
+
+                if mqtt_client:
+                    mqtt_send_status(mqtt_client, inverter_ser, data,
+                            topic=inverter.get('mqtt', {}).get('topic', None))
+
+def mqtt_send_status(broker, inverter_ser, data, topic=None):
+    """
+    Publish StatusResponse object
+
+    :param paho.mqtt.client.Client broker: mqtt-client instance
+    :param str inverter_ser: inverter serial
+    :param hoymiles.StatusResponse data: decoded inverter StatusResponse
+    :param topic: custom mqtt topic prefix (default: hoymiles/{inverter_ser})
+    :type topic: str
+    """
+
+    if not topic:
+        topic = f'hoymiles/{inverter_ser}'
+
+    # AC Data
+    phase_id = 0
+    for phase in data['phases']:
+        broker.publish(f'{topic}/emeter/{phase_id}/power', phase['power'])
+        broker.publish(f'{topic}/emeter/{phase_id}/voltage', phase['voltage'])
+        broker.publish(f'{topic}/emeter/{phase_id}/current', phase['current'])
+        phase_id = phase_id + 1
+
+    # DC Data
+    string_id = 0
+    for string in data['strings']:
+        broker.publish(f'{topic}/emeter-dc/{string_id}/total', string['energy_total']/1000)
+        broker.publish(f'{topic}/emeter-dc/{string_id}/power', string['power'])
+        broker.publish(f'{topic}/emeter-dc/{string_id}/voltage', string['voltage'])
+        broker.publish(f'{topic}/emeter-dc/{string_id}/current', string['current'])
+        string_id = string_id + 1
+    # Global
+    broker.publish(f'{topic}/frequency', data['frequency'])
+    broker.publish(f'{topic}/temperature', data['temperature'])
+
+def mqtt_on_command(client, userdata, message):
+    """
+    Handle commands to topic
+        hoymiles/{inverter_ser}/command
+    frame a payload and put onto command_queue
+
+    Inverters must have mqtt.send_raw_enabled: true configured
+
+    This can be used to inject debug payloads
+    The message must be in hexlified format
+
+    Use of variables:
+    tttttttt gets expanded to a current int(time)
+
+    Example injects exactly the same as we normally use to poll data:
+      mosquitto -h broker -t inverter_topic/command -m 800b00tttttttt0000000500000000
+
+    This allows for even faster hacking during runtime
+
+    :param paho.mqtt.client.Client client: mqtt-client instance
+    :param dict userdata: Userdata
+    :param dict message: mqtt-client message object
+    """
+    try:
+        inverter_ser = next(
+                item[0] for item in mqtt_command_topic_subs if item[1] == message.topic)
+    except StopIteration:
+        print('Unexpedtedly received mqtt message for {message.topic}')
+
+    if inverter_ser:
+        p_message = message.payload.decode('utf-8').lower()
+
+        # Expand tttttttt to current time for use in hexlified payload
+        expand_time = ''.join(f'{b:02x}' for b in struct.pack('>L', int(time.time())))
+        p_message = p_message.replace('tttttttt', expand_time)
+
+        if (len(p_message) < 2048 \
+            and len(p_message) % 2 == 0 \
+            and re.match(r'^[a-f0-9]+$', p_message)):
+            payload = bytes.fromhex(p_message)
+            # commands must start with \x80
+            if payload[0] == 0x80:
+                command_queue[str(inverter_ser)].append(
+                    hoymiles.frame_payload(payload[1:]))
+
+if __name__ == '__main__':
+    parser = argparse.ArgumentParser(description='Ahoy - Hoymiles solar inverter gateway', prog="hoymiles")
+    parser.add_argument("-c", "--config-file", nargs="?", required=True,
+        help="configuration file")
+    parser.add_argument("--log-transactions", action="store_true", default=False,
+        help="Enable transaction logging output")
+    parser.add_argument("--verbose", action="store_true", default=False,
+        help="Enable debug output")
+    global_config = parser.parse_args()
+
+    # Load ahoy.yml config file
+    try:
+        if isinstance(global_config.config_file, str) == True:
+            with open(global_config.config_file, 'r') as yf:
+                cfg = yaml.load(yf, Loader=SafeLoader)
+        else:
+            with open('ahoy.yml', 'r') as yf:
+                cfg = yaml.load(yf, Loader=SafeLoader)
+    except FileNotFoundError:
+        print("Could not load config file. Try --help")
+        sys.exit(2)
+    except yaml.YAMLError as ye:
+        print('Failed to load config frile {global_config.config_file}: {ye}')
+        sys.exit(1)
+
+    ahoy_config = dict(cfg.get('ahoy', {}))
+
+    # Prepare for multiple transceivers, makes them configurable (currently
+    # only one supported)
+    for radio_config in ahoy_config.get('nrf', [{}]):
+        radio = RF24(
+                radio_config.get('ce_pin', 22),
+                radio_config.get('cs_pin', 0),
+                radio_config.get('spispeed', 1000000))
+        hmradio = hoymiles.HoymilesNRF(device=radio)
+
+    mqtt_client = None
+
+    command_queue = {}
+    mqtt_command_topic_subs = []
+
+    if global_config.log_transactions:
+        hoymiles.HOYMILES_TRANSACTION_LOGGING=True
+    if global_config.verbose:
+        hoymiles.HOYMILES_DEBUG_LOGGING=True
+
+    mqtt_config = ahoy_config.get('mqtt', [])
+    if not mqtt_config.get('disabled', False):
+        mqtt_client = paho.mqtt.client.Client()
+        mqtt_client.username_pw_set(mqtt_config.get('user', None), mqtt_config.get('password', None))
+        mqtt_client.connect(mqtt_config.get('host', '127.0.0.1'), mqtt_config.get('port', 1883))
+        mqtt_client.loop_start()
+        mqtt_client.on_message = mqtt_on_command
+
+    if not radio.begin():
+        raise RuntimeError('Can\'t open radio')
+
+    inverters = [inverter.get('serial') for inverter in ahoy_config.get('inverters', [])]
+    for inverter in ahoy_config.get('inverters', []):
+        inverter_ser = inverter.get('serial')
+        command_queue[str(inverter_ser)] = []
+
+        #
+        # Enables and subscribe inverter to mqtt /command-Topic
+        #
+        if inverter.get('mqtt', {}).get('send_raw_enabled', False):
+            topic_item = (
+                    str(inverter_ser),
+                    inverter.get('mqtt', {}).get('topic', f'hoymiles/{inverter_ser}') + '/command'
+                    )
+            mqtt_client.subscribe(topic_item[1])
+            mqtt_command_topic_subs.append(topic_item)
+
+    loop_interval = ahoy_config.get('interval', 1)
+    try:
+        while True:
+            t_loop_start = time.time()
+
+            main_loop()
+
+            print('', end='', flush=True)
+
+            if loop_interval > 0 and (time.time() - t_loop_start) < loop_interval:
+                time.sleep(time.time() % loop_interval)
+
+    except KeyboardInterrupt:
+        radio.powerDown()
+        sys.exit()

tools/rpi/hoymiles/decoders/__init__.py

@@ -1,16 +1,35 @@
 #!/usr/bin/python3
 # -*- coding: utf-8 -*-
 import struct
+import crcmod
+from datetime import timedelta
+
+f_crc_m = crcmod.predefined.mkPredefinedCrcFun('modbus')
 
 class StatusResponse:
+    """Inverter StatusResponse object"""
     e_keys  = ['voltage','current','power','energy_total','energy_daily']
 
     def unpack(self, fmt, base):
+        """
+        Data unpack helper
+
+        :param str fmt: struct format string
+        :param int base: unpack base position from self.response bytes
+        :return: unpacked values
+        :rtype: tuple
+        """
         size = struct.calcsize(fmt)
         return struct.unpack(fmt, self.response[base:base+size])
 
     @property
     def phases(self):
+        """
+        AC power data
+
+        :retrun: list of dict's
+        :rtype: list
+        """
         phases = []
         p_exists = True
         while p_exists:
@@ -29,6 +48,12 @@ class StatusResponse:
 
     @property
     def strings(self):
+        """
+        DC PV-string data
+
+        :retrun: list of dict's
+        :rtype: list
+        """
         strings = []
         s_exists = True
         while s_exists:
@@ -46,6 +71,12 @@ class StatusResponse:
         return strings
 
     def __dict__(self):
+        """
+        Get all known data
+
+        :return: dict of properties
+        :rtype: dict
+        """
         data = {}
         data['phases'] = self.phases
         data['strings'] = self.strings
@@ -54,55 +85,255 @@ class StatusResponse:
         return data
 
 class UnknownResponse:
+    """
+    Debugging helper for unknown payload format
+    """
     @property
     def hex_ascii(self):
+        """
+        Generate white-space separated byte representation
+
+        :return: hexlifierd byte string
+        :rtype: str
+        """
         return ' '.join([f'{b:02x}' for b in self.response])
 
+    @property
+    def valid_crc(self):
+        """
+        Checks if self.response has valid Modbus CRC
+
+        :return: if crc is available and correct
+        :rtype: bool
+        """
+        # check crc
+        pcrc = struct.unpack('>H', self.response[-2:])[0]
+        return f_crc_m(self.response[:-2]) == pcrc
+
     @property
     def dump_longs(self):
+        """Get all data, interpreted as long"""
+        if len(self.response) < 5:
+            return None
+
         res = self.response
-        n = len(res)/4
+
+        r = len(res) % 16
+        res = res[:r*-1]
 
         vals = None
-        if n % 4 == 0:
+        if len(res) % 16 == 0:
+            n = len(res)/4
             vals = struct.unpack(f'>{int(n)}L', res)
 
         return vals
 
     @property
     def dump_longs_pad1(self):
-        res = self.response[1:]
+        """Get all data, interpreted as long"""
-        n = len(res)/4
+        if len(self.response) < 7:
+            return None
+
+        res = self.response[2:]
+
+        r = len(res) % 16
+        res = res[:r*-1]
 
         vals = None
-        if n % 4 == 0:
+        if len(res) % 16 == 0:
+            n = len(res)/4
+            vals = struct.unpack(f'>{int(n)}L', res)
+
+        return vals
+
+    @property
+    def dump_longs_pad2(self):
+        """Get all data, interpreted as long"""
+        if len(self.response) < 9:
+            return None
+
+        res = self.response[4:]
+
+        r = len(res) % 16
+        res = res[:r*-1]
+
+        vals = None
+        if len(res) % 16 == 0:
+            n = len(res)/4
+            vals = struct.unpack(f'>{int(n)}L', res)
+
+        return vals
+
+    @property
+    def dump_longs_pad3(self):
+        """Get all data, interpreted as long"""
+        if len(self.response) < 11:
+            return None
+
+        res = self.response[6:]
+
+        r = len(res) % 16
+        res = res[:r*-1]
+
+        vals = None
+        if len(res) % 16 == 0:
+            n = len(res)/4
             vals = struct.unpack(f'>{int(n)}L', res)
 
         return vals
 
     @property
     def dump_shorts(self):
-        n = len(self.response)/2
+        """Get all data, interpreted as short"""
+        if len(self.response) < 5:
+            return None
+
+        res = self.response
+
+        r = len(res) % 4
+        res = res[:r*-1]
 
         vals = None
-        if n % 2 == 0:
+        if len(res) % 4 == 0:
-            vals = struct.unpack(f'>{int(n)}H', self.response)
+            n = len(res)/2
+            vals = struct.unpack(f'>{int(n)}H', res)
+
         return vals
 
     @property
     def dump_shorts_pad1(self):
+        """Get all data, interpreted as short"""
+        if len(self.response) < 6:
+            return None
+
         res = self.response[1:]
-        n = len(res)/2
+
+        r = len(res) % 4
+        res = res[:r*-1]
 
         vals = None
-        if n % 2 == 0:
+        if len(res) % 4 == 0:
+            n = len(res)/2
             vals = struct.unpack(f'>{int(n)}H', res)
+
         return vals
 
+class EventsResponse(UnknownResponse):
+
+    alarm_codes = {
+            1: 'Inverter start',
+            2: 'Producing power',
+            121: 'Over temperature protection',
+            125: 'Grid configuration parameter error',
+            126: 'Software error code 126',
+            127: 'Firmware error',
+            128: 'Software error code 128',
+            129: 'Software error code 129',
+            130: 'Offline',
+            141: 'Grid overvoltage',
+            142: 'Average grid overvoltage',
+            143: 'Grid undervoltage',
+            144: 'Grid overfrequency',
+            145: 'Grid underfrequency',
+            146: 'Rapid grid frequency change',
+            147: 'Power grid outage',
+            148: 'Grid disconnection',
+            149: 'Island detected',
+            205: 'Input port 1 & 2 overvoltage',
+            206: 'Input port 3 & 4 overvoltage',
+            207: 'Input port 1 & 2 undervoltage',
+            208: 'Input port 3 & 4 undervoltage',
+            209: 'Port 1 no input',
+            210: 'Port 2 no input',
+            211: 'Port 3 no input',
+            212: 'Port 4 no input',
+            213: 'PV-1 & PV-2 abnormal wiring',
+            214: 'PV-3 & PV-4 abnormal wiring',
+            215: 'PV-1 Input overvoltage',
+            216: 'PV-1 Input undervoltage',
+            217: 'PV-2 Input overvoltage',
+            218: 'PV-2 Input undervoltage',
+            219: 'PV-3 Input overvoltage',
+            220: 'PV-3 Input undervoltage',
+            221: 'PV-4 Input overvoltage',
+            222: 'PV-4 Input undervoltage',
+            301: 'Hardware error code 301',
+            302: 'Hardware error code 302',
+            303: 'Hardware error code 303',
+            304: 'Hardware error code 304',
+            305: 'Hardware error code 305',
+            306: 'Hardware error code 306',
+            307: 'Hardware error code 307',
+            308: 'Hardware error code 308',
+            309: 'Hardware error code 309',
+            310: 'Hardware error code 310',
+            311: 'Hardware error code 311',
+            312: 'Hardware error code 312',
+            313: 'Hardware error code 313',
+            314: 'Hardware error code 314',
+            5041: 'Error code-04 Port 1',
+            5042: 'Error code-04 Port 2',
+            5043: 'Error code-04 Port 3',
+            5044: 'Error code-04 Port 4',
+            5051: 'PV Input 1 Overvoltage/Undervoltage',
+            5052: 'PV Input 2 Overvoltage/Undervoltage',
+            5053: 'PV Input 3 Overvoltage/Undervoltage',
+            5054: 'PV Input 4 Overvoltage/Undervoltage',
+            5060: 'Abnormal bias',
+            5070: 'Over temperature protection',
+            5080: 'Grid Overvoltage/Undervoltage',
+            5090: 'Grid Overfrequency/Underfrequency',
+            5100: 'Island detected',
+            5120: 'EEPROM reading and writing error',
+            5150: '10 min value grid overvoltage',
+            5200: 'Firmware error',
+            8310: 'Shut down',
+            9000: 'Microinverter is suspected of being stolen'
+            }
+
+    def __init__(self, response):
+        self.response = response
+
+        crc_valid = self.valid_crc
+        if crc_valid:
+            print(' payload has valid modbus crc')
+            self.response = response[:-2]
+
+        status = self.response[:2]
+
+        chunk_size = 12
+        for c in range(2, len(self.response), chunk_size):
+            chunk = self.response[c:c+chunk_size]
+
+            print(' '.join([f'{b:02x}' for b in chunk]) + ': ')
+
+            opcode, a_code, a_count, uptime_sec = struct.unpack('>BBHH', chunk[0:6])
+            a_text = self.alarm_codes.get(a_code, 'N/A')
+
+            print(f' uptime={timedelta(seconds=uptime_sec)} a_count={a_count} opcode={opcode} a_code={a_code} a_text={a_text}')
+
+            for fmt in ['BBHHHHH']:
+                print(f' {fmt:7}: ' + str(struct.unpack('>' + fmt, chunk)))
+            print(end='', flush=True)
+
 class DEBUG_DecodeAny(UnknownResponse):
+    """Default decoder"""
     def __init__(self, response):
+        """
+        Try interpret and print unknown response data
+
+        :param bytes response: response payload bytes
+        """
         self.response = response
 
+        crc_valid = self.valid_crc
+        if crc_valid:
+            print(' payload has valid modbus crc')
+            self.response = response[:-2]
+
+        l_payload = len(self.response)
+        print(f' payload has {l_payload} bytes')
+
         longs = self.dump_longs
         if not longs:
             print(' type long      : unable to decode (len or not mod 4)')
@@ -115,6 +346,18 @@ class DEBUG_DecodeAny(UnknownResponse):
         else:
             print(' type long pad1 : ' + str(longs))
 
+        longs = self.dump_longs_pad2
+        if not longs:
+            print(' type long pad2 : unable to decode (len or not mod 4)')
+        else:
+            print(' type long pad2 : ' + str(longs))
+
+        longs = self.dump_longs_pad3
+        if not longs:
+            print(' type long pad3 : unable to decode (len or not mod 4)')
+        else:
+            print(' type long pad3 : ' + str(longs))
+
         shorts = self.dump_shorts
         if not shorts:
             print(' type short     : unable to decode (len or not mod 2)')
@@ -127,6 +370,18 @@ class DEBUG_DecodeAny(UnknownResponse):
         else:
             print(' type short pad1: ' + str(shorts))
 
+        try:
+            if len(self.response) > 2:
+                print(' type utf-8  : ' + self.response.decode('utf-8'))
+        except UnicodeDecodeError:
+            print(' type utf-8  : utf-8 decode error')
+
+        try:
+            if len(self.response) > 2:
+                print(' type ascii  : ' + self.response.decode('ascii'))
+        except UnicodeDecodeError:
+            print(' type ascii  : ascii decode error')
+
 
 # 1121-Series Intervers, 1 MPPT, 1 Phase
 class HM300_Decode0B(StatusResponse):
@@ -166,6 +421,15 @@ class HM300_Decode0B(StatusResponse):
     def temperature(self):
         return self.unpack('>H', 26)[0]/10
 
+class HM300_Decode11(EventsResponse):
+    def __init__(self, response):
+        super().__init__(response)
+
+class HM300_Decode12(EventsResponse):
+    def __init__(self, response):
+        super().__init__(response)
+
+
 
 # 1141-Series Inverters, 2 MPPT, 1 Phase
 class HM600_Decode0B(StatusResponse):
@@ -220,9 +484,13 @@ class HM600_Decode0B(StatusResponse):
     def temperature(self):
         return self.unpack('>H', 38)[0]/10
 
-class HM600_Decode0C(HM600_Decode0B):
+class HM600_Decode11(EventsResponse):
     def __init__(self, response):
-        self.response = response
+        super().__init__(response)
+
+class HM600_Decode12(EventsResponse):
+    def __init__(self, response):
+        super().__init__(response)
 
 
 # 1161-Series Inverters, 4 MPPT, 1 Phase
@@ -309,3 +577,11 @@ class HM1200_Decode0B(StatusResponse):
     @property
     def temperature(self):
         return self.unpack('>H', 58)[0]/10
+
+class HM1200_Decode11(EventsResponse):
+    def __init__(self, response):
+        super().__init__(response)
+
+class HM1200_Decode12(EventsResponse):
+    def __init__(self, response):
+        super().__init__(response)