Feature: Dump hex data from meter to MQTT (#1071)

* Send raw data debug to MQTT

* Publish hexdump to /data

* Sensor for /data, but it doesnt work

lib/AmsMqttHandler/include/AmsMqttHandler.h

@@ -57,7 +57,7 @@ public:
     virtual bool publishTemperatures(AmsConfiguration*, HwTools*) { return false; };
     virtual bool publishPrices(PriceService* ps) { return false; };
     virtual bool publishSystem(HwTools*, PriceService*, EnergyAccounting*) { return false; };
-    virtual bool publishRaw(String data) { return false; };
+    virtual bool publishRaw(uint8_t* raw, size_t length) { return false; };
     virtual bool publishFirmware() { return false; };
     virtual void onMessage(String &topic, String &payload) {};
 

lib/DomoticzMqttHandler/include/DomoticzMqttHandler.h

@@ -25,7 +25,7 @@ public:
     bool publishTemperatures(AmsConfiguration*, HwTools*);
     bool publishPrices(PriceService*);
     bool publishSystem(HwTools* hw, PriceService* ps, EnergyAccounting* ea);
-    bool publishRaw(String data);
+    bool publishRaw(uint8_t* raw, size_t length);
 
     void onMessage(String &topic, String &payload);
 

lib/DomoticzMqttHandler/src/DomoticzMqttHandler.cpp

@@ -103,7 +103,7 @@ uint8_t DomoticzMqttHandler::getFormat() {
     return 3;
 }
 
-bool DomoticzMqttHandler::publishRaw(String data) {
+bool DomoticzMqttHandler::publishRaw(uint8_t* raw, size_t length) {
     return false;
 }
 

lib/HomeAssistantMqttHandler/include/HomeAssistantMqttHandler.h

@@ -27,7 +27,7 @@ public:
     bool publishTemperatures(AmsConfiguration*, HwTools*);
     bool publishPrices(PriceService*);
     bool publishSystem(HwTools* hw, PriceService* ps, EnergyAccounting* ea);
-    bool publishRaw(String data);
+    bool publishRaw(uint8_t* raw, size_t length);
     bool publishFirmware();
 
     bool postConnect();
@@ -51,7 +51,7 @@ private:
     String updateTopic;
     String sensorNamePrefix;
 
-    bool l1Init, l2Init, l2eInit, l3Init, l3eInit, l4Init, l4eInit, rtInit, rteInit, pInit, sInit, rInit, fInit;
+    bool l1Init, l2Init, l2eInit, l3Init, l3eInit, l4Init, l4eInit, rtInit, rteInit, pInit, sInit, rInit, fInit, dInit;
     bool tInit[32] = {false};
     uint8_t priceImportInit = 0, priceExportInit = 0;
     uint32_t lastThresholdPublish = 0;

lib/HomeAssistantMqttHandler/include/HomeAssistantStatic.h

@@ -124,5 +124,6 @@ const HomeAssistantSensor SystemSensors[SystemSensorCount] PROGMEM = {
 
 const HomeAssistantSensor TemperatureSensor PROGMEM = {"Temperature sensor %s", "/temperatures", "temperatures['%s']", 900, "°C", "temperature", "measurement", ""};
 
+const HomeAssistantSensor DataSensor PROGMEM = {"Data", "/data", "data", 900, "", "", "", ""};
 
 #endif

lib/HomeAssistantMqttHandler/json/hadiscover.json

@@ -3,7 +3,6 @@
     "stat_t" : "%s%s",
     "uniq_id" : "%s_%s",
     "obj_id" : "%s_%s",
-    "unit_of_meas" : "%s",
     "val_tpl" : "{{ value_json.%s | is_defined }}",
     "expire_after" : %d,
     "dev" : {
@@ -13,5 +12,8 @@
       "sw" : "%s",
       "mf" : "%s",
       "cu" : "%s"
-    }%s%s%s%s%s%s
+    }
+    %s%s%s
+    %s%s%s
+    %s%s%s
 }

lib/HomeAssistantMqttHandler/src/HomeAssistantMqttHandler.cpp

@@ -20,7 +20,7 @@
 #endif
 
 void HomeAssistantMqttHandler::setHomeAssistantConfig(HomeAssistantConfig config, char* hostname) {
-    l1Init = l2Init = l2eInit = l3Init = l3eInit = l4Init = l4eInit = rtInit = rteInit = pInit = sInit = rInit = fInit = false;
+    l1Init = l2Init = l2eInit = l3Init = l3eInit = l4Init = l4eInit = rtInit = rteInit = pInit = sInit = rInit = fInit = dInit = false;
 
     if(strlen(config.discoveryNameTag) > 0) {
         snprintf_P(json, 128, PSTR("AMS reader (%s)"), config.discoveryNameTag);
@@ -541,7 +541,6 @@ void HomeAssistantMqttHandler::publishSensor(const HomeAssistantSensor sensor) {
         mqttConfig.publishTopic, sensor.topic,
         deviceUid.c_str(), uid.c_str(),
         deviceUid.c_str(), uid.c_str(),
-        sensor.uom,
         sensor.path,
         sensor.ttl,
         deviceUid.c_str(),
@@ -550,13 +549,20 @@ void HomeAssistantMqttHandler::publishSensor(const HomeAssistantSensor sensor) {
         FirmwareVersion::VersionString,
         manufacturer.c_str(),
         deviceUrl.c_str(),
+
         strlen_P(sensor.devcl) > 0 ? ",\"dev_cla\":\"" : "",
         strlen_P(sensor.devcl) > 0 ? (char *) FPSTR(sensor.devcl) : "",
         strlen_P(sensor.devcl) > 0 ? "\"" : "",
+
         strlen_P(sensor.stacl) > 0 ? ",\"stat_cla\":\"" : "",
         strlen_P(sensor.stacl) > 0 ? (char *) FPSTR(sensor.stacl) : "",
-        strlen_P(sensor.stacl) > 0 ? "\"" : ""
+        strlen_P(sensor.stacl) > 0 ? "\"" : "",
+
+        strlen_P(sensor.uom) > 0 ? ",\"unit_of_meas\":\"" : "",
+        strlen_P(sensor.uom) > 0 ? (char *) FPSTR(sensor.uom) : "",
+        strlen_P(sensor.uom) > 0 ? "\"" : ""
     );
+
     mqtt.publish(sensorTopic + "/" + deviceUid + "_" + uid + "/config", json, true, 0);
     loop();
 }
@@ -831,8 +837,26 @@ uint8_t HomeAssistantMqttHandler::getFormat() {
     return 4;
 }
 
-bool HomeAssistantMqttHandler::publishRaw(String data) {
+bool HomeAssistantMqttHandler::publishRaw(uint8_t* raw, size_t length) {
-    return false;
+	if(strlen(mqttConfig.publishTopic) == 0 || !mqtt.connected())
+		return false;
+    
+    if(length <= 0 || length > BufferSize) return false;
+
+    if(!dInit) {
+        // Not sure how this sensor should be defined in HA, so skipping for now
+        //publishSensor(DataSensor);
+        dInit = true;
+    }
+
+    String str = toHex(raw, length);
+
+    snprintf_P(json, BufferSize, PSTR("{\"data\":\"%s\"}"), str.c_str());
+    char topic[192];
+    snprintf_P(topic, 192, PSTR("%s/data"), mqttConfig.publishTopic);
+    bool ret = mqtt.publish(topic, json);
+    loop();
+    return ret;
 }
 
 bool HomeAssistantMqttHandler::publishFirmware() {
@@ -865,7 +889,7 @@ void HomeAssistantMqttHandler::onMessage(String &topic, String &payload) {
             if (debugger->isActive(RemoteDebug::INFO))
             #endif
             debugger->printf_P(PSTR("Received online status from HA, resetting sensor status\n"));
-            l1Init = l2Init = l2eInit = l3Init = l3eInit = l4Init = l4eInit = rtInit = rteInit = pInit = sInit = rInit = false;
+            l1Init = l2Init = l2eInit = l3Init = l3eInit = l4Init = l4eInit = rtInit = rteInit = pInit = sInit = rInit = fInit = dInit = false;
             for(uint8_t i = 0; i < 32; i++) tInit[i] = false;
             priceImportInit = 0;
             priceExportInit = 0;

lib/JsonMqttHandler/include/JsonMqttHandler.h

@@ -23,7 +23,7 @@ public:
     bool publishTemperatures(AmsConfiguration*, HwTools*);
     bool publishPrices(PriceService*);
     bool publishSystem(HwTools* hw, PriceService* ps, EnergyAccounting* ea);
-    bool publishRaw(String data);
+    bool publishRaw(uint8_t* raw, size_t length);
     bool publishFirmware();
 
     void onMessage(String &topic, String &payload);

lib/JsonMqttHandler/src/JsonMqttHandler.cpp

@@ -472,8 +472,20 @@ uint8_t JsonMqttHandler::getFormat() {
     return 0;
 }
 
-bool JsonMqttHandler::publishRaw(String data) {
+bool JsonMqttHandler::publishRaw(uint8_t* raw, size_t length) {
-    return false;
+	if(strlen(mqttConfig.publishTopic) == 0 || !mqtt.connected())
+		return false;
+    
+    if(length <= 0 || length > BufferSize) return false;
+
+    String str = toHex(raw, length);
+
+    snprintf_P(json, BufferSize, PSTR("{\"data\":\"%s\"}"), str.c_str());
+    char topic[192];
+    snprintf_P(topic, 192, PSTR("%s/data"), mqttConfig.publishTopic);
+    bool ret = mqtt.publish(topic, json);
+    loop();
+    return ret;
 }
 
 bool JsonMqttHandler::publishFirmware() {

lib/MeterCommunicators/include/MeterCommunicator.h

@@ -13,6 +13,7 @@
 #endif
 #include "AmsData.h"
 #include "AmsConfiguration.h"
+#include "AmsMqttHandler.h"
 
 class MeterCommunicator {
 public:
@@ -24,6 +25,13 @@ public:
     virtual bool isConfigChanged();
     virtual void ackConfigChanged();
     virtual void getCurrentConfig(MeterConfig& meterConfig);
+    virtual void setMqttHandlerForDebugging(AmsMqttHandler* mqttHandler) {
+        this->mqttDebug = mqttHandler;
+    };
+
+protected:
+    AmsMqttHandler* mqttDebug = NULL;
+
 };
 
 #endif

lib/MeterCommunicators/include/PassiveMeterCommunicator.h

@@ -14,7 +14,7 @@
 #include "AmsConfiguration.h"
 #include "DataParsers.h"
 #include "Timezone.h"
-#include "PassthroughMqttHandler.h"
+#include "AmsMqttHandler.h"
 
 #if defined(ESP8266)
 #include "SoftwareSerial.h"
@@ -36,7 +36,6 @@ public:
     bool isConfigChanged();
     void ackConfigChanged();
     void getCurrentConfig(MeterConfig& meterConfig);
-    void setPassthroughMqttHandler(PassthroughMqttHandler*);
 
     HardwareSerial* getHwSerial();
     void rxerr(int err);
@@ -51,8 +50,6 @@ protected:
     bool configChanged = false;
     Timezone* tz;
 
-    PassthroughMqttHandler* pt = NULL;
-
     uint8_t *hanBuffer = NULL;
     uint16_t hanBufferSize = 0;
     Stream *hanSerial;

lib/MeterCommunicators/src/PassiveMeterCommunicator.cpp

@@ -174,8 +174,8 @@ bool PassiveMeterCommunicator::loop() {
         lastError = pos;
 		printHanReadError(pos);
 		len += hanSerial->readBytes(hanBuffer+len, hanBufferSize-len);
-        if(pt != NULL) {
+        if(mqttDebug != NULL) {
-            pt->publishBytes(hanBuffer, len);
+            mqttDebug->publishRaw(hanBuffer, len);
         }
 		#if defined(AMS_REMOTE_DEBUG)
 		if (debugger->isActive(RemoteDebug::VERBOSE))
@@ -229,8 +229,8 @@ AmsData* PassiveMeterCommunicator::getData(AmsData& meterState) {
 	char* payload = ((char *) (hanBuffer)) + pos;
 	if(maxDetectedPayloadSize < pos) maxDetectedPayloadSize = pos;
 	if(ctx.type == DATA_TAG_DLMS) {
-        if(pt != NULL) {
+        if(mqttDebug != NULL) {
-            pt->publishBytes((uint8_t*) payload, ctx.length);
+            mqttDebug->publishRaw((uint8_t*) payload, ctx.length);
         }
 
 		#if defined(AMS_REMOTE_DEBUG)
@@ -405,8 +405,8 @@ int16_t PassiveMeterCommunicator::unwrapData(uint8_t *buf, DataParserContext &co
 				if (debugger->isActive(RemoteDebug::VERBOSE))
 				#endif
 				debugger->printf_P(PSTR("HDLC frame:\n"));
-                if(pt != NULL) {
+                if(mqttDebug != NULL) {
-                    pt->publishBytes(buf, curLen);
+                    mqttDebug->publishRaw(buf, curLen);
                 }
                 break;
             case DATA_TAG_MBUS:
@@ -414,8 +414,8 @@ int16_t PassiveMeterCommunicator::unwrapData(uint8_t *buf, DataParserContext &co
 				if (debugger->isActive(RemoteDebug::VERBOSE))
 				#endif
 				debugger->printf_P(PSTR("MBUS frame:\n"));
-                if(pt != NULL) {
+                if(mqttDebug != NULL) {
-                    pt->publishBytes(buf, curLen);
+                    mqttDebug->publishRaw(buf, curLen);
                 }
                 break;
             case DATA_TAG_GBT:
@@ -447,8 +447,8 @@ int16_t PassiveMeterCommunicator::unwrapData(uint8_t *buf, DataParserContext &co
 				if (debugger->isActive(RemoteDebug::VERBOSE))
 				#endif
 				debugger->printf_P(PSTR("DSMR frame:\n"));
-                if(pt != NULL) {
+                if(mqttDebug != NULL) {
-                    pt->publishString((char*) buf);
+                    mqttDebug->publishRaw(buf, curLen);
                 }
                 break;
 			case DATA_TAG_SNRM:

lib/RawMqttHandler/include/RawMqttHandler.h

@@ -26,7 +26,7 @@ public:
     bool publishTemperatures(AmsConfiguration*, HwTools*);
     bool publishPrices(PriceService*);
     bool publishSystem(HwTools* hw, PriceService* ps, EnergyAccounting* ea);
-    bool publishRaw(String data);
+    bool publishRaw(uint8_t* raw, size_t length);
 
     void onMessage(String &topic, String &payload);
 

lib/RawMqttHandler/src/RawMqttHandler.cpp

@@ -396,8 +396,16 @@ uint8_t RawMqttHandler::getFormat() {
     return full ? 3 : 2;
 }
 
-bool RawMqttHandler::publishRaw(String data) {
+bool RawMqttHandler::publishRaw(uint8_t* raw, size_t length) {
-    return false;
+	if(topic.isEmpty() || !connected())
+		return false;
+    
+    if(length <= 0 || length > BufferSize) return false;
+
+    String str = toHex(raw, length);
+    bool ret = mqtt.publish(topic + "/data", str);
+    loop();
+    return ret;
 }
 
 void RawMqttHandler::onMessage(String &topic, String &payload) {

src/AmsToMqttBridge.cpp

@@ -1041,6 +1041,9 @@ void handleMeterConfig() {
 			debugE_P(PSTR("Unknown meter source selected: %d"), meterConfig.source);
 		}
 		ws.setMeterConfig(meterConfig.distributionSystem, meterConfig.mainFuse, meterConfig.productionCapacity);
+		if(mc != NULL && Debug.isActive(RemoteDebug::DEBUG)) {
+			mc->setMqttHandlerForDebugging(mqttHandler);
+		}
 		config.ackMeterChanged();
 	}
 }
@@ -1557,6 +1560,9 @@ void postConnect() {
 		if(!debug.telnet) {
 			Debug.stop();
 		}
+		if(mc != NULL && Debug.isActive(RemoteDebug::DEBUG)) {
+			mc->setMqttHandlerForDebugging(mqttHandler);
+		}
 	} else {
 		Debug.stop();
 	}
@@ -1632,6 +1638,7 @@ void MQTT_connect() {
 		if(mqttHandler->getFormat() != mqttConfig.payloadFormat) {
 			delete mqttHandler;
 			mqttHandler = NULL;
+			mc->setMqttHandlerForDebugging(NULL);
 		} else if(config.isMqttChanged()) {
 			mqttHandler->setConfig(mqttConfig);
 			switch(mqttConfig.payloadFormat) {
@@ -1680,10 +1687,14 @@ void MQTT_connect() {
 				break;
 			case 255:
 				mqttHandler = new PassthroughMqttHandler(mqttConfig, &Debug, (char*) commonBuffer, &updater);
+				mc->setMqttHandlerForDebugging(mqttHandler);
 				break;
 		}
 	}
 	ws.setMqttHandler(mqttHandler);
+	if(mc != NULL && Debug.isActive(RemoteDebug::DEBUG)) {
+		mc->setMqttHandlerForDebugging(mqttHandler);
+	}
 
 	if(mqttHandler != NULL) {
 		mqttHandler->connect();