160mhz esp32 and some error handling in loop()

doc/Denmark/Kamstrup_encrypted_OBIS.txt

@@ -2,7 +2,7 @@
 1.1.2.8.0.255 - Active- Energy
 1.1.3.8.0.255 - Reactive+ Energy
 1.1.4.8.0.255 - Reactive- Energy
-1.1.0.0.1.255 - Electricity ID?
+1.1.0.0.1.255 - Meter number 1
 1.1.1.7.0.255 - Active+ Instantaneous value
 1.1.2.7.0.255 - Active- Instantaneous value
 1.1.3.7.0.255 - Reactive+ Instantaneous value

frames/Kamstrup-1p.raw

@@ -0,0 +1,51 @@
+7E A0 BA 2B 21 13 ED AA E6 E7 00 0F 00 00 00 00
+0C 07 E6 02 05 06 0D 00 0A FF 80 00 00 
+02 19 
+0A 0E 4B 61 6D 73 74 72 75 70 5F 56 30 30 30 31 
+09 06 01 01 00 00 05 FF 0A 10 35 37 30 36 35 36 37 32 37 31 35 33 33 32 30 37 
+09 06 01 01 60 01 01 FF 0A 12 36 38 36 31 31 31 31 42 4E 32 34 32 31 30 31 30 34 30 
+09 06 01 01 01 07 00 FF 06 00 00 02 68 
+09 06 01 01 02 07 00 FF 06 00 00 00 00 
+09 06 01 01 03 07 00 FF 06 00 00 00 53 
+09 06 01 01 04 07 00 FF 06 00 00 00 00 
+09 06 01 01 1F 07 00 FF 06 00 00 01 22 
+00 00 00 00 
+09 06 01 01 20 07 00 FF 12 00 E2 
+00 00 00 00 
+05 D8 7E
+
+7E A0 BA 2B 21 13 ED AA E6 E7 00 0F 00 00 00 00
+0C 07 E6 02 05 06 0D 00 14 FF 80 00 00 
+02 19 
+0A 0E 4B 61 6D 73 74 72 75 70 5F 56 30 30 30 31 
+09 06 01 01 00 00 05 FF 0A 10 35 37 30 36 35 36 37 32 37 31 35 33 33 32 30 37 
+09 06 01 01 60 01 01 FF 0A 12 36 38 36 31 31 31 31 42 4E 32 34 32 31 30 31 30 34 30 
+09 06 01 01 01 07 00 FF 06 00 00 02 68 
+09 06 01 01 02 07 00 FF 06 00 00 00 00 
+09 06 01 01 03 07 00 FF 06 00 00 00 53 
+09 06 01 01 04 07 00 FF 06 00 00 00 00 
+09 06 01 01 1F 07 00 FF 06 00 00 01 23 
+00 00 00 00 
+09 06 01 01 20 07 00 FF 12 00 E1 
+00 00 00 00 
+8E 5E 7E
+
+7E A1 04 2B 21 13 77 6E E6 E7 00 0F 00 00 00 00
+0C 07 E6 02 05 06 0D 00 19 FF 80 00 00 
+02 23 0A 0E 4B 61 6D 73 74 72 75 70 5F 56 30 30 30 31 
+09 06 01 01 00 00 05 FF 0A 10 35 37 30 36 35 36 37 32 37 31 35 33 33 32 30 37 
+09 06 01 01 60 01 01 FF 0A 12 36 38 36 31 31 31 31 42 4E 32 34 32 31 30 31 30 34 30 
+09 06 01 01 01 07 00 FF 06 00 00 02 6B 
+09 06 01 01 02 07 00 FF 06 00 00 00 00 
+09 06 01 01 03 07 00 FF 06 00 00 00 54 
+09 06 01 01 04 07 00 FF 06 00 00 00 00 
+09 06 01 01 1F 07 00 FF 06 00 00 01 25 
+00 00 00 00 
+09 06 01 01 20 07 00 FF 12 00 E1 
+00 00 00 00 
+09 06 00 01 01 00 00 FF 09 0C 07 E6 02 05 06 0D 00 19 FF 80 00 00 
+09 06 01 01 01 08 00 FF 06 00 12 CF 93 
+09 06 01 01 02 08 00 FF 06 00 00 00 00
+09 06 01 01 03 08 00 FF 06 00 00 8C CE 
+09 06 01 01 04 08 00 FF 06 00 05 E5 04 
+7F E9 7E

platformio.ini

@@ -17,11 +17,12 @@ extra_scripts =
     scripts/makeweb.py
 build_flags =
     -D WEBSOCKET_DISABLED=1
+build_unflags = -fno-exceptions
 
 
 [env:esp32]
 platform = https://github.com/platformio/platform-espressif32.git#feature/arduino-upstream
-board = esp32dev
+board = esp32doit-devkit-v1
 framework = arduino
 lib_deps = ${common.lib_deps}
 lib_ignore = ${common.lib_ignore}
@@ -30,3 +31,4 @@ extra_scripts =
     scripts/makeweb.py
 build_flags =
     -D WEBSOCKET_DISABLED=1
+board_build.f_cpu = 160000000L

src/AmsData.h

@@ -69,7 +69,7 @@ protected:
     String listId, meterId, meterModel;
     time_t meterTimestamp = 0;
     uint16_t activeImportPower = 0, reactiveImportPower = 0, activeExportPower = 0, reactiveExportPower = 0;
-    float l1voltage = 0, l2voltage = 0, l3voltage = 0, l1current = 0, l2current = 0, l3current = 0;
+    double l1voltage = 0, l2voltage = 0, l3voltage = 0, l1current = 0, l2current = 0, l3current = 0;
     float powerFactor = 0, l1PowerFactor = 0, l2PowerFactor = 0, l3PowerFactor = 0;
     float activeImportCounter = 0, reactiveImportCounter = 0, activeExportCounter = 0, reactiveExportCounter = 0;
     bool threePhase = false, twoPhase = false, counterEstimated = false;

src/AmsDataStorage.cpp

@@ -253,7 +253,7 @@ void AmsDataStorage::setHour(uint8_t hour, int32_t val) {
     day.points[hour] = val / 10;
 }
 
-int16_t AmsDataStorage::getHour(uint8_t hour) {
+int32_t AmsDataStorage::getHour(uint8_t hour) {
     if(hour < 0 || hour > 24) return 0;
     return day.points[hour] * 10;
 }

src/AmsDataStorage.h

@@ -28,7 +28,7 @@ public:
     AmsDataStorage(RemoteDebug*);
     void setTimezone(Timezone*);
     bool update(AmsData*);
-    int16_t getHour(uint8_t);
+    int32_t getHour(uint8_t);
     int32_t getDay(uint8_t);
     bool load();
     bool save();

src/AmsToMqttBridge.ino

@@ -18,7 +18,7 @@ ADC_MODE(ADC_VCC);
 #if defined(ESP32)
 #include <esp_task_wdt.h>
 #endif
-#define WDT_TIMEOUT 30
+#define WDT_TIMEOUT 60
 
 #include "AmsToMqttBridge.h"
 #include "AmsStorage.h"
@@ -333,7 +333,11 @@ unsigned long lastErrorBlink = 0;
 int lastError = 0;
 
 void loop() {
-	Debug.handle();
+	try {
+		Debug.handle();
+	} catch(const std::exception& e) {
+		Serial.printf("Exception in Debug loop (%s)\n", e.what());
+	}
 	unsigned long now = millis();
 	if(gpioConfig.apPin != 0xFF) {
 		if (digitalRead(gpioConfig.apPin) == LOW) {
@@ -363,7 +367,11 @@ void loop() {
 		if (WiFi.status() != WL_CONNECTED) {
 			wifiConnected = false;
 			Debug.stop();
-			WiFi_connect();
+			try {
+				WiFi_connect();
+			} catch(const std::exception& e) {
+				debugE("Exception in WiFi connect (%s)", e.what());
+			}
 		} else {
 			wifiReconnectCount = 0;
 			if(!wifiConnected) {
@@ -377,10 +385,14 @@ void loop() {
 					}
 					DebugConfig debug;
 					if(config.getDebugConfig(debug)) {
-						Debug.begin(wifi.hostname, (uint8_t) debug.level);
-						Debug.setSerialEnabled(debug.serial);
-						if(!debug.telnet) {
-							Debug.stop();
+						try {
+							Debug.begin(wifi.hostname, (uint8_t) debug.level);
+							Debug.setSerialEnabled(debug.serial);
+							if(!debug.telnet) {
+								Debug.stop();
+							}
+						} catch(const std::exception& e) {
+							Serial.printf("Exception in Debug setup (%s)\n", e.what());
 						}
 					}
 					if(Debug.isActive(RemoteDebug::INFO)) {
@@ -439,33 +451,45 @@ void loop() {
 			}
 
 			#if defined(ESP32)
-			if(eapi != NULL && ntpEnabled) {
-				if(eapi->loop() && mqtt != NULL && mqttHandler != NULL && mqtt->connected()) {
-					mqttHandler->publishPrices(eapi);
-				}
-			}
-			
-			if(config.isEntsoeChanged()) {
-				EntsoeConfig entsoe;
-				if(config.getEntsoeConfig(entsoe) && strlen(entsoe.token) > 0) {
-					if(eapi == NULL) {
-						eapi = new EntsoeApi(&Debug);
-						ws.setEntsoeApi(eapi);
+			try {
+				if(eapi != NULL && ntpEnabled) {
+					if(eapi->loop() && mqtt != NULL && mqttHandler != NULL && mqtt->connected()) {
+						mqttHandler->publishPrices(eapi);
 					}
-					eapi->setup(entsoe);
-				} else if(eapi != NULL) {
-					delete eapi;
-					eapi = NULL;
-					ws.setEntsoeApi(NULL);
 				}
-				config.ackEntsoeChange();
+				
+				if(config.isEntsoeChanged()) {
+					EntsoeConfig entsoe;
+					if(config.getEntsoeConfig(entsoe) && strlen(entsoe.token) > 0) {
+						if(eapi == NULL) {
+							eapi = new EntsoeApi(&Debug);
+							ws.setEntsoeApi(eapi);
+						}
+						eapi->setup(entsoe);
+					} else if(eapi != NULL) {
+						delete eapi;
+						eapi = NULL;
+						ws.setEntsoeApi(NULL);
+					}
+					config.ackEntsoeChange();
+				}
+			} catch(const std::exception& e) {
+				debugE("Exception in ENTSO-E loop (%s)", e.what());
 			}
 			#endif
-			ws.loop();
+			try {
+				ws.loop();
+			} catch(const std::exception& e) {
+				debugE("Exception in Web server loop (%s)", e.what());
+			}
 		}
-		if(mqtt != NULL) { // Run loop regardless, to let MQTT do its work.
-			mqtt->loop();
-			delay(10); // Needed to preserve power. After adding this, the voltage is super smooth on a HAN powered device
+		if(mqtt != NULL) {
+			try {
+				mqtt->loop();
+				delay(10); // Needed to preserve power. After adding this, the voltage is super smooth on a HAN powered device
+			} catch(const std::exception& e) {
+				debugE("Exception in MQTT loop (%s)", e.what());
+			}
 		}
 	} else {
 		if(dnsServer != NULL) {
@@ -488,17 +512,29 @@ void loop() {
 		hc = NULL;
 	}
 
-	if(readHanPort() || now - meterState.getLastUpdateMillis() > 30000) {
-		if(now - lastTemperatureRead > 15000) {
-			unsigned long start = millis();
-			hw.updateTemperatures();
-			lastTemperatureRead = now;
+	try {
+		if(readHanPort() || now - meterState.getLastUpdateMillis() > 30000) {
+			if(now - lastTemperatureRead > 15000) {
+				try {
+					unsigned long start = millis();
+					hw.updateTemperatures();
+					lastTemperatureRead = now;
 
-			if(mqtt != NULL && mqttHandler != NULL && WiFi.getMode() != WIFI_AP && WiFi.status() == WL_CONNECTED && mqtt->connected() && !topic.isEmpty()) {
-				mqttHandler->publishTemperatures(&config, &hw);
+					try {
+						if(mqtt != NULL && mqttHandler != NULL && WiFi.getMode() != WIFI_AP && WiFi.status() == WL_CONNECTED && mqtt->connected() && !topic.isEmpty()) {
+							mqttHandler->publishTemperatures(&config, &hw);
+						}
+					} catch(const std::exception& e) {
+						debugE("Exception while publishing temperatures to MQTT (%s)", e.what());
+					}
+					debugD("Used %d ms to update temperature", millis()-start);
+				} catch(const std::exception& e) {
+					debugE("Exception while updating temperatures (%s)", e.what());
+				}
 			}
-			debugD("Used %d ms to update temperature", millis()-start);
 		}
+	} catch(const std::exception& e) {
+		debugE("Exception in readHanPort (%s)", e.what());
 	}
 	delay(1); // Needed for auto modem sleep
 	#if defined(ESP32)
@@ -706,7 +742,11 @@ bool readHanPort() {
 		int pos = HDLC_FRAME_INCOMPLETE;
 		while(hanSerial->available() && pos == HDLC_FRAME_INCOMPLETE) {
 			buf[len++] = hanSerial->read();
-			pos = HDLC_validate((uint8_t *) buf, len, hc, &timestamp);
+			try {
+				pos = HDLC_validate((uint8_t *) buf, len, hc, &timestamp);
+			} catch(const std::exception& e) {
+				debugE("Exception while parsing validating HDLC (%s)", e.what());
+			}
 		}
 		if(len > 0) {
 			if(len >= BUF_SIZE) {
@@ -770,7 +810,11 @@ bool readHanPort() {
 			while(hanSerial->available()) hanSerial->read();
 			if(pos > 0) {
 				debugD("Valid data, start at byte %d", pos);
-				data = IEC6205675(((char *) (buf)) + pos, meterState.getMeterType(), meterConfig.distributionSystem, timestamp, hc);
+				try {
+					data = IEC6205675(((char *) (buf)) + pos, meterState.getMeterType(), meterConfig.distributionSystem, timestamp, hc);
+				} catch(const std::exception& e) {
+					debugE("Exception while parsing IEC62056-7-5 (%s)", e.what());
+				}
 			} else {
 				if(Debug.isActive(RemoteDebug::WARNING)) {
 					switch(pos) {
@@ -823,7 +867,11 @@ bool readHanPort() {
 		}
 	} else if(currentMeterType == 2) {
 		String payload = hanSerial->readString();
-		data = IEC6205621(payload);
+		try {
+			data = IEC6205621(payload);
+		} catch(const std::exception& e) {
+			debugE("Exception while parsing IEC62056-21 (%s)", e.what());
+		}
 		if(data.getListType() == 0) {
 			currentMeterType = 1;
 			return false;
@@ -839,17 +887,29 @@ bool readHanPort() {
 		if(!hw.ledBlink(LED_GREEN, 1))
 			hw.ledBlink(LED_INTERNAL, 1);
 		if(mqttEnabled && mqttHandler != NULL && mqtt != NULL) {
-			if(mqttHandler->publish(&data, &meterState)) {
-				if(data.getListType() == 3 && eapi != NULL) {
-					mqttHandler->publishPrices(eapi);
+			try {
+				if(mqttHandler->publish(&data, &meterState)) {
+					if(data.getListType() == 3 && eapi != NULL) {
+						try {
+							mqttHandler->publishPrices(eapi);
+						} catch(const std::exception& e) {
+							debugE("Exception while publishing prices to MQTT (%s)", e.what());
+						}
+					}
+					if(data.getListType() >= 2) {
+						try {
+							mqttHandler->publishSystem(&hw);
+						} catch(const std::exception& e) {
+							debugE("Exception while publishing system info to MQTT (%s)", e.what());
+						}
+					}
 				}
-				if(data.getListType() >= 2) {
-					mqttHandler->publishSystem(&hw);
+				if(mqtt != NULL) {
+					mqtt->loop();
+					delay(10);
 				}
-			}
-			if(mqtt != NULL) {
-				mqtt->loop();
-				delay(10);
+			} catch(const std::exception& e) {
+				debugE("Exception while publishing AMS data to MQTT (%s)", e.what());
 			}
 		}
 
@@ -1024,7 +1084,7 @@ void MQTT_connect() {
 		mqtt->disconnect();
 		yield();
 	} else {
-		mqtt = new MQTTClient(512);
+		mqtt = new MQTTClient(1024);
 		ws.setMqtt(mqtt);
 	}
 
@@ -1123,7 +1183,11 @@ void MQTT_connect() {
 		config.ackMqttChange();
 		
 		if(mqttHandler != NULL) {
-			mqttHandler->publishSystem(&hw);
+			try {
+				mqttHandler->publishSystem(&hw);
+			} catch(const std::exception& e) {
+				debugE("Exception while publishing system info to MQTT (%s)", e.what());
+			}
 		}
 	} else {
 		if (Debug.isActive(RemoteDebug::ERROR)) {

src/IEC6205675.cpp

@@ -308,6 +308,14 @@ IEC6205675::IEC6205675(const char* d, uint8_t useMeterType, uint8_t distribution
                 l2PowerFactor /= 100;
             if(l3PowerFactor != 0)
                 l3PowerFactor /= 100;
+
+            int watt = abs((l1voltage * l1current) + (l2voltage * l2current) + (l3voltage * l3current));
+            if(watt / (activeImportPower + activeExportPower  + reactiveImportPower + reactiveExportPower) > 2) {
+                l1current = l1current != 0 ? l1current / 10 : 0;
+                l2current = l2current != 0 ? l2current / 10 : 0;
+                l3current = l3current != 0 ? l3current / 10 : 0;
+            }
+
         } else if(meterType == AmsTypeSagemcom) {
             CosemData* meterTs = getCosemDataAt(1, ((char *) (d)));
             if(meterTs != NULL) {
@@ -344,7 +352,7 @@ IEC6205675::IEC6205675(const char* d, uint8_t useMeterType, uint8_t distribution
             if(l2current == 0.0 && l1current > 0.0 && l3current > 0.0) {
                 l2current = (((activeImportPower - activeExportPower) * sqrt(3)) - (l1voltage * l1current) - (l3voltage * l3current)) / l2voltage;
                 if(activeExportPower == 0.0) {
-                    l2current = max((float) 0.0, l2current);
+                    l2current = max((double) 0.0, l2current);
                 }
             }
         } else if(twoPhase && l1current > 0.0 && l2current > 0.0 && l3current > 0.0) {
@@ -380,7 +388,8 @@ CosemData* IEC6205675::getCosemDataAt(uint8_t index, const char* ptr) {
                 pos += 5;
                 break;
             case CosemTypeNull:
-                return NULL;
+                pos += 1;
+                break;
             default:
                 pos += 2;
         }
@@ -423,7 +432,8 @@ CosemData* IEC6205675::findObis(uint8_t* obis, int matchlength, const char* ptr)
                 pos += 5;
                 break;
             case CosemTypeNull:
-                return NULL;
+                pos += 1;
+                break;
             default:
                 pos += 2;
         }

src/ams/hdlc.cpp

@@ -268,7 +268,7 @@ int HDLC_validate(const uint8_t* d, int length, HDLCConfig* config, CosemDateTim
         } else if(dateTime->base.type == CosemTypeDateTime) {
             memcpy(timestamp, ptr, dateTime->base.length);
         } else if(dateTime->base.type == 0x0C) { // Kamstrup bug...
-            memcpy(timestamp, ptr, 0x0C);
+            memcpy(timestamp, ptr, 13);
             ptr += 13;
         } else {
             return HDLC_TIMESTAMP_UNKNOWN;

src/entsoe/EntsoeApi.cpp

@@ -13,7 +13,7 @@ EntsoeApi::EntsoeApi(RemoteDebug* Debug) {
 
     client.setInsecure();
     https.setFollowRedirects(HTTPC_STRICT_FOLLOW_REDIRECTS);
-    https.setTimeout(20000);
+    https.setTimeout(50000);
 
     // Entso-E uses CET/CEST
     TimeChangeRule CEST = {"CEST", Last, Sun, Mar, 2, 120};

src/mqtt/JsonMqttHandler.cpp

@@ -13,7 +13,7 @@ bool JsonMqttHandler::publish(AmsData* data, AmsData* previousState) {
 		return false;
 
     if(data->getListType() == 1) {
-        char json[192];
+        char json[256];
         snprintf_P(json, sizeof(json), JSON1_JSON,
             WiFi.macAddress().c_str(),
             clientId.c_str(),
@@ -26,7 +26,7 @@ bool JsonMqttHandler::publish(AmsData* data, AmsData* previousState) {
         );
         return mqtt->publish(topic, json);
     } else if(data->getListType() == 2) {
-        char json[384];
+        char json[512];
         snprintf_P(json, sizeof(json), JSON2_JSON,
             WiFi.macAddress().c_str(),
             clientId.c_str(),
@@ -52,7 +52,7 @@ bool JsonMqttHandler::publish(AmsData* data, AmsData* previousState) {
         return mqtt->publish(topic, json);
     } else if(data->getListType() == 3) {
         if(data->getPowerFactor() == 0) {
-            char json[512];
+            char json[768];
             snprintf_P(json, sizeof(json), JSON3_JSON,
                 WiFi.macAddress().c_str(),
                 clientId.c_str(),

web/application.js

@@ -726,19 +726,19 @@ var fetch = function() {
             var a = 0;
             var r = 1;
             var arr = [['Phase', 'Amperage', { role: 'style' }, { role: 'annotation' }]];
-            if(json.i1) {
+            if(json.i1 || json.u1) {
                 var i1 = parseFloat(json.i1);
                 a = Math.max(a, i1);
                 var pct = (parseFloat(json.i1)/parseInt(json.mf))*100;
                 arr[r++] = ['L1', pct, "color: " + ampcol(pct) + ";opacity: 0.9;", i1 + "A"];
             }
-            if(json.i2) {
+            if(json.i2 || json.u2) {
                 var i2 = parseFloat(json.i2);
                 a = Math.max(a, i2);
                 var pct = (parseFloat(json.i2)/parseInt(json.mf))*100;
                 arr[r++] = ['L2', pct, "color: " + ampcol(pct) + ";opacity: 0.9;", i2 + "A"];
             }
-            if(json.i3) {
+            if(json.i3 || json.u3) {
                 var i3 = parseFloat(json.i3);
                 a = Math.max(a, i3);
                 var pct = (parseFloat(json.i3)/parseInt(json.mf))*100;

web/index.html

@@ -44,8 +44,8 @@
             <div class="text-center overlay-plot">
                 <div id="xp" class="plot1"></div>
                 <span class="plot-overlay">
-                    <span class="xpo">{PO}</span> 
-                    <span class="xpoa">W</span>
+                    <span class="epo">{PO}</span> 
+                    <span class="epoa">W</span>
                     <br/>
                     <span class="pol">Export</span>
                 </span>