Merge branch 'main' into fix/tariff_peaks_wrong_day

lib/AmsData/include/AmsData.h

@@ -7,8 +7,7 @@
 #ifndef _AMSDATA_H
 #define _AMSDATA_H
 
-#include "Arduino.h"
+#include <WString.h>
-#include <Timezone.h>
 #include "OBIScodes.h"
 
 enum AmsType {
@@ -28,7 +27,7 @@ public:
     AmsData();
 
     void apply(AmsData& other);
-    void apply(const OBIS_code_t obis, double value);
+    void apply(const OBIS_code_t obis, double value, uint64_t millis64);
 
     uint64_t getLastUpdateMillis();
 

lib/AmsData/src/AmsData.cpp

@@ -5,6 +5,7 @@
  */
 
 #include "AmsData.h"
+#include <algorithm>
 
 AmsData::AmsData() {}
 
@@ -17,7 +18,6 @@ void AmsData::apply(AmsData& other) {
                 uint32_t power = (activeImportPower + other.getActiveImportPower()) / 2;
                 float add = power * (((float) ms) / 3600000.0);
                 activeImportCounter += add / 1000.0;
-                //Serial.printf("%dW, %dms, %.6fkWh added\n", other.getActiveImportPower(), ms, add);
             }
 
             if(other.getListType() > 1) {
@@ -112,7 +112,7 @@ void AmsData::apply(AmsData& other) {
         this->activeExportPower = other.getActiveExportPower();
 }
 
-void AmsData::apply(OBIS_code_t obis, double value) {
+void AmsData::apply(OBIS_code_t obis, double value, uint64_t millis64) {
     if(obis.sensor == 0 && obis.gr == 0 && obis.tariff == 0) {
         meterType = value;
     }
@@ -127,138 +127,137 @@ void AmsData::apply(OBIS_code_t obis, double value) {
         }
     }
     if(obis.tariff != 0) {
-        Serial.println("Tariff not implemented");
         return;
     }
     if(obis.gr == 7) { // Instant values
         switch(obis.sensor) {
             case 1:
                 activeImportPower = value;
-                listType = max(listType, (uint8_t) 2);
+                listType = std::max(listType, (uint8_t) 2);
                 break;
             case 2:
                 activeExportPower = value;
-                listType = max(listType, (uint8_t) 2);
+                listType = std::max(listType, (uint8_t) 2);
                 break;
             case 3:
                 reactiveImportPower = value;
-                listType = max(listType, (uint8_t) 2);
+                listType = std::max(listType, (uint8_t) 2);
                 break;
             case 4:
                 reactiveExportPower = value;
-                listType = max(listType, (uint8_t) 2);
+                listType = std::max(listType, (uint8_t) 2);
                 break;
             case 13:
                 powerFactor = value;
-                listType = max(listType, (uint8_t) 4);
+                listType = std::max(listType, (uint8_t) 4);
                 break;
             case 21:
                 l1activeImportPower = value;
-                listType = max(listType, (uint8_t) 4);
+                listType = std::max(listType, (uint8_t) 4);
                 break;
             case 22:
                 l1activeExportPower = value;
-                listType = max(listType, (uint8_t) 4);
+                listType = std::max(listType, (uint8_t) 4);
                 break;
             case 31:
                 l1current = value;
-                listType = max(listType, (uint8_t) 2);
+                listType = std::max(listType, (uint8_t) 2);
                 break;
             case 32:
                 l1voltage = value;
-                listType = max(listType, (uint8_t) 2);
+                listType = std::max(listType, (uint8_t) 2);
                 break;
             case 33:
                 l1PowerFactor = value;
-                listType = max(listType, (uint8_t) 4);
+                listType = std::max(listType, (uint8_t) 4);
                 break;
             case 41:
                 l2activeImportPower = value;
-                listType = max(listType, (uint8_t) 4);
+                listType = std::max(listType, (uint8_t) 4);
                 break;
             case 42:
                 l2activeExportPower = value;
-                listType = max(listType, (uint8_t) 4);
+                listType = std::max(listType, (uint8_t) 4);
                 break;
             case 51:
                 l2current = value;
-                listType = max(listType, (uint8_t) 2);
+                listType = std::max(listType, (uint8_t) 2);
                 break;
             case 52:
                 l2voltage = value;
-                listType = max(listType, (uint8_t) 2);
+                listType = std::max(listType, (uint8_t) 2);
                 break;
             case 53:
                 l2PowerFactor = value;
-                listType = max(listType, (uint8_t) 4);
+                listType = std::max(listType, (uint8_t) 4);
                 break;
             case 61:
                 l3activeImportPower = value;
-                listType = max(listType, (uint8_t) 4);
+                listType = std::max(listType, (uint8_t) 4);
                 break;
             case 62:
                 l3activeExportPower = value;
-                listType = max(listType, (uint8_t) 4);
+                listType = std::max(listType, (uint8_t) 4);
                 break;
             case 71:
                 l3current = value;
-                listType = max(listType, (uint8_t) 2);
+                listType = std::max(listType, (uint8_t) 2);
                 break;
             case 72:
                 l3voltage = value;
-                listType = max(listType, (uint8_t) 2);
+                listType = std::max(listType, (uint8_t) 2);
                 break;
             case 73:
                 l3PowerFactor = value;
-                listType = max(listType, (uint8_t) 4);
+                listType = std::max(listType, (uint8_t) 4);
                 break;
         }
     } else if(obis.gr == 8) { // Accumulated values
         switch(obis.sensor) {
             case 1:
                 activeImportCounter = value;
-                listType = max(listType, (uint8_t) 3);
+                listType = std::max(listType, (uint8_t) 3);
                 break;
             case 2:
                 activeExportCounter = value;
-                listType = max(listType, (uint8_t) 3);
+                listType = std::max(listType, (uint8_t) 3);
                 break;
             case 3:
                 reactiveImportCounter = value;
-                listType = max(listType, (uint8_t) 3);
+                listType = std::max(listType, (uint8_t) 3);
                 break;
             case 4:
                 reactiveExportCounter = value;
-                listType = max(listType, (uint8_t) 3);
+                listType = std::max(listType, (uint8_t) 3);
                 break;
             case 21:
                 l1activeImportCounter = value;
-                listType = max(listType, (uint8_t) 4);
+                listType = std::max(listType, (uint8_t) 4);
                 break;
             case 22:
                 l1activeExportCounter = value;
-                listType = max(listType, (uint8_t) 4);
+                listType = std::max(listType, (uint8_t) 4);
                 break;
             case 41:
                 l2activeImportCounter = value;
-                listType = max(listType, (uint8_t) 4);
+                listType = std::max(listType, (uint8_t) 4);
                 break;
             case 42:
                 l2activeExportCounter = value;
-                listType = max(listType, (uint8_t) 4);
+                listType = std::max(listType, (uint8_t) 4);
                 break;
             case 61:
                 l3activeImportCounter = value;
-                listType = max(listType, (uint8_t) 4);
+                listType = std::max(listType, (uint8_t) 4);
                 break;
             case 62:
                 l3activeExportCounter = value;
-                listType = max(listType, (uint8_t) 4);
+                listType = std::max(listType, (uint8_t) 4);
                 break;
         }
     }
     if(listType > 0)
-        lastUpdateMillis = millis();
+        lastUpdateMillis = millis64;
 
     threePhase = l1voltage > 0 && l2voltage > 0 && l3voltage > 0;
     if(!threePhase)

lib/AmsDataStorage/src/AmsDataStorage.cpp

@@ -639,25 +639,22 @@ bool AmsDataStorage::isDayHappy(time_t now) {
         return false;
     }
 
-    if(now < FirmwareVersion::BuildEpoch) return false;
+    // If the timestamp is before the firware was built, there is something seriously wrong..
-
+    if(now < FirmwareVersion::BuildEpoch) {
-    if(now < day.lastMeterReadTime) {
         return false;
     }
-    // There are cases where the meter reports before the hour. The update method will then receive the meter timestamp as reference, thus there will not be 3600s between. 
+
-    // Leaving a 100s buffer for these cases
+    // If the timestamp is before the last time we updated, there is also something wrong..
-    if(now-day.lastMeterReadTime > 3500) {
+    if(now < day.lastMeterReadTime) {
         return false;
     }
 
     tmElements_t tm, last;
     breakTime(tz->toLocal(now), tm);
     breakTime(tz->toLocal(day.lastMeterReadTime), last);
-    if(tm.Hour != last.Hour) {
-        return false;
-    }
 
-    return true;
+    // If the timestamp is at the same day and hour as last update, we are happy
+    return tm.Day == last.Day && tm.Hour == last.Hour;
 }
 
 bool AmsDataStorage::isMonthHappy(time_t now) {

lib/EnergyAccounting/include/EnergyAccounting.h

@@ -7,8 +7,6 @@
 #ifndef _ENERGYACCOUNTING_H
 #define _ENERGYACCOUNTING_H
 
-#include "Arduino.h"
-#include "AmsData.h"
 #include "AmsDataStorage.h"
 #include "PriceService.h"
 
@@ -83,7 +81,7 @@ public:
     void setPriceService(PriceService *ps);
     void setTimezone(Timezone*);
     EnergyAccountingConfig* getConfig();
-    bool update(AmsData* amsData);
+    bool update(time_t now, uint64_t lastUpdatedMillis, uint8_t listType, uint32_t activeImportPower, uint32_t activeExportPower);
     bool load();
     bool save();
     bool isInitialized();

lib/EnergyAccounting/src/EnergyAccounting.cpp

@@ -54,9 +54,8 @@ bool EnergyAccounting::isInitialized() {
     return this->init;
 }
 
-bool EnergyAccounting::update(AmsData* amsData) {
+bool EnergyAccounting::update(time_t now, uint64_t lastUpdatedMillis, uint8_t listType, uint32_t activeImportPower, uint32_t activeExportPower) {
     if(config == NULL) return false;
-    time_t now = time(nullptr);
     if(now < FirmwareVersion::BuildEpoch) return false;
     if(tz == NULL) {
         return false;
@@ -90,7 +89,7 @@ bool EnergyAccounting::update(AmsData* amsData) {
         calcDayCost();
     }
 
-    if(local.Hour != realtimeData->currentHour && (amsData->getListType() >= 3 || local.Minute == 1)) {
+    if(local.Hour != realtimeData->currentHour && (listType >= 3 || local.Minute == 1)) {
         tmElements_t oneHrAgo, oneHrAgoLocal;
         breakTime(now-3600, oneHrAgo);
         uint16_t val = round(ds->getHourImport(oneHrAgo.Hour) / 10.0);
@@ -156,9 +155,9 @@ bool EnergyAccounting::update(AmsData* amsData) {
         }
     }
 
-    if(realtimeData->lastImportUpdateMillis < amsData->getLastUpdateMillis()) {
+    if(realtimeData->lastImportUpdateMillis < lastUpdatedMillis) {
-        unsigned long ms = amsData->getLastUpdateMillis() - realtimeData->lastImportUpdateMillis;
+        unsigned long ms = lastUpdatedMillis - realtimeData->lastImportUpdateMillis;
-        float kwhi = (amsData->getActiveImportPower() * (((float) ms) / 3600000.0)) / 1000.0;
+        float kwhi = (activeImportPower * (((float) ms) / 3600000.0)) / 1000.0;
         if(kwhi > 0) {
             realtimeData->use += kwhi;
             float importPrice = ps == NULL ? PRICE_NO_VALUE : ps->getCurrentPrice(PRICE_DIRECTION_IMPORT);
@@ -168,12 +167,12 @@ bool EnergyAccounting::update(AmsData* amsData) {
                 realtimeData->costDay += cost;
             }
         }
-        realtimeData->lastImportUpdateMillis = amsData->getLastUpdateMillis();
+        realtimeData->lastImportUpdateMillis = lastUpdatedMillis;
     }
 
-    if(amsData->getListType() > 1 && realtimeData->lastExportUpdateMillis < amsData->getLastUpdateMillis()) {
+    if(listType > 1 && realtimeData->lastExportUpdateMillis < lastUpdatedMillis) {
-        unsigned long ms = amsData->getLastUpdateMillis() - realtimeData->lastExportUpdateMillis;
+        unsigned long ms = lastUpdatedMillis - realtimeData->lastExportUpdateMillis;
-        float kwhe = (amsData->getActiveExportPower() * (((float) ms) / 3600000.0)) / 1000.0;
+        float kwhe = (activeExportPower * (((float) ms) / 3600000.0)) / 1000.0;
         if(kwhe > 0) {
             realtimeData->produce += kwhe;
             float exportPrice = ps == NULL ? PRICE_NO_VALUE : ps->getCurrentPrice(PRICE_DIRECTION_EXPORT);
@@ -183,7 +182,7 @@ bool EnergyAccounting::update(AmsData* amsData) {
                 realtimeData->incomeDay += income;
             }
         }
-        realtimeData->lastExportUpdateMillis = amsData->getLastUpdateMillis();
+        realtimeData->lastExportUpdateMillis = lastUpdatedMillis;
     }
 
     if(config != NULL) {

lib/MeterCommunicators/include/IEC6205675.h

@@ -11,6 +11,7 @@
 #include "AmsConfiguration.h"
 #include "DataParser.h"
 #include "Cosem.h"
+#include "Timezone.h"
 #if defined(AMS_REMOTE_DEBUG)
 #include "RemoteDebug.h"
 #endif

lib/MeterCommunicators/src/KmpCommunicator.cpp

@@ -24,8 +24,6 @@ void KmpCommunicator::configure(MeterConfig& meterConfig) {
 }
 
 bool KmpCommunicator::loop() {
-	uint64_t now = millis64();
-
     bool ret = talker->loop();
     int lastError = getLastError();
     if(ret) {
@@ -58,35 +56,36 @@ AmsData* KmpCommunicator::getData(AmsData& meterState) {
     if(talker == NULL) return NULL;
     KmpDataHolder kmpData;
     talker->getData(kmpData);
+	uint64_t now = millis64();
     AmsData* data = new AmsData();
-    data->apply(OBIS_ACTIVE_IMPORT_COUNT, kmpData.activeImportCounter);
+    data->apply(OBIS_ACTIVE_IMPORT_COUNT, kmpData.activeImportCounter, now);
-    data->apply(OBIS_ACTIVE_EXPORT_COUNT, kmpData.activeExportCounter);
+    data->apply(OBIS_ACTIVE_EXPORT_COUNT, kmpData.activeExportCounter, now);
-    data->apply(OBIS_REACTIVE_IMPORT_COUNT, kmpData.reactiveImportCounter);
+    data->apply(OBIS_REACTIVE_IMPORT_COUNT, kmpData.reactiveImportCounter, now);
-    data->apply(OBIS_REACTIVE_EXPORT_COUNT, kmpData.reactiveExportCounter);
+    data->apply(OBIS_REACTIVE_EXPORT_COUNT, kmpData.reactiveExportCounter, now);
-    data->apply(OBIS_ACTIVE_IMPORT, kmpData.activeImportPower);
+    data->apply(OBIS_ACTIVE_IMPORT, kmpData.activeImportPower, now);
-    data->apply(OBIS_ACTIVE_EXPORT, kmpData.activeExportPower);
+    data->apply(OBIS_ACTIVE_EXPORT, kmpData.activeExportPower, now);
-    data->apply(OBIS_REACTIVE_IMPORT, kmpData.reactiveImportPower);
+    data->apply(OBIS_REACTIVE_IMPORT, kmpData.reactiveImportPower, now);
-    data->apply(OBIS_REACTIVE_EXPORT, kmpData.reactiveExportPower);
+    data->apply(OBIS_REACTIVE_EXPORT, kmpData.reactiveExportPower, now);
-    data->apply(OBIS_VOLTAGE_L1, kmpData.l1voltage);
+    data->apply(OBIS_VOLTAGE_L1, kmpData.l1voltage, now);
-    data->apply(OBIS_VOLTAGE_L2, kmpData.l2voltage);
+    data->apply(OBIS_VOLTAGE_L2, kmpData.l2voltage, now);
-    data->apply(OBIS_VOLTAGE_L3, kmpData.l3voltage);
+    data->apply(OBIS_VOLTAGE_L3, kmpData.l3voltage, now);
-    data->apply(OBIS_CURRENT_L1, kmpData.l1current);
+    data->apply(OBIS_CURRENT_L1, kmpData.l1current, now);
-    data->apply(OBIS_CURRENT_L2, kmpData.l2current);
+    data->apply(OBIS_CURRENT_L2, kmpData.l2current, now);
-    data->apply(OBIS_CURRENT_L3, kmpData.l3current);
+    data->apply(OBIS_CURRENT_L3, kmpData.l3current, now);
-    data->apply(OBIS_POWER_FACTOR_L1, kmpData.l1PowerFactor);
+    data->apply(OBIS_POWER_FACTOR_L1, kmpData.l1PowerFactor, now);
-    data->apply(OBIS_POWER_FACTOR_L2, kmpData.l2PowerFactor);
+    data->apply(OBIS_POWER_FACTOR_L2, kmpData.l2PowerFactor, now);
-    data->apply(OBIS_POWER_FACTOR_L3, kmpData.l3PowerFactor);
+    data->apply(OBIS_POWER_FACTOR_L3, kmpData.l3PowerFactor, now);
-    data->apply(OBIS_POWER_FACTOR, kmpData.powerFactor);
+    data->apply(OBIS_POWER_FACTOR, kmpData.powerFactor, now);
-    data->apply(OBIS_ACTIVE_IMPORT_L1, kmpData.l1activeImportPower);
+    data->apply(OBIS_ACTIVE_IMPORT_L1, kmpData.l1activeImportPower, now);
-    data->apply(OBIS_ACTIVE_IMPORT_L2, kmpData.l2activeImportPower);
+    data->apply(OBIS_ACTIVE_IMPORT_L2, kmpData.l2activeImportPower, now);
-    data->apply(OBIS_ACTIVE_IMPORT_L3, kmpData.l3activeImportPower);
+    data->apply(OBIS_ACTIVE_IMPORT_L3, kmpData.l3activeImportPower, now);
-    data->apply(OBIS_ACTIVE_EXPORT_L1, kmpData.l1activeExportPower);
+    data->apply(OBIS_ACTIVE_EXPORT_L1, kmpData.l1activeExportPower, now);
-    data->apply(OBIS_ACTIVE_EXPORT_L2, kmpData.l2activeExportPower);
+    data->apply(OBIS_ACTIVE_EXPORT_L2, kmpData.l2activeExportPower, now);
-    data->apply(OBIS_ACTIVE_EXPORT_L3, kmpData.l3activeExportPower);
+    data->apply(OBIS_ACTIVE_EXPORT_L3, kmpData.l3activeExportPower, now);
-    data->apply(OBIS_ACTIVE_IMPORT_COUNT_L1, kmpData.l1activeImportCounter);
+    data->apply(OBIS_ACTIVE_IMPORT_COUNT_L1, kmpData.l1activeImportCounter, now);
-    data->apply(OBIS_ACTIVE_IMPORT_COUNT_L2, kmpData.l2activeImportCounter);
+    data->apply(OBIS_ACTIVE_IMPORT_COUNT_L2, kmpData.l2activeImportCounter, now);
-    data->apply(OBIS_ACTIVE_IMPORT_COUNT_L3, kmpData.l3activeImportCounter);
+    data->apply(OBIS_ACTIVE_IMPORT_COUNT_L3, kmpData.l3activeImportCounter, now);
-    data->apply(OBIS_METER_ID, kmpData.meterId);
+    data->apply(OBIS_METER_ID, kmpData.meterId, now);
-    data->apply(OBIS_NULL, AmsTypeKamstrup);
+    data->apply(OBIS_NULL, AmsTypeKamstrup, now);
     return data;
 }

lib/RealtimePlot/include/RealtimePlot.h

@@ -7,7 +7,6 @@
 #ifndef _REALTIMEPLOT_H
 #define _REALTIMEPLOT_H
 
-#include <stdint.h>
 #include "AmsData.h"
 
 #define REALTIME_SAMPLE 10000

lib/RealtimePlot/src/RealtimePlot.cpp

@@ -4,6 +4,7 @@
  * 
  */
 
+#include "Arduino.h"
 #include "RealtimePlot.h"
 #include <stdlib.h>
 

src/AmsToMqttBridge.cpp

@@ -1577,6 +1577,7 @@ void handleDataSuccess(AmsData* data) {
 
 	time_t dataUpdateTime = now;
 	if(abs(now - meterTime) < 300) {
+		// If the meter timestamp is close to our internal clock, use meter timestamp, because that is best for data tracking
 		dataUpdateTime = meterTime;
 	}
 
@@ -1589,14 +1590,14 @@ void handleDataSuccess(AmsData* data) {
 		debugD_P(PSTR("READY to update (internal clock %02d:%02d:%02d UTC, meter clock: %02d:%02d:%02d, list type %d, est: %d, using clock: %d)"), tm.Hour, tm.Minute, tm.Second, mtm.Hour, mtm.Minute, mtm.Second, data->getListType(), wasCounterEstimated, dataUpdateTime == now);
 		tmElements_t dtm;
 		breakTime(dataUpdateTime, dtm);
-		if(dtm.Minute < 2 && data->getListType() >= 3) {
+		if(dtm.Minute < 1 && data->getListType() >= 3) {
 			debugD_P(PSTR("Updating data storage using actual data"));
 			saveData = ds.update(data, dataUpdateTime);
 
 			#if defined(_CLOUDCONNECTOR_H)
 			if(saveData && cloud != NULL) cloud->forceUpdate();
 			#endif
-		} else if(dtm.Minute == 2) {
+		} else if(dtm.Minute == 1) {
 			debugW_P(PSTR("Did not receive necessary data for previous hour, clearing"));
 			AmsData nullData;
 			saveData = ds.update(&nullData, dataUpdateTime);
@@ -1611,7 +1612,7 @@ void handleDataSuccess(AmsData* data) {
 		debugD_P(PSTR("NOT Ready to update (internal clock %02d:%02d:%02d UTC, meter clock: %02d:%02d:%02d, list type %d, est: %d)"), tm.Hour, tm.Minute, tm.Second, mtm.Hour, mtm.Minute, mtm.Second, data->getListType(), wasCounterEstimated);
 	}
 
-	if(ea.update(data)) {
+	if(ea.update(dataUpdateTime, data->getLastUpdateMillis(), data->getListType(), data->getActiveImportPower(), data->getActiveExportPower())) {
 		debugI_P(PSTR("Saving energy accounting"));
 		if(!ea.save()) {
 			debugW_P(PSTR("Unable to save energy accounting"));