Buffer improvements and fixed special characters in ssid

lib/AmsConfiguration/include/AmsConfiguration.h

@@ -252,6 +252,7 @@ public:
 	bool getMeterConfig(MeterConfig&);
 	bool setMeterConfig(MeterConfig&);
 	void clearMeter(MeterConfig&);
+	void setMeterChanged();
 	bool isMeterChanged();
 	void ackMeterChanged();
 

lib/AmsConfiguration/src/AmsConfiguration.cpp

@@ -266,6 +266,10 @@ void AmsConfiguration::ackMeterChanged() {
 	meterChanged = false;
 }
 
+void AmsConfiguration::setMeterChanged() {
+	meterChanged = true;
+}
+
 bool AmsConfiguration::getDebugConfig(DebugConfig& config) {
 	if(hasConfig()) {
 		EEPROM.begin(EEPROM_SIZE);

lib/AmsConfiguration/src/hexutils.cpp

@@ -32,7 +32,7 @@ bool stripNonAscii(uint8_t* in, uint16_t size, bool extended) {
 		if(extended && (in[i] < 32 || in[i] == 127 || in[i] == 129 || in[i] == 141 || in[i] == 143 || in[i] == 144 || in[i] == 157)) {
 			memset(in+i, ' ', 1);
 			ret = true;
-		} else if(in[i] < 32 || in[i] > 126) {
+		} else if(!extended && (in[i] < 32 || in[i] > 126)) {
 			memset(in+i, ' ', 1);
 			ret = true;
 		}

lib/AmsData/include/AmsMqttHandler.h

@@ -25,7 +25,7 @@ public:
 protected:
     MQTTClient* mqtt;
     char* json;
-    uint16_t BufferSize = 1024;
+    uint16_t BufferSize = 2048;
 };
 
 #endif

lib/CloudConnector/include/CloudConnector.h

@@ -0,0 +1,47 @@
+#ifndef _CLOUDCONNECTOR_H
+#define _CLOUDCONNECTOR_H
+
+#include "RemoteDebug.h"
+#include "mbedtls/ssl.h"
+#include "mbedtls/platform.h"
+#include "mbedtls/net.h"
+#include "mbedtls/esp_debug.h"
+#include "mbedtls/entropy.h"
+#include "mbedtls/ctr_drbg.h"
+#include "mbedtls/error.h"
+#include "mbedtls/certs.h"
+#include "mbedtls/rsa.h"
+
+
+const unsigned char PUBLIC_KEY[] = \
+"-----BEGIN PUBLIC KEY-----\n"\
+"MIGfMA0GCSqGSIb3DQEBAQUAA4GNADCBiQKBgQDoIo0CSuuX3tAdF7KPssdlzJNX\n"\
+"QryhgVV1rQIFPhHv3SxzyKtRrRM9s0CVfymcibhnEBXxxg3pxlGmwI/R6k7HHXJN\n"\
+"lBsXzzDtZ/GHDVnw+xRakTfRT0Zt+xdJSH5xJNWq4EwpvJfjA22L1Nz4dKSpgWMx\n"\
+"VRndAaXf0s7Q1XBz2wIDAQAB\n"\
+"-----END PUBLIC KEY-----\0";
+
+
+//const unsigned char PUBLIC_KEY[] = { 0x30, 0x81, 0x9f, 0x30, 0x0d, 0x06, 0x09, 0x2a, 0x86, 0x48, 0x86, 0xf7, 0x0d, 0x01, 0x01, 0x01, 0x05, 0x00, 0x03, 0x81, 0x8d, 0x00, 0x30, 0x81, 0x89, 0x02, 0x81, 0x81, 0x00, 0xe8, 0x22, 0x8d, 0x02, 0x4a, 0xeb, 0x97, 0xde, 0xd0, 0x1d, 0x17, 0xb2, 0x8f, 0xb2, 0xc7, 0x65, 0xcc, 0x93, 0x57, 0x42, 0xbc, 0xa1, 0x81, 0x55, 0x75, 0xad, 0x02, 0x05, 0x3e, 0x11, 0xef, 0xdd, 0x2c, 0x73, 0xc8, 0xab, 0x51, 0xad, 0x13, 0x3d, 0xb3, 0x40, 0x95, 0x7f, 0x29, 0x9c, 0x89, 0xb8, 0x67, 0x10, 0x15, 0xf1, 0xc6, 0x0d, 0xe9, 0xc6, 0x51, 0xa6, 0xc0, 0x8f, 0xd1, 0xea, 0x4e, 0xc7, 0x1d, 0x72, 0x4d, 0x94, 0x1b, 0x17, 0xcf, 0x30, 0xed, 0x67, 0xf1, 0x87, 0x0d, 0x59, 0xf0, 0xfb, 0x14, 0x5a, 0x91, 0x37, 0xd1, 0x4f, 0x46, 0x6d, 0xfb, 0x17, 0x49, 0x48, 0x7e, 0x71, 0x24, 0xd5, 0xaa, 0xe0, 0x4c, 0x29, 0xbc, 0x97, 0xe3, 0x03, 0x6d, 0x8b, 0xd4, 0xdc, 0xf8, 0x74, 0xa4, 0xa9, 0x81, 0x63, 0x31, 0x55, 0x19, 0xdd, 0x01, 0xa5, 0xdf, 0xd2, 0xce, 0xd0, 0xd5, 0x70, 0x73, 0xdb, 0x02, 0x03, 0x01, 0x00, 0x01};
+
+struct CloudData {
+    uint8_t type;
+	int16_t data;
+} __attribute__((packed));
+
+class CloudConnector {
+public:
+    CloudConnector(RemoteDebug*);
+    void setup(const unsigned char * key);
+    void send();
+
+private:
+    RemoteDebug* debugger;
+
+    unsigned char buf[4096];
+    mbedtls_rsa_context* rsa = nullptr;
+
+    void debugPrint(byte *buffer, int start, int length);
+
+};
+#endif

lib/CloudConnector/src/CloudConnector.cpp

@@ -0,0 +1,56 @@
+#include "CloudConnector.h"
+
+CloudConnector::CloudConnector(RemoteDebug* debugger) {
+    this->debugger = debugger;
+    mbedtls_pk_context pk;
+    mbedtls_pk_init(&pk);
+
+    int error_code = 0;
+    if((error_code =  mbedtls_pk_parse_public_key(&pk, PUBLIC_KEY, sizeof(PUBLIC_KEY))) == 0){
+        debugger->printf("RSA public key OK\n");
+        rsa = mbedtls_pk_rsa(pk);
+    } else {
+        debugger->printf("RSA public key read error: ");
+        mbedtls_strerror(error_code, (char*) buf, 4096);
+        debugger->printf("%s\n", buf);
+    }
+    debugger->flush();
+    //send();
+}
+
+void CloudConnector::send() {
+    if(rsa != nullptr && mbedtls_rsa_check_pubkey(rsa) == 0) {
+        memset(buf, 0, 4096);
+
+        CloudData data = {65, 127};
+        unsigned char toEncrypt[4096] = {0};
+
+        debugger->println("RSA clear data: ");
+        debugPrint(toEncrypt, 0, 256);
+
+        mbedtls_rsa_rsaes_pkcs1_v15_encrypt(rsa, NULL, NULL, MBEDTLS_RSA_PUBLIC, 256, toEncrypt, buf);
+
+        //byte hashResult[32];
+        //mbedtls_sha256(toEncrypt, strlen((char*) toEncrypt), hashResult, 0);
+        //int success = mbedtls_rsa_rsassa_pkcs1_v15_sign(rsa, NULL, NULL, MBEDTLS_RSA_PUBLIC, MBEDTLS_MD_SHA256,  strlen((char*) hashResult), hashResult, buf);
+        debugger->println("RSA encrypted data: ");
+        debugPrint(buf, 0, 256);
+    } else {
+        debugger->println("RSA key is invalid");
+    }
+}
+void CloudConnector::debugPrint(byte *buffer, int start, int length) {
+	for (int i = start; i < start + length; i++) {
+		if (buffer[i] < 0x10)
+			debugger->print(F("0"));
+		debugger->print(buffer[i], HEX);
+		debugger->print(F(" "));
+		if ((i - start + 1) % 16 == 0)
+			debugger->println(F(""));
+		else if ((i - start + 1) % 4 == 0)
+			debugger->print(F(" "));
+
+		yield(); // Let other get some resources too
+	}
+	debugger->println(F(""));
+}

lib/JsonMqttHandler/src/JsonMqttHandler.cpp

@@ -13,6 +13,8 @@ bool JsonMqttHandler::publish(AmsData* data, AmsData* previousState, EnergyAccou
 	if(topic.isEmpty() || !mqtt->connected())
 		return false;
 
+    bool ret = false;
+
     String meterModel = data->getMeterModel();
     meterModel.replace("\\", "\\\\");
     if(data->getListType() == 1) {
@@ -32,7 +34,7 @@ bool JsonMqttHandler::publish(AmsData* data, AmsData* previousState, EnergyAccou
             ea->getProducedThisHour(),
             ea->getProducedToday()
         );
-        return mqtt->publish(topic, json);
+        ret = mqtt->publish(topic, json);
     } else if(data->getListType() == 2) {
         snprintf_P(json, BufferSize, JSON2_JSON,
             WiFi.macAddress().c_str(),
@@ -62,7 +64,7 @@ bool JsonMqttHandler::publish(AmsData* data, AmsData* previousState, EnergyAccou
             ea->getProducedThisHour(),
             ea->getProducedToday()
         );
-        return mqtt->publish(topic, json);
+        ret = mqtt->publish(topic, json);
     } else if(data->getListType() == 3) {
         snprintf_P(json, BufferSize, JSON3_JSON,
             WiFi.macAddress().c_str(),
@@ -97,7 +99,7 @@ bool JsonMqttHandler::publish(AmsData* data, AmsData* previousState, EnergyAccou
             ea->getProducedThisHour(),
             ea->getProducedToday()
         );
-        return mqtt->publish(topic, json);
+        ret = mqtt->publish(topic, json);
     } else if(data->getListType() == 4) {
         snprintf_P(json, BufferSize, JSON4_JSON,
             WiFi.macAddress().c_str(),
@@ -142,9 +144,11 @@ bool JsonMqttHandler::publish(AmsData* data, AmsData* previousState, EnergyAccou
             ea->getProducedThisHour(),
             ea->getProducedToday()
         );
-        return mqtt->publish(topic, json);
+        ret = mqtt->publish(topic, json);
     }
-    return false;
+    mqtt->loop();
+    delay(10);
+    return ret;
 }
 
 bool JsonMqttHandler::publishTemperatures(AmsConfiguration* config, HwTools* hw) {
@@ -169,7 +173,10 @@ bool JsonMqttHandler::publishTemperatures(AmsConfiguration* config, HwTools* hw)
 	}
 	char* pos = json+strlen(json);
 	snprintf(count == 0 ? pos : pos-1, 8, "}}");
-    return mqtt->publish(topic, json);
+    bool ret = mqtt->publish(topic, json);
+    mqtt->loop();
+    delay(10);
+    return ret;
 }
 
 bool JsonMqttHandler::publishPrices(EntsoeApi* eapi) {
@@ -301,7 +308,10 @@ bool JsonMqttHandler::publishPrices(EntsoeApi* eapi) {
         ts3hr,
         ts6hr
     );
-    return mqtt->publish(topic, json);
+    bool ret = mqtt->publish(topic, json);
+    mqtt->loop();
+    delay(10);
+    return ret;
 }
 
 bool JsonMqttHandler::publishSystem(HwTools* hw, EntsoeApi* eapi, EnergyAccounting* ea) {
@@ -317,5 +327,8 @@ bool JsonMqttHandler::publishSystem(HwTools* hw, EntsoeApi* eapi, EnergyAccounti
         hw->getTemperature(),
         VERSION
     );
-    return mqtt->publish(topic, json);
+    bool ret = mqtt->publish(topic, json);
+    mqtt->loop();
+    delay(10);
+    return ret;
 }

src/AmsToMqttBridge.h

@@ -6,6 +6,7 @@
 #define INVALID_BUTTON_PIN  0xFFFFFFFF
 
 #define MAX_PEM_SIZE 4096
+#define MAX_RX_BUFFER_SIZE 1024
 
 #define METER_SOURCE_NONE 0
 #define METER_SOURCE_SERIAL 1

src/AmsToMqttBridge.ino

@@ -100,6 +100,7 @@ AmsMqttHandler* mqttHandler = NULL;
 Stream *hanSerial;
 SoftwareSerial *swSerial = NULL;
 HardwareSerial *hwSerial = NULL;
+size_t rxBufferSize = 256;
 
 GpioConfig gpioConfig;
 MeterConfig meterConfig;
@@ -418,11 +419,19 @@ void loop() {
 		}
 		if(hwSerial->hasOverrun()) {
 			meterState.setLastError(METER_ERROR_BUFFER);
+			if(rxBufferSize < MAX_RX_BUFFER_SIZE) {
+				rxBufferSize += 256;
+				debugI("Incresing RX buffer to %d bytes", rxBufferSize);
+				config.setMeterChanged();
+			}
 		}
 		#endif
 	} else if(swSerial != NULL) {
 		if(swSerial->overflow()) {
 			meterState.setLastError(METER_ERROR_BUFFER);
+			rxBufferSize += 256;
+			debugI("Incresing RX buffer to %d bytes", rxBufferSize);
+			config.setMeterChanged();
 		}
 	}
 
@@ -658,6 +667,11 @@ void rxerr(int err) {
 			break;
 		case 2:
 			debugE("Serial buffer full");
+			if(rxBufferSize < MAX_RX_BUFFER_SIZE) {
+				rxBufferSize += 256;
+				debugI("Incresing RX buffer to %d bytes", rxBufferSize);
+				config.setMeterChanged();
+			}
 			break;
 		case 3:
 			debugE("Serial FIFO overflow");
@@ -739,6 +753,7 @@ void setupHanPort(GpioConfig& gpioConfig, uint32_t baud, uint8_t parityOrdinal,
 				break;
 		}
 
+		hwSerial->setRxBufferSize(rxBufferSize);
 		#if defined(CONFIG_IDF_TARGET_ESP32S2) || defined(CONFIG_IDF_TARGET_ESP32C3)
 			hwSerial->begin(baud, serialConfig, -1, -1, invert);
 			uart_set_pin(UART_NUM_1, UART_PIN_NO_CHANGE, pin, UART_PIN_NO_CHANGE, UART_PIN_NO_CHANGE);
@@ -758,7 +773,6 @@ void setupHanPort(GpioConfig& gpioConfig, uint32_t baud, uint8_t parityOrdinal,
 			}
 		#endif
 
-		hwSerial->setRxBufferSize(768);
 		#if defined(ESP32)
 		hwSerial->onReceiveError(rxerr);
 		#endif
@@ -790,7 +804,7 @@ void setupHanPort(GpioConfig& gpioConfig, uint32_t baud, uint8_t parityOrdinal,
 		}
 
 		swSerial = new SoftwareSerial(pin, -1, invert);
-		swSerial->begin(baud, serialConfig);
+		swSerial->begin(baud, serialConfig, pin, -1, invert, rxBufferSize);
 		hanSerial = swSerial;
 
 		Serial.end();
@@ -1394,7 +1408,7 @@ void MQTT_connect() {
 		}
 		yield();
 	} else {
-		mqtt = new MQTTClient(1024);
+		mqtt = new MQTTClient(1500);
 		ws.setMqtt(mqtt);
 	}