Various changes for version updates and some debugging for upload

lib/Timezone/library.properties

@@ -7,4 +7,4 @@ paragraph=The primary aim of the Timezone library is to convert Universal Coordi
 category=Timing
 url=https://github.com/JChristensen/Timezone
 architectures=*
-depends=Time (=1.6.0)
+depends=Time (=1.6.1)

platformio.ini

@@ -2,10 +2,10 @@
 extra_configs = platformio-user.ini
 
 [common]
-lib_deps = file://lib/HanReader, file://lib/Timezone, MQTT@2.5.0, DallasTemperature@3.9.1, EspSoftwareSerial@6.9.0, RemoteDebug@3.0.5, Time@1.6.0
+lib_deps = file://lib/HanReader, file://lib/Timezone, MQTT@2.5.0, DallasTemperature@3.9.1, EspSoftwareSerial@6.14.1, https://github.com/gskjold/RemoteDebug.git@3.0.5, Time@1.6.1
 
 [env:esp8266]
-platform = espressif8266@2.6.2
+platform = espressif8266@3.2.0
 board = esp12e
 board_build.ldscript = eagle.flash.4m2m.ld
 framework = arduino
@@ -13,13 +13,18 @@ lib_deps = ${common.lib_deps}
 extra_scripts =
     pre:scripts/addversion.py
     scripts/makeweb.py
+build_flags =
+    -D WEBSOCKET_DISABLED=1
+
 
 [env:esp32]
-platform = espressif32@2.1.0
+platform = espressif32@3.3.2
 board = esp32dev
-board_build.partitions = no_ota.csv
+;board_build.partitions = no_ota.csv
 framework = arduino
 lib_deps = ${common.lib_deps}
 extra_scripts =
     pre:scripts/addversion.py
     scripts/makeweb.py
+build_flags =
+    -D WEBSOCKET_DISABLED=1

sdkconfig.defaults

@@ -0,0 +1,18 @@
+CONFIG_ENABLE_ARDUINO_DEPENDS=y
+CONFIG_AUTOSTART_ARDUINO=y
+CONFIG_ARDUINO_RUN_CORE1=y
+CONFIG_ARDUINO_RUNNING_CORE=1
+CONFIG_ARDUINO_EVENT_RUN_CORE1=y
+CONFIG_ARDUINO_EVENT_RUNNING_CORE=1
+CONFIG_ARDUINO_UDP_RUN_CORE1=y
+CONFIG_ARDUINO_UDP_RUNNING_CORE=1
+CONFIG_DISABLE_HAL_LOCKS=y
+CONFIG_ARDUHAL_LOG_DEFAULT_LEVEL_ERROR=y
+CONFIG_ARDUHAL_LOG_DEFAULT_LEVEL=1
+CONFIG_ARDUHAL_PARTITION_SCHEME_DEFAULT=y
+CONFIG_ARDUHAL_PARTITION_SCHEME="default"
+CONFIG_AUTOCONNECT_WIFI=y
+CONFIG_ARDUINO_SELECTIVE_WiFi=y
+CONFIG_MBEDTLS_PSK_MODES=y
+CONFIG_MBEDTLS_KEY_EXCHANGE_PSK=y
+CONFIG_FREERTOS_UNICORE=y

src/AmsToMqttBridge.h

@@ -7,6 +7,8 @@
 
 #define EPOCH_2021_01_01 1609459200
 
+#define MAX_PEM_SIZE 4096
+
 #include <SoftwareSerial.h>
 
 #if defined(ESP8266)

src/AmsToMqttBridge.ino

@@ -45,7 +45,6 @@ ADC_MODE(ADC_VCC);
 
 #include "Uptime.h"
 
-#define WEBSOCKET_DISABLED true
 #include "RemoteDebug.h"
 
 HwTools hw;
@@ -188,17 +187,18 @@ void setup() {
 	WiFi.softAPdisconnect(true);
 	WiFi.mode(WIFI_OFF);
 
-	bool spiffs = false;
+	bool hasFs = false;
 #if defined(ESP32)
 	debugD("ESP32 SPIFFS");
-	spiffs = SPIFFS.begin(true);
+	hasFs = SPIFFS.begin(true);
+	debugD(" size: %d", SPIFFS.totalBytes());
 #else
 	debugD("ESP8266 SPIFFS");
-	spiffs = SPIFFS.begin();
+	hasFs = SPIFFS.begin();
 #endif
 	delay(1);
 
-	if(spiffs) {
+	if(hasFs) {
 		bool flashed = false;
 		if(SPIFFS.exists(FILE_FIRMWARE)) {
 			if(Debug.isActive(RemoteDebug::INFO)) debugI("Found firmware");
@@ -798,21 +798,41 @@ void MQTT_connect() {
 			char *ca = NULL;
 			char *cert = NULL;
 			char *key = NULL;
+			File file;
 
 			if(SPIFFS.exists(FILE_MQTT_CA)) {
 				debugI("Found MQTT CA file");
-				File file = SPIFFS.open(FILE_MQTT_CA, "r");
+				file = SPIFFS.open(FILE_MQTT_CA, "r");
-				secureClient->loadCACert(file, file.size());
+				#if defined(ESP8266)
+					char caStr[MAX_PEM_SIZE];
+					file.readBytes(caStr, file.size());
+					BearSSL::X509List *serverTrustedCA = new BearSSL::X509List(caStr);
+					secureClient->setTrustAnchors(serverTrustedCA);
+				#elif defined(ESP32)
+					secureClient->loadCACert(file, file.size());
+				#endif
 			}
-			if(SPIFFS.exists(FILE_MQTT_CERT)) {
+
-				debugI("Found MQTT certificate file");
+			if(SPIFFS.exists(FILE_MQTT_CERT) && SPIFFS.exists(FILE_MQTT_KEY)) {
-				File file = SPIFFS.open(FILE_MQTT_CERT, "r");
+				#if defined(ESP8266)
-				secureClient->loadCertificate(file, file.size());
+					char certStr[MAX_PEM_SIZE];
-			}
+					file = SPIFFS.open(FILE_MQTT_CERT, "r");
-			if(SPIFFS.exists(FILE_MQTT_KEY)) {
+					file.readBytes(certStr, file.size());
-				debugI("Found MQTT key file");
+				 	BearSSL::X509List *serverCertList = new BearSSL::X509List(certStr);
-				File file = SPIFFS.open(FILE_MQTT_KEY, "r");
+					char keyStr[MAX_PEM_SIZE];
-				secureClient->loadPrivateKey(file, file.size());
+					file = SPIFFS.open(FILE_MQTT_KEY, "r");
+					file.readBytes(keyStr, file.size());
+  					BearSSL::PrivateKey *serverPrivKey = new BearSSL::PrivateKey(keyStr);
+					secureClient->setClientRSACert(serverCertList, serverPrivKey);
+				#elif defined(ESP32)
+					debugI("Found MQTT certificate file");
+					file = SPIFFS.open(FILE_MQTT_CERT, "r");
+					secureClient->loadCertificate(file, file.size());
+
+					debugI("Found MQTT key file");
+					file = SPIFFS.open(FILE_MQTT_KEY, "r");
+					secureClient->loadPrivateKey(file, file.size());
+				#endif
 			}
 			SPIFFS.end();
 		}

src/HwTools.cpp

@@ -217,6 +217,7 @@ bool HwTools::ledBlink(uint8_t color, uint8_t blink) {
         if(i != blink)
             delay(50);
     }
+    return true;
 }
 
 bool HwTools::writeLedPin(uint8_t color, uint8_t state) {

src/entsoe/EntsoeApi.cpp

@@ -184,9 +184,7 @@ bool EntsoeApi::retrieve(const char* url, Stream* doc) {
     #endif
     
     HTTPClient https;
-    #if defined(ESP8266)
+    https.setFollowRedirects(HTTPC_STRICT_FOLLOW_REDIRECTS);
-        https.setFollowRedirects(true);
-    #endif
 
     if(https.begin(client, url)) {
         printD("Connection established");

src/mqtt/JsonMqttHandler.cpp

@@ -80,6 +80,7 @@ bool JsonMqttHandler::publish(AmsData* data, AmsData* previousState) {
         );
         return mqtt->publish(topic, json);
     }
+    return false;
 }
 
 bool JsonMqttHandler::publishTemperatures(AmsConfiguration* config, HwTools* hw) {

src/web/AmsWebServer.cpp

@@ -198,7 +198,7 @@ void AmsWebServer::temperaturePost() {
 		server.send (302, "text/plain", "");
 	} else {
 		printE("Error saving configuration");
-		String html = "<html><body><h1>Error saving configuration!</h1></form>";
+		String html = "<html><body><h1>Error saving configuration!</h1></body></html>";
 		server.send(500, "text/html", html);
 	}
 }
@@ -895,7 +895,7 @@ void AmsWebServer::handleSetup() {
 			server.send(303);
 		} else {
 			printE("Error saving configuration");
-			String html = "<html><body><h1>Error saving configuration!</h1></form>";
+			String html = "<html><body><h1>Error saving configuration!</h1></body></html>";
 			server.send(500, "text/html", html);
 		}
 	}
@@ -1083,7 +1083,7 @@ void AmsWebServer::handleSave() {
 		}
 	} else {
 		printE("Error saving configuration");
-		String html = "<html><body><h1>Error saving configuration!</h1></form>";
+		String html = "<html><body><h1>Error saving configuration!</h1></body></html>";
 		server.send(500, "text/html", html);
 	}
 }
@@ -1235,31 +1235,75 @@ void AmsWebServer::uploadFile(const char* path) {
     if(upload.status == UPLOAD_FILE_START){
 		if(uploading) {
 			printE("Upload already in progress");
-			String html = "<html><body><h1>Upload already in progress!</h1></form>";
+			String html = "<html><body><h1>Upload already in progress!</h1></body></html>";
 			server.send(500, "text/html", html);
 		} else if (!SPIFFS.begin()) {
 			printE("An Error has occurred while mounting SPIFFS");
-			String html = "<html><body><h1>Unable to mount SPIFFS!</h1></form>";
+			String html = "<html><body><h1>Unable to mount SPIFFS!</h1></body></html>";
 			server.send(500, "text/html", html);
 		} else {
 			uploading = true;
 			if(debugger->isActive(RemoteDebug::DEBUG)) {
 				debugger->printf("handleFileUpload file: %s\n", path);
 			}
+			#if defined(ESP32)
+				if(debugger->isActive(RemoteDebug::DEBUG)) {
+					debugger->printf("handleFileUpload Free heap: %lu\n", ESP.getFreeHeap());
+					debugger->printf("handleFileUpload SPIFFS size: %lu\n", SPIFFS.totalBytes());
+					debugger->printf("handleFileUpload SPIFFS used: %lu\n", SPIFFS.usedBytes());
+					debugger->printf("handleFileUpload SPIFFS free: %lu\n", SPIFFS.totalBytes()-SPIFFS.usedBytes());
+				}
+			#endif
 		    file = SPIFFS.open(path, "w");
-            file.write(upload.buf, upload.currentSize);
+			if(debugger->isActive(RemoteDebug::DEBUG)) {
+				debugger->printf("handleFileUpload Open file and write: %lu\n", upload.currentSize);
+			}
+            size_t written = file.write(upload.buf, upload.currentSize);
+			if(debugger->isActive(RemoteDebug::DEBUG)) {
+				debugger->printf("handleFileUpload Written: %lu\n", written);
+			}
 	    } 
     } else if(upload.status == UPLOAD_FILE_WRITE) {
-        if(file)
+		if(debugger->isActive(RemoteDebug::DEBUG)) {
-            file.write(upload.buf, upload.currentSize);
+			debugger->printf("handleFileUpload Writing: %lu\n", upload.currentSize);
+		}
+        if(file) {
+            size_t written = file.write(upload.buf, upload.currentSize);
+			if(debugger->isActive(RemoteDebug::DEBUG)) {
+				debugger->printf("handleFileUpload Written: %lu\n", written);
+			}
+			delay(1);
+			if(written != upload.currentSize) {
+				#if defined(ESP32)
+					if(debugger->isActive(RemoteDebug::DEBUG)) {
+						debugger->printf("handleFileUpload Free heap: %lu\n", ESP.getFreeHeap());
+						debugger->printf("handleFileUpload SPIFFS size: %lu\n", SPIFFS.totalBytes());
+						debugger->printf("handleFileUpload SPIFFS used: %lu\n", SPIFFS.usedBytes());
+						debugger->printf("handleFileUpload SPIFFS free: %lu\n", SPIFFS.totalBytes()-SPIFFS.usedBytes());
+					}
+				#endif
+
+				file.flush();
+				file.close();
+				SPIFFS.remove(path);
+				SPIFFS.end();
+
+				printE("An Error has occurred while writing file");
+				String html = "<html><body><h1>Unable to write file!</h1></body></html>";
+				server.send(500, "text/html", html);
+			}
+		}
     } else if(upload.status == UPLOAD_FILE_END) {
         if(file) {
 			file.flush();
             file.close();
-			SPIFFS.end();
+		    file = SPIFFS.open(path, "r");
 			if(debugger->isActive(RemoteDebug::DEBUG)) {
 				debugger->printf("handleFileUpload Size: %lu\n", upload.totalSize);
+				debugger->printf("handleFileUpload File size: %lu\n", file.size());
 			}
+            file.close();
+			SPIFFS.end();
         } else {
             server.send(500, "text/plain", "500: couldn't create file");
         }
@@ -1315,16 +1359,16 @@ void AmsWebServer::firmwareDownload() {
 		WiFiClientSecure client;
 #if defined(ESP8266)
 		client.setBufferSizes(512, 512);
+		String url = "https://github.com/gskjold/AmsToMqttBridge/releases/download/" + version + "/ams2mqtt-esp8266-" + versionStripped + ".bin";
+#elif defined(ESP32)
+		String url = "https://github.com/gskjold/AmsToMqttBridge/releases/download/" + version + "/ams2mqtt-esp32-" + versionStripped + ".bin";
+#endif
 		client.setInsecure();
-#endif
-		String url = "https://github.com/gskjold/AmsToMqttBridge/releases/download/" + version + "/ams2mqtt-esp12e-" + versionStripped + ".bin";
 		HTTPClient https;
-#if defined(ESP8266)
+		https.setFollowRedirects(HTTPC_STRICT_FOLLOW_REDIRECTS);
-		https.setFollowRedirects(true);
-#endif
 
+		https.addHeader("Referer", "https://github.com/gskjold/AmsToMqttBridge/releases");
 		if(https.begin(client, url)) {
-			https.addHeader("Referer", "https://github.com/gskjold/AmsToMqttBridge/releases");
 			printD("HTTP client setup successful");
 			int status = https.GET();
 			if(status == HTTP_CODE_OK) {
@@ -1332,7 +1376,6 @@ void AmsWebServer::firmwareDownload() {
 				if(SPIFFS.begin()) {
 					printI("Downloading firmware to SPIFFS");
 					file = SPIFFS.open(FILE_FIRMWARE, "w");
-					// The following does not work... Maybe someone will make it work in the future? It seems to be disconnected at this point.
 					int len = https.writeToStream(&file);
 					file.close();
 					SPIFFS.end();
@@ -1365,6 +1408,7 @@ void AmsWebServer::firmwareDownload() {
 			server.send(303);
 		}
 		https.end();
+		client.stop();
 	} else {
 		printI("No firmware version specified...");
 		server.sendHeader("Location","/");