From 1e323ac3b90dd8a2e7c5ca31da1387e403e6020b Mon Sep 17 00:00:00 2001 From: Gunnar Skjold Date: Sun, 31 Oct 2021 09:33:25 +0100 Subject: [PATCH] Various changes for version updates and some debugging for upload --- lib/Timezone/library.properties | 2 +- platformio.ini | 13 ++++-- sdkconfig.defaults | 18 ++++++++ src/AmsToMqttBridge.h | 2 + src/AmsToMqttBridge.ino | 52 +++++++++++++++------- src/HwTools.cpp | 1 + src/entsoe/EntsoeApi.cpp | 4 +- src/mqtt/JsonMqttHandler.cpp | 1 + src/web/AmsWebServer.cpp | 76 ++++++++++++++++++++++++++------- 9 files changed, 129 insertions(+), 40 deletions(-) create mode 100644 sdkconfig.defaults diff --git a/lib/Timezone/library.properties b/lib/Timezone/library.properties index c2378c69..b5ab9f09 100644 --- a/lib/Timezone/library.properties +++ b/lib/Timezone/library.properties @@ -7,4 +7,4 @@ paragraph=The primary aim of the Timezone library is to convert Universal Coordi category=Timing url=https://github.com/JChristensen/Timezone architectures=* -depends=Time (=1.6.0) +depends=Time (=1.6.1) diff --git a/platformio.ini b/platformio.ini index 0e62cff3..45840dd0 100755 --- a/platformio.ini +++ b/platformio.ini @@ -2,10 +2,10 @@ extra_configs = platformio-user.ini [common] -lib_deps = file://lib/HanReader, file://lib/Timezone, MQTT@2.5.0, DallasTemperature@3.9.1, EspSoftwareSerial@6.9.0, RemoteDebug@3.0.5, Time@1.6.0 +lib_deps = file://lib/HanReader, file://lib/Timezone, MQTT@2.5.0, DallasTemperature@3.9.1, EspSoftwareSerial@6.14.1, https://github.com/gskjold/RemoteDebug.git@3.0.5, Time@1.6.1 [env:esp8266] -platform = espressif8266@2.6.2 +platform = espressif8266@3.2.0 board = esp12e board_build.ldscript = eagle.flash.4m2m.ld framework = arduino @@ -13,13 +13,18 @@ lib_deps = ${common.lib_deps} extra_scripts = pre:scripts/addversion.py scripts/makeweb.py +build_flags = + -D WEBSOCKET_DISABLED=1 + [env:esp32] -platform = espressif32@2.1.0 +platform = espressif32@3.3.2 board = esp32dev -board_build.partitions = no_ota.csv +;board_build.partitions = no_ota.csv framework = arduino lib_deps = ${common.lib_deps} extra_scripts = pre:scripts/addversion.py scripts/makeweb.py +build_flags = + -D WEBSOCKET_DISABLED=1 diff --git a/sdkconfig.defaults b/sdkconfig.defaults new file mode 100644 index 00000000..ead57919 --- /dev/null +++ b/sdkconfig.defaults @@ -0,0 +1,18 @@ +CONFIG_ENABLE_ARDUINO_DEPENDS=y +CONFIG_AUTOSTART_ARDUINO=y +CONFIG_ARDUINO_RUN_CORE1=y +CONFIG_ARDUINO_RUNNING_CORE=1 +CONFIG_ARDUINO_EVENT_RUN_CORE1=y +CONFIG_ARDUINO_EVENT_RUNNING_CORE=1 +CONFIG_ARDUINO_UDP_RUN_CORE1=y +CONFIG_ARDUINO_UDP_RUNNING_CORE=1 +CONFIG_DISABLE_HAL_LOCKS=y +CONFIG_ARDUHAL_LOG_DEFAULT_LEVEL_ERROR=y +CONFIG_ARDUHAL_LOG_DEFAULT_LEVEL=1 +CONFIG_ARDUHAL_PARTITION_SCHEME_DEFAULT=y +CONFIG_ARDUHAL_PARTITION_SCHEME="default" +CONFIG_AUTOCONNECT_WIFI=y +CONFIG_ARDUINO_SELECTIVE_WiFi=y +CONFIG_MBEDTLS_PSK_MODES=y +CONFIG_MBEDTLS_KEY_EXCHANGE_PSK=y +CONFIG_FREERTOS_UNICORE=y diff --git a/src/AmsToMqttBridge.h b/src/AmsToMqttBridge.h index 284e599e..e5470cea 100644 --- a/src/AmsToMqttBridge.h +++ b/src/AmsToMqttBridge.h @@ -7,6 +7,8 @@ #define EPOCH_2021_01_01 1609459200 +#define MAX_PEM_SIZE 4096 + #include #if defined(ESP8266) diff --git a/src/AmsToMqttBridge.ino b/src/AmsToMqttBridge.ino index 2634e56e..0fd42689 100644 --- a/src/AmsToMqttBridge.ino +++ b/src/AmsToMqttBridge.ino @@ -45,7 +45,6 @@ ADC_MODE(ADC_VCC); #include "Uptime.h" -#define WEBSOCKET_DISABLED true #include "RemoteDebug.h" HwTools hw; @@ -188,17 +187,18 @@ void setup() { WiFi.softAPdisconnect(true); WiFi.mode(WIFI_OFF); - bool spiffs = false; + bool hasFs = false; #if defined(ESP32) debugD("ESP32 SPIFFS"); - spiffs = SPIFFS.begin(true); + hasFs = SPIFFS.begin(true); + debugD(" size: %d", SPIFFS.totalBytes()); #else debugD("ESP8266 SPIFFS"); - spiffs = SPIFFS.begin(); + hasFs = SPIFFS.begin(); #endif delay(1); - if(spiffs) { + if(hasFs) { bool flashed = false; if(SPIFFS.exists(FILE_FIRMWARE)) { if(Debug.isActive(RemoteDebug::INFO)) debugI("Found firmware"); @@ -798,21 +798,41 @@ void MQTT_connect() { char *ca = NULL; char *cert = NULL; char *key = NULL; + File file; if(SPIFFS.exists(FILE_MQTT_CA)) { debugI("Found MQTT CA file"); - File file = SPIFFS.open(FILE_MQTT_CA, "r"); - secureClient->loadCACert(file, file.size()); + file = SPIFFS.open(FILE_MQTT_CA, "r"); + #if defined(ESP8266) + char caStr[MAX_PEM_SIZE]; + file.readBytes(caStr, file.size()); + BearSSL::X509List *serverTrustedCA = new BearSSL::X509List(caStr); + secureClient->setTrustAnchors(serverTrustedCA); + #elif defined(ESP32) + secureClient->loadCACert(file, file.size()); + #endif } - if(SPIFFS.exists(FILE_MQTT_CERT)) { - debugI("Found MQTT certificate file"); - File file = SPIFFS.open(FILE_MQTT_CERT, "r"); - secureClient->loadCertificate(file, file.size()); - } - if(SPIFFS.exists(FILE_MQTT_KEY)) { - debugI("Found MQTT key file"); - File file = SPIFFS.open(FILE_MQTT_KEY, "r"); - secureClient->loadPrivateKey(file, file.size()); + + if(SPIFFS.exists(FILE_MQTT_CERT) && SPIFFS.exists(FILE_MQTT_KEY)) { + #if defined(ESP8266) + char certStr[MAX_PEM_SIZE]; + file = SPIFFS.open(FILE_MQTT_CERT, "r"); + file.readBytes(certStr, file.size()); + BearSSL::X509List *serverCertList = new BearSSL::X509List(certStr); + char keyStr[MAX_PEM_SIZE]; + file = SPIFFS.open(FILE_MQTT_KEY, "r"); + file.readBytes(keyStr, file.size()); + BearSSL::PrivateKey *serverPrivKey = new BearSSL::PrivateKey(keyStr); + secureClient->setClientRSACert(serverCertList, serverPrivKey); + #elif defined(ESP32) + debugI("Found MQTT certificate file"); + file = SPIFFS.open(FILE_MQTT_CERT, "r"); + secureClient->loadCertificate(file, file.size()); + + debugI("Found MQTT key file"); + file = SPIFFS.open(FILE_MQTT_KEY, "r"); + secureClient->loadPrivateKey(file, file.size()); + #endif } SPIFFS.end(); } diff --git a/src/HwTools.cpp b/src/HwTools.cpp index eb3cf446..0a61dd73 100644 --- a/src/HwTools.cpp +++ b/src/HwTools.cpp @@ -217,6 +217,7 @@ bool HwTools::ledBlink(uint8_t color, uint8_t blink) { if(i != blink) delay(50); } + return true; } bool HwTools::writeLedPin(uint8_t color, uint8_t state) { diff --git a/src/entsoe/EntsoeApi.cpp b/src/entsoe/EntsoeApi.cpp index 68d98b83..821e3d65 100644 --- a/src/entsoe/EntsoeApi.cpp +++ b/src/entsoe/EntsoeApi.cpp @@ -184,9 +184,7 @@ bool EntsoeApi::retrieve(const char* url, Stream* doc) { #endif HTTPClient https; - #if defined(ESP8266) - https.setFollowRedirects(true); - #endif + https.setFollowRedirects(HTTPC_STRICT_FOLLOW_REDIRECTS); if(https.begin(client, url)) { printD("Connection established"); diff --git a/src/mqtt/JsonMqttHandler.cpp b/src/mqtt/JsonMqttHandler.cpp index b7959ca2..5c216d6d 100644 --- a/src/mqtt/JsonMqttHandler.cpp +++ b/src/mqtt/JsonMqttHandler.cpp @@ -80,6 +80,7 @@ bool JsonMqttHandler::publish(AmsData* data, AmsData* previousState) { ); return mqtt->publish(topic, json); } + return false; } bool JsonMqttHandler::publishTemperatures(AmsConfiguration* config, HwTools* hw) { diff --git a/src/web/AmsWebServer.cpp b/src/web/AmsWebServer.cpp index eb0418bc..37b5f2f2 100644 --- a/src/web/AmsWebServer.cpp +++ b/src/web/AmsWebServer.cpp @@ -198,7 +198,7 @@ void AmsWebServer::temperaturePost() { server.send (302, "text/plain", ""); } else { printE("Error saving configuration"); - String html = "

Error saving configuration!

"; + String html = "

Error saving configuration!

"; server.send(500, "text/html", html); } } @@ -895,7 +895,7 @@ void AmsWebServer::handleSetup() { server.send(303); } else { printE("Error saving configuration"); - String html = "

Error saving configuration!

"; + String html = "

Error saving configuration!

"; server.send(500, "text/html", html); } } @@ -1083,7 +1083,7 @@ void AmsWebServer::handleSave() { } } else { printE("Error saving configuration"); - String html = "

Error saving configuration!

"; + String html = "

Error saving configuration!

"; server.send(500, "text/html", html); } } @@ -1235,31 +1235,75 @@ void AmsWebServer::uploadFile(const char* path) { if(upload.status == UPLOAD_FILE_START){ if(uploading) { printE("Upload already in progress"); - String html = "

Upload already in progress!

"; + String html = "

Upload already in progress!

"; server.send(500, "text/html", html); } else if (!SPIFFS.begin()) { printE("An Error has occurred while mounting SPIFFS"); - String html = "

Unable to mount SPIFFS!

"; + String html = "

Unable to mount SPIFFS!

"; server.send(500, "text/html", html); } else { uploading = true; if(debugger->isActive(RemoteDebug::DEBUG)) { debugger->printf("handleFileUpload file: %s\n", path); } + #if defined(ESP32) + if(debugger->isActive(RemoteDebug::DEBUG)) { + debugger->printf("handleFileUpload Free heap: %lu\n", ESP.getFreeHeap()); + debugger->printf("handleFileUpload SPIFFS size: %lu\n", SPIFFS.totalBytes()); + debugger->printf("handleFileUpload SPIFFS used: %lu\n", SPIFFS.usedBytes()); + debugger->printf("handleFileUpload SPIFFS free: %lu\n", SPIFFS.totalBytes()-SPIFFS.usedBytes()); + } + #endif file = SPIFFS.open(path, "w"); - file.write(upload.buf, upload.currentSize); + if(debugger->isActive(RemoteDebug::DEBUG)) { + debugger->printf("handleFileUpload Open file and write: %lu\n", upload.currentSize); + } + size_t written = file.write(upload.buf, upload.currentSize); + if(debugger->isActive(RemoteDebug::DEBUG)) { + debugger->printf("handleFileUpload Written: %lu\n", written); + } } } else if(upload.status == UPLOAD_FILE_WRITE) { - if(file) - file.write(upload.buf, upload.currentSize); + if(debugger->isActive(RemoteDebug::DEBUG)) { + debugger->printf("handleFileUpload Writing: %lu\n", upload.currentSize); + } + if(file) { + size_t written = file.write(upload.buf, upload.currentSize); + if(debugger->isActive(RemoteDebug::DEBUG)) { + debugger->printf("handleFileUpload Written: %lu\n", written); + } + delay(1); + if(written != upload.currentSize) { + #if defined(ESP32) + if(debugger->isActive(RemoteDebug::DEBUG)) { + debugger->printf("handleFileUpload Free heap: %lu\n", ESP.getFreeHeap()); + debugger->printf("handleFileUpload SPIFFS size: %lu\n", SPIFFS.totalBytes()); + debugger->printf("handleFileUpload SPIFFS used: %lu\n", SPIFFS.usedBytes()); + debugger->printf("handleFileUpload SPIFFS free: %lu\n", SPIFFS.totalBytes()-SPIFFS.usedBytes()); + } + #endif + + file.flush(); + file.close(); + SPIFFS.remove(path); + SPIFFS.end(); + + printE("An Error has occurred while writing file"); + String html = "

Unable to write file!

"; + server.send(500, "text/html", html); + } + } } else if(upload.status == UPLOAD_FILE_END) { if(file) { file.flush(); file.close(); - SPIFFS.end(); + file = SPIFFS.open(path, "r"); if(debugger->isActive(RemoteDebug::DEBUG)) { debugger->printf("handleFileUpload Size: %lu\n", upload.totalSize); + debugger->printf("handleFileUpload File size: %lu\n", file.size()); } + file.close(); + SPIFFS.end(); } else { server.send(500, "text/plain", "500: couldn't create file"); } @@ -1315,16 +1359,16 @@ void AmsWebServer::firmwareDownload() { WiFiClientSecure client; #if defined(ESP8266) client.setBufferSizes(512, 512); + String url = "https://github.com/gskjold/AmsToMqttBridge/releases/download/" + version + "/ams2mqtt-esp8266-" + versionStripped + ".bin"; +#elif defined(ESP32) + String url = "https://github.com/gskjold/AmsToMqttBridge/releases/download/" + version + "/ams2mqtt-esp32-" + versionStripped + ".bin"; +#endif client.setInsecure(); -#endif - String url = "https://github.com/gskjold/AmsToMqttBridge/releases/download/" + version + "/ams2mqtt-esp12e-" + versionStripped + ".bin"; HTTPClient https; -#if defined(ESP8266) - https.setFollowRedirects(true); -#endif + https.setFollowRedirects(HTTPC_STRICT_FOLLOW_REDIRECTS); + https.addHeader("Referer", "https://github.com/gskjold/AmsToMqttBridge/releases"); if(https.begin(client, url)) { - https.addHeader("Referer", "https://github.com/gskjold/AmsToMqttBridge/releases"); printD("HTTP client setup successful"); int status = https.GET(); if(status == HTTP_CODE_OK) { @@ -1332,7 +1376,6 @@ void AmsWebServer::firmwareDownload() { if(SPIFFS.begin()) { printI("Downloading firmware to SPIFFS"); file = SPIFFS.open(FILE_FIRMWARE, "w"); - // The following does not work... Maybe someone will make it work in the future? It seems to be disconnected at this point. int len = https.writeToStream(&file); file.close(); SPIFFS.end(); @@ -1365,6 +1408,7 @@ void AmsWebServer::firmwareDownload() { server.send(303); } https.end(); + client.stop(); } else { printI("No firmware version specified..."); server.sendHeader("Location","/");