Added option to select firmware channel (#1060)

* 15min prices WIP

* WIP more changes for 15min prices

* More work on 15min pricing

* Fixed some errors

* Some changes after testing

* Graphical changes for 15min pricing

* Adjustments on MQTT handlers after switching to 15min prices

* Reverted some MQTT changes

* Adapted HA integration for 15min pricing

* Adapted JSON payload for 15min

* Adjustments during testing

* Set default price interval

* Fixed refresh of price graph when data changes

* Bugfixes

* Fixed some issues with raw payload

* Adjustments for meter timestamp from Kamstrup

* Updated readme

* Added detailed breakdown of payloads coming from Norwegian meters

* Minor changes relating to price

* Fixed byte alignment on price config

* Changes to support RC upgraders

.github/workflows/prerelease.yml

@@ -82,3 +82,4 @@ jobs:
       version: ${{ needs.prepare.outputs.version }}
       upload_url: ${{ needs.prepare.outputs.upload_url }}
       subfolder: /rc
+      is_esp32: false

.github/workflows/release-deploy-env.yml

@@ -20,6 +20,11 @@ on:
         required: false
         type: string
         default: ''
+      is_esp32:
+        description: 'Whether the build is for ESP32 based firmware'
+        required: false
+        type: boolean
+        default: true
 
 jobs:
   build-and-deploy:
@@ -79,11 +84,11 @@ jobs:
         CI: false
 
     - name: PlatformIO lib install
-      env:
-        GITHUB_TAG: v${{ inputs.version }}
       run: pio lib install
 
     - name: Build firmware
+      env:
+        GITHUB_TAG: v${{ inputs.version }}
       run: pio run -e ${{ inputs.env }}
 
     - name: Create zip file
@@ -119,10 +124,13 @@ jobs:
       run: aws s3 cp firmware.md5 s3://${{ secrets.AWS_S3_BUCKET }}/firmware${{ inputs.subfolder }}/ams2mqtt-${{ inputs.env }}-${{ inputs.version }}.md5
 
     - name: Upload bootloader to S3
+      if: ${{ inputs.is_esp32 }}
       run: aws s3 cp .pio/build/${{ inputs.env }}/bootloader.bin s3://${{ secrets.AWS_S3_BUCKET }}/firmware${{ inputs.subfolder }}/ams2mqtt-${{ inputs.env }}-${{ inputs.version }}-bootloader.bin
     
     - name: Upload partition table to S3
+      if: ${{ inputs.is_esp32 }}
       run: aws s3 cp .pio/build/${{ inputs.env }}/partitions.bin s3://${{ secrets.AWS_S3_BUCKET }}/firmware${{ inputs.subfolder }}/ams2mqtt-${{ inputs.env }}-${{ inputs.version }}-partitions.bin
 
     - name: Upload app0 to S3
+      if: ${{ inputs.is_esp32 }}
       run: aws s3 cp ~/.platformio/packages/framework-arduinoespressif32/tools/partitions/boot_app0.bin s3://${{ secrets.AWS_S3_BUCKET }}/firmware${{ inputs.subfolder }}/ams2mqtt-${{ inputs.env }}-${{ inputs.version }}-app0.bin

.github/workflows/release.yml

@@ -76,3 +76,4 @@ jobs:
       env: esp8266
       version: ${{ needs.prepare.outputs.version }}
       upload_url: ${{ needs.prepare.outputs.upload_url }}
+      is_esp32: false

lib/AmsConfiguration/include/AmsConfiguration.h

@@ -50,6 +50,11 @@
 #define LED_BEHAVIOUR_ERROR_ONLY 3
 #define LED_BEHAVIOUR_OFF 9
 
+#define FIRMWARE_CHANNEL_STABLE 0
+#define FIRMWARE_CHANNEL_EARLY 1
+#define FIRMWARE_CHANNEL_RC 2
+#define FIRMWARE_CHANNEL_SNAPSHOT 3
+
 struct ResetDataContainer {
 	uint8_t cause;
 	uint8_t last_cause;
@@ -63,6 +68,7 @@ struct SystemConfig {
 	uint8_t dataCollectionConsent; // 0 = unknown, 1 = accepted, 2 = declined
 	char country[3];
 	uint8_t energyspeedometer;
+	uint8_t firmwareChannel;
 }; // 8
 
 struct NetworkConfig {
@@ -239,14 +245,14 @@ struct UiConfig {
 }; // 15
 
 struct UpgradeInformation {
-	char fromVersion[8];
-	char toVersion[8];
+	char fromVersion[16];
+	char toVersion[16];
     uint32_t size;
     uint16_t block_position;
     uint8_t retry_count;
     uint8_t reboot_count;
 	int8_t errorCode;
-}; // 25
+}; // 41+3
 
 struct CloudConfig {
 	bool enabled;

lib/AmsConfiguration/src/AmsConfiguration.cpp

@@ -15,6 +15,11 @@ bool AmsConfiguration::getSystemConfig(SystemConfig& config) {
 	uint8_t configVersion = EEPROM.read(EEPROM_CONFIG_ADDRESS);
 	EEPROM.get(CONFIG_SYSTEM_START, config);
 	EEPROM.end();
+
+	if(config.firmwareChannel > 3) {
+		config.firmwareChannel = 0;
+	}
+
 	if(configVersion == EEPROM_CHECK_SUM) {
 		return true;
 	} else {
@@ -27,6 +32,7 @@ bool AmsConfiguration::getSystemConfig(SystemConfig& config) {
 		}
 		config.userConfigured = false;
 		config.dataCollectionConsent = 0;
+		config.firmwareChannel = 0;
 		config.energyspeedometer = 0;
 		memset(config.country, 0, 3);
 		return false;
@@ -42,6 +48,9 @@ bool AmsConfiguration::setSystemConfig(SystemConfig& config) {
 		sysChanged |= config.dataCollectionConsent != existing.dataCollectionConsent;
 		sysChanged |= strcmp(config.country, existing.country) != 0;
 		sysChanged |= config.energyspeedometer != existing.energyspeedometer;
+		sysChanged |= config.firmwareChannel != existing.firmwareChannel;
+	} else {
+		sysChanged = true;
 	}
 	EEPROM.begin(EEPROM_SIZE);
 	stripNonAscii((uint8_t*) config.country, 2);
@@ -979,6 +988,7 @@ void AmsConfiguration::clear() {
 	EEPROM.get(CONFIG_SYSTEM_START, sys);
 	sys.userConfigured = false;
 	sys.dataCollectionConsent = 0;
+	sys.firmwareChannel = 0;
 	sys.energyspeedometer = 0;
 	memset(sys.country, 0, 3);
 	EEPROM.put(CONFIG_SYSTEM_START, sys);

lib/AmsFirmwareUpdater/include/AmsFirmwareUpdater.h

@@ -60,6 +60,13 @@ public:
     bool isUpgradeInformationChanged();
     void ackUpgradeInformationChanged();
 
+    void setFirmwareChannel(uint8_t channel) {
+        if(firmwareChannel != channel) {
+            firmwareChannel = channel;
+            lastVersionCheck = 0;
+        }
+    }
+
     bool startFirmwareUpload(uint32_t size, const char* version);
     bool addFirmwareUploadChunk(uint8_t* buf, size_t length);
     bool completeFirmwareUpload(uint32_t size);
@@ -95,10 +102,11 @@ private:
     String md5;
 
     uint32_t lastVersionCheck = 0;
-    uint8_t firmwareVariant;
+    uint8_t firmwareChannel;
     bool autoUpgrade;
-    char nextVersion[10];
+    char nextVersion[17];
 
+    void getChannelName(char * buffer);
 
     bool fetchNextVersion();
     bool fetchVersionDetails();

lib/AmsFirmwareUpdater/src/AmsFirmwareUpdater.cpp

@@ -22,7 +22,7 @@ this->debugger = debugger;
     this->hw = hw;
     this->meterState = meterState;
     memset(nextVersion, 0, sizeof(nextVersion));
-    firmwareVariant = 0;
+    firmwareChannel = 0;
     autoUpgrade = false;
 }
 
@@ -208,15 +208,33 @@ void AmsFirmwareUpdater::loop() {
     }
 }
 
+void AmsFirmwareUpdater::getChannelName(char * buffer) {
+    switch(firmwareChannel) {
+        case FIRMWARE_CHANNEL_EARLY:
+            strcpy(buffer, PSTR("early"));
+            break;
+        case FIRMWARE_CHANNEL_RC:
+            strcpy(buffer, PSTR("rc"));
+            break;
+        case FIRMWARE_CHANNEL_SNAPSHOT:
+            strcpy(buffer, PSTR("snapshot"));
+            break;  
+        default:
+            strcpy(buffer, PSTR("stable"));
+            break;
+    }
+}
+
 bool AmsFirmwareUpdater::fetchNextVersion() {
     HTTPClient http;
     const char * headerkeys[] = { "x-version" };
     http.collectHeaders(headerkeys, 1);
 
-    char firmwareVariant[10] = "stable";
+    char channel[10] = "";
+    getChannelName(channel);
 
     char url[128];
-    snprintf_P(url, 128, PSTR("http://hub.amsleser.no/hub/firmware/%s/%s/next"), chipType, firmwareVariant);
+    snprintf_P(url, 128, PSTR("http://hub.amsleser.no/hub/firmware/%s/%s/next"), chipType, channel);
     #if defined(ESP8266)
     WiFiClient client;
     client.setTimeout(5000);
@@ -263,10 +281,11 @@ bool AmsFirmwareUpdater::fetchVersionDetails() {
     const char * headerkeys[] = { "x-size" };
     http.collectHeaders(headerkeys, 1);
 
-    char firmwareVariant[10] = "stable";
+    char channel[10] = "";
+    getChannelName(channel);
 
     char url[128];
-    snprintf_P(url, 128, PSTR("http://hub.amsleser.no/hub/firmware/%s/%s/%s/details"), chipType, firmwareVariant, updateStatus.toVersion);
+    snprintf_P(url, 128, PSTR("http://hub.amsleser.no/hub/firmware/%s/%s/%s/details"), chipType, channel, updateStatus.toVersion);
     #if defined(ESP8266)
     WiFiClient client;
     client.setTimeout(5000);
@@ -319,10 +338,11 @@ bool AmsFirmwareUpdater::fetchFirmwareChunk(HTTPClient& http) {
     char range[24];
     snprintf_P(range, 24, PSTR("bytes=%lu-%lu"), start, end);
 
-    char firmwareVariant[10] = "stable";
+    char channel[10] = "";
+    getChannelName(channel);
 
     char url[128];
-    snprintf_P(url, 128, PSTR("http://hub.amsleser.no/hub/firmware/%s/%s/%s/chunk"), chipType, firmwareVariant, updateStatus.toVersion);
+    snprintf_P(url, 128, PSTR("http://hub.amsleser.no/hub/firmware/%s/%s/%s/chunk"), chipType, channel, updateStatus.toVersion);
     #if defined(ESP8266)
     WiFiClient client;
     client.setTimeout(5000);

lib/SvelteUi/app/src/lib/StatusPage.svelte

@@ -1,5 +1,5 @@
 <script>
-    import { metertype, boardtype, isBusPowered, getBaseChip } from './Helpers.js';
+    import { metertype, boardtype, isBusPowered, getBaseChip, wiki } from './Helpers.js';
     import { getSysinfo, sysinfoStore } from './DataStores.js';
     import { upgrade, upgradeWarningText } from './UpgradeHelper';
     import { translationsStore } from './TranslationService.js';
@@ -108,6 +108,15 @@
         });
     };
 
+    function changeFirmwareChannel() {
+        const formData = new FormData();
+        formData.append('channel', sysinfo.upgrade.c);
+        fetch('fwchannel', {
+            method: 'POST',
+            body: formData
+        });
+    };
+
     $: {
         if(configFiles.length == 1) {
             let file = configFiles[0];
@@ -209,25 +218,37 @@
     {/if}
     <div class="cnt">
         <strong class="text-sm">{translations.status?.firmware?.title ?? "Firmware"}</strong>
+        <a href="{wiki('Firmware-Channels')}" target="_blank" class="float-right">&#9432;</a>
+        {#if sysinfo.fwconsent === 1}
+            <div class="my-2">
+                Channel: 
+                <select class="in-s w-full" bind:value={sysinfo.upgrade.c} on:change={changeFirmwareChannel}>
+                    <option value={0}>Stable</option>
+                    <option value={1}>Early</option>
+                    <option value={2}>Release Candidate</option>
+                    <option value={3} disabled>Snapshot</option>
+                </select>
+            </div>
+        {/if}
         <div class="my-2">
             {translations.status?.firmware?.installed ?? "Installed"}: {sysinfo.version}
         </div>
         {#if sysinfo.upgrade.t && sysinfo.upgrade.t != sysinfo.version && sysinfo.upgrade.e != 0 && sysinfo.upgrade.e != 123}
-        <div class="my-2">
-            <div class="bd-yellow">
-                {(translations.status?.firmware?.failed ?? "Upgrade from {0} to {1} failed").replace('{0}', sysinfo.upgrade.f).replace('{1}', sysinfo.upgrade.t)}
-                {(translations.errors?.upgrade?.[sysinfo.upgrade.e] ?? sysinfo.upgrade.e)}
+            <div class="my-2">
+                <div class="bd-yellow">
+                    {(translations.status?.firmware?.failed ?? "Upgrade from {0} to {1} failed").replace('{0}', sysinfo.upgrade.f).replace('{1}', sysinfo.upgrade.t)}
+                    {(translations.errors?.upgrade?.[sysinfo.upgrade.e] ?? sysinfo.upgrade.e)}
+                </div>
             </div>
-        </div>
         {/if}
         {#if sysinfo.upgrade.n}
             <div class="my-2 flex">
                 {translations.status?.firmware?.latest ?? "Latest"}: 
                 <a href={"https://github.com/UtilitechAS/amsreader-firmware/releases/tag/" + sysinfo.upgrade.n} class="ml-2 text-blue-600 hover:text-blue-800" target='_blank' rel="noreferrer">{sysinfo.upgrade.n}</a>
                 {#if (sysinfo.security == 0 || data.a) && sysinfo.fwconsent === 1 && sysinfo.upgrade.n && sysinfo.upgrade.n != sysinfo.version}
-                <div class="flex-none ml-2 text-green-500" title={translations.status?.firmware?.install ?? "Install"}>
-                    <button on:click={askUpgrade}>&#8659;</button>
-                </div>
+                    <div class="flex-none ml-2 text-green-500" title={translations.status?.firmware?.install ?? "Install"}>
+                        <button on:click={askUpgrade}>&#8659;</button>
+                    </div>
                 {/if}
             </div>
             {#if sysinfo.fwconsent === 2}
@@ -237,22 +258,22 @@
             {/if}
         {/if}
         {#if (sysinfo.security == 0 || data.a) && isBusPowered(sysinfo.board) }
-        <div class="bd-red">
-            {upgradeWarningText(boardtype(sysinfo.chip, sysinfo.board))}
-        </div>
+            <div class="bd-red">
+                {upgradeWarningText(boardtype(sysinfo.chip, sysinfo.board))}
+            </div>
         {/if}
         {#if sysinfo.security == 0 || data.a}
-        <div class="my-2 flex">
-            <form action="firmware" enctype="multipart/form-data" method="post" on:submit={() => firmwareUploading=true} autocomplete="off">
-                <input style="display:none" name="file" type="file" accept=".bin" bind:this={firmwareFileInput} bind:files={firmwareFiles}>
-                {#if firmwareFiles.length == 0}
-                <button type="button" on:click={()=>{firmwareFileInput.click();}} class="btn-pri-sm float-right">{translations.status?.firmware?.btn_select_file ?? "Select file"}</button>
-                {:else}
-                {firmwareFiles[0].name}
-                <button type="submit" class="btn-pri-sm float-right ml-2">{translations.btn?.upload ?? "Upload"}</button>
-                {/if}
-            </form>
-        </div>
+            <div class="my-2 flex">
+                <form action="firmware" enctype="multipart/form-data" method="post" on:submit={() => firmwareUploading=true} autocomplete="off">
+                    <input style="display:none" name="file" type="file" accept=".bin" bind:this={firmwareFileInput} bind:files={firmwareFiles}>
+                    {#if firmwareFiles.length == 0}
+                    <button type="button" on:click={()=>{firmwareFileInput.click();}} class="btn-pri-sm float-right">{translations.status?.firmware?.btn_select_file ?? "Select file"}</button>
+                    {:else}
+                    {firmwareFiles[0].name}
+                    <button type="submit" class="btn-pri-sm float-right ml-2">{translations.btn?.upload ?? "Upload"}</button>
+                    {/if}
+                </form>
+            </div>
         {/if}
     </div>
     {#if sysinfo.security == 0 || data.a}

lib/SvelteUi/include/AmsWebServer.h

@@ -146,6 +146,7 @@ private:
 	void firmwarePost();
 	void firmwareUpload();
 	void isAliveCheck();
+	void fwchannel();
 
 	void mqttCaUpload();
 	void mqttCaDelete();

lib/SvelteUi/json/sysinfo.json

@@ -53,6 +53,7 @@
     "boot_reason": %d,
     "ex_cause": %d,
     "upgrade": {
+        "c": %d,
         "e": %d,
         "f": "%s",
         "t": "%s",

lib/SvelteUi/src/AmsWebServer.cpp

@@ -145,6 +145,7 @@ void AmsWebServer::setup(AmsConfiguration* config, GpioConfig* gpioConfig, AmsDa
 	server.on(context + F("/firmware"), HTTP_GET, std::bind(&AmsWebServer::firmwareHtml, this));
 	server.on(context + F("/firmware"), HTTP_POST, std::bind(&AmsWebServer::firmwarePost, this), std::bind(&AmsWebServer::firmwareUpload, this));
 	server.on(context + F("/is-alive"), HTTP_GET, std::bind(&AmsWebServer::isAliveCheck, this));
+	server.on(context + F("/fwchannel"), HTTP_POST, std::bind(&AmsWebServer::fwchannel, this));
 
 	server.on(context + F("/reset"), HTTP_POST, std::bind(&AmsWebServer::factoryResetPost, this));
 
@@ -479,6 +480,7 @@ void AmsWebServer::sysinfoJson() {
 		ESP.getResetInfoPtr()->reason,
 		ESP.getResetInfoPtr()->exccause,
 		#endif
+		sys.firmwareChannel,
 		upinfo.errorCode,
 		upinfo.fromVersion,
 		upinfo.toVersion,
@@ -1341,6 +1343,7 @@ void AmsWebServer::handleSave() {
 			
 			sys.userConfigured = success;
 			sys.dataCollectionConsent = 0;
+			sys.firmwareChannel = 0;
 			config->setSystemConfig(sys);
 
 			performRestart = true;
@@ -2022,6 +2025,17 @@ void AmsWebServer::isAliveCheck() {
 	server.send(200);
 }
 
+void AmsWebServer::fwchannel() {
+	if(!checkSecurity(1))
+		return;
+
+	SystemConfig sys;
+	config->getSystemConfig(sys);
+	sys.firmwareChannel = server.arg(F("channel")).toInt();
+	config->setSystemConfig(sys);
+	server.send(200);
+}
+
 void AmsWebServer::factoryResetPost() {
 	if(!checkSecurity(1))
 		return;

src/AmsToMqttBridge.cpp

@@ -1120,6 +1120,7 @@ void handleSystem(unsigned long now) {
 	if(config.isSystemConfigChanged()) {
 		config.getSystemConfig(sysConfig);
 		config.ackSystemConfigChanged();
+		updater.setFirmwareChannel(sysConfig.firmwareChannel);
 	}
 
 	unsigned long start, end;