Testing OTA updates from github repo

.github/workflows/release.yml

@@ -5,6 +5,11 @@ on:
     tags:
       - 'v*.*.*'
 
+permissions:
+  contents: write
+  pages: write
+  id-token: write
+
 
 jobs:
   build:
@@ -211,3 +216,24 @@ jobs:
         asset_path: esp32c3.zip
         asset_name: ams2mqtt-esp32c3-${{ steps.release_tag.outputs.tag }}.zip
         asset_content_type: application/zip
+
+    - name: Package firmware manifests
+      run: |
+        python scripts/package_firmware.py --output dist --channel stable --version ${{ steps.release_tag.outputs.tag }}
+        ls -R dist
+
+    - name: Upload Pages artifact
+      uses: actions/upload-pages-artifact@v3
+      with:
+        path: dist
+
+  deploy:
+    needs: build
+    runs-on: ubuntu-latest
+    environment:
+      name: github-pages
+      url: ${{ steps.deploy.outputs.page_url }}
+    steps:
+      - name: Deploy to GitHub Pages
+        id: deploy
+        uses: actions/deploy-pages@v4

README.md

@@ -8,6 +8,41 @@ Later development have added Energy usage graph for both day and month, as well
 
 Go to the [WiKi](https://github.com/UtilitechAS/amsreader-firmware/wiki) for information on how to get your own device! And find the latest prebuilt firmware file at the [release section](https://github.com/UtilitechAS/amsreader-firmware/releases).
 
+## OTA updates from GitHub releases
+
+The firmware now supports downloading updates straight from GitHub Pages using a
+lightweight manifest file. Each time you push a tag like `v1.2.3`, the
+`.github/workflows/release.yml` pipeline will:
+
+1. Build every supported PlatformIO environment and publish `.bin`/`.zip`
+	assets on the release page (existing behaviour).
+2. Run `scripts/package_firmware.py` to assemble a static structure under
+	`dist/firmware/<chip>/<channel>/` with the firmware binary, an MD5 checksum,
+	and a `manifest.json` pointing at the binary.
+3. Deploy the contents of `dist/` to GitHub Pages, yielding public URLs such as
+	`https://<your-user>.github.io/neas-amsreader-firmware-test/firmware/esp32s2/stable/manifest.json`.
+
+> ℹ️ Make sure GitHub Pages for the repository is configured to "GitHub
+> Actions" under *Settings → Pages* the first time you run the workflow.
+
+To make the device follow those releases, set the default OTA endpoint before
+flashing:
+
+```cpp
+#define FIRMWARE_UPDATE_BASE_URL "https://<your-user>.github.io/neas-amsreader-firmware-test"
+#define FIRMWARE_UPDATE_CHANNEL  "stable"
+```
+
+You can override the defaults either by editing
+`lib/AmsFirmwareUpdater/include/UpgradeDefaults.h` or by adding corresponding
+`-D` flags in `platformio.ini`. When a release is available, the device fetches
+`manifest.json`, compares the `version` against its current firmware, and then
+downloads the referenced binary in chunks.
+
+If you need parallel release tracks (for example `beta` versus `stable`), pass
+`--channel beta` to `package_firmware.py` inside your automation and override
+`FIRMWARE_UPDATE_CHANNEL` for the devices you want on that track.
+
 ## Building this project with PlatformIO
 To build this project, you need [PlatformIO](https://platformio.org/) installed.
 

lib/AmsFirmwareUpdater/include/AmsFirmwareUpdater.h

@@ -4,6 +4,7 @@
 #include "HwTools.h"
 #include "AmsData.h"
 #include "AmsConfiguration.h"
+#include "UpgradeDefaults.h"
 
 #if defined(ESP32)
     #include "esp_flash_partitions.h"
@@ -100,6 +101,20 @@ private:
     char nextVersion[10];
 
 
+#if FIRMWARE_UPDATE_USE_MANIFEST
+    struct FirmwareManifestInfo {
+        bool loaded = false;
+        String version;
+        uint32_t size = 0;
+        String downloadUrl;
+        String md5;
+        unsigned long fetchedAt = 0;
+    } manifestInfo;
+
+    bool loadManifest(bool force = false);
+#endif
+
+
     bool fetchNextVersion();
     bool fetchVersionDetails();
     bool fetchFirmwareChunk(HTTPClient& http);

lib/AmsFirmwareUpdater/include/UpgradeDefaults.h

@@ -8,7 +8,7 @@
 // ----------------------------------------------------------------------------
 
 #ifndef FIRMWARE_UPDATE_BASE_URL
-#define FIRMWARE_UPDATE_BASE_URL "http://firmware.neas.no"
+#define FIRMWARE_UPDATE_BASE_URL "https://github.com/EivindH06/neas-amsreader-firmware-test"
 #endif
 
 #ifndef FIRMWARE_UPDATE_CHANNEL
@@ -18,3 +18,11 @@
 #ifndef FIRMWARE_UPDATE_USER_AGENT
 #define FIRMWARE_UPDATE_USER_AGENT "NEAS-Firmware-Updater"
 #endif
+
+#ifndef FIRMWARE_UPDATE_USE_MANIFEST
+#define FIRMWARE_UPDATE_USE_MANIFEST 1
+#endif
+
+#ifndef FIRMWARE_UPDATE_MANIFEST_NAME
+#define FIRMWARE_UPDATE_MANIFEST_NAME "manifest.json"
+#endif

lib/AmsFirmwareUpdater/src/AmsFirmwareUpdater.cpp

@@ -2,6 +2,7 @@
 #include "AmsStorage.h"
 #include "FirmwareVersion.h"
 #include "UpgradeDefaults.h"
+#include <ArduinoJson.h>
 
 #if defined(ESP32)
 #include "esp_ota_ops.h"
@@ -210,6 +211,23 @@ void AmsFirmwareUpdater::loop() {
 }
 
 bool AmsFirmwareUpdater::fetchNextVersion() {
+#if FIRMWARE_UPDATE_USE_MANIFEST
+    if(!loadManifest(true)) {
+        return false;
+    }
+    if(manifestInfo.version.isEmpty()) {
+        return false;
+    }
+
+    strncpy(nextVersion, manifestInfo.version.c_str(), sizeof(nextVersion) - 1);
+    nextVersion[sizeof(nextVersion) - 1] = '\0';
+
+    if(autoUpgrade && strcmp(updateStatus.toVersion, nextVersion) != 0) {
+        strcpy(updateStatus.toVersion, nextVersion);
+        updateStatus.size = 0;
+    }
+    return strlen(nextVersion) > 0;
+#else
     HTTPClient http;
     const char * headerkeys[] = { "x-version" };
     http.collectHeaders(headerkeys, 1);
@@ -247,9 +265,26 @@ bool AmsFirmwareUpdater::fetchNextVersion() {
         http.end();
     }
     return false;
+#endif
 }
 
 bool AmsFirmwareUpdater::fetchVersionDetails() {
+#if FIRMWARE_UPDATE_USE_MANIFEST
+    if(!loadManifest(false)) {
+        return false;
+    }
+    if(manifestInfo.version.isEmpty() || manifestInfo.size == 0) {
+        return false;
+    }
+
+    updateStatus.size = manifestInfo.size;
+    if(manifestInfo.md5.length() > 0) {
+        md5 = manifestInfo.md5;
+    } else {
+        md5 = F("unknown");
+    }
+    return true;
+#else
     HTTPClient http;
     const char * headerkeys[] = { "x-size" };
     http.collectHeaders(headerkeys, 1);
@@ -299,9 +334,49 @@ bool AmsFirmwareUpdater::fetchVersionDetails() {
         http.end();
     }
     return false;
+#endif
 }
 
 bool AmsFirmwareUpdater::fetchFirmwareChunk(HTTPClient& http) {
+#if FIRMWARE_UPDATE_USE_MANIFEST
+    if(!loadManifest(false)) {
+        return false;
+    }
+    if(manifestInfo.downloadUrl.isEmpty()) {
+        return false;
+    }
+
+    uint32_t start = updateStatus.block_position * UPDATE_BUF_SIZE;
+    uint32_t end = start + (UPDATE_BUF_SIZE * 1);
+    char range[24];
+    snprintf_P(range, 24, PSTR("bytes=%lu-%lu"), start, end);
+
+    const char* url = manifestInfo.downloadUrl.c_str();
+
+#if defined(ESP8266)
+    WiFiClient client;
+    client.setTimeout(5000);
+    if(http.begin(client, url)) {
+#elif defined(ESP32)
+    if(http.begin(url)) {
+#endif
+        http.useHTTP10(true);
+        http.setTimeout(30000);
+        http.setFollowRedirects(HTTPC_FORCE_FOLLOW_REDIRECTS);
+        http.setUserAgent(FIRMWARE_UPDATE_USER_AGENT);
+        http.addHeader(F("Cache-Control"), "no-cache");
+        http.addHeader(F("x-AMS-version"), FirmwareVersion::VersionString);
+        http.addHeader(F("Range"), range);
+        int status = http.GET();
+        if(status == HTTP_CODE_PARTIAL_CONTENT || status == HTTP_CODE_OK) {
+            if(md5.equals(F("unknown")) && manifestInfo.md5.length() > 0) {
+                md5 = manifestInfo.md5;
+            }
+            return true;
+        }
+    }
+    return false;
+#else
     const char * headerkeys[] = { "x-MD5" };
     http.collectHeaders(headerkeys, 1);
 
@@ -334,8 +409,91 @@ bool AmsFirmwareUpdater::fetchFirmwareChunk(HTTPClient& http) {
         }
     }
     return false;
+#endif
 }
 
+#if FIRMWARE_UPDATE_USE_MANIFEST
+bool AmsFirmwareUpdater::loadManifest(bool force) {
+    if(manifestInfo.loaded && !force) {
+        return true;
+    }
+
+    HTTPClient http;
+    char url[256];
+    snprintf(url, sizeof(url), "%s/firmware/%s/%s/%s", FIRMWARE_UPDATE_BASE_URL, chipType, FIRMWARE_UPDATE_CHANNEL, FIRMWARE_UPDATE_MANIFEST_NAME);
+
+#if defined(ESP8266)
+    WiFiClient client;
+    client.setTimeout(5000);
+    if(!http.begin(client, url)) {
+        return manifestInfo.loaded;
+    }
+#elif defined(ESP32)
+    if(!http.begin(url)) {
+        return manifestInfo.loaded;
+    }
+#endif
+
+    http.useHTTP10(true);
+    http.setTimeout(30000);
+    http.setFollowRedirects(HTTPC_FORCE_FOLLOW_REDIRECTS);
+    http.setUserAgent(FIRMWARE_UPDATE_USER_AGENT);
+    http.addHeader(F("Cache-Control"), "no-cache");
+
+    bool success = false;
+    int status = http.GET();
+    if(status == HTTP_CODE_OK) {
+        WiFiClient* stream = http.getStreamPtr();
+        DynamicJsonDocument doc(2048);
+        DeserializationError err = deserializeJson(doc, *stream);
+        if(!err) {
+            String version = doc["version"].as<String>();
+            String download = doc["url"].as<String>();
+            if(download.length() == 0 && doc["download_url"].is<String>()) {
+                download = doc["download_url"].as<String>();
+            }
+            uint32_t size = doc["size"] | 0;
+            String checksum = doc["md5"].as<String>();
+
+            if(version.length() > 0 && download.length() > 0 && size > 0) {
+                String manifestUrl = url;
+                int lastSlash = manifestUrl.lastIndexOf('/');
+                if(lastSlash >= 0) {
+                    manifestUrl = manifestUrl.substring(0, lastSlash + 1);
+                }
+
+                if(download.startsWith("http://") || download.startsWith("https://")) {
+                    manifestInfo.downloadUrl = download;
+                } else {
+                    manifestInfo.downloadUrl = manifestUrl + download;
+                }
+
+                manifestInfo.version = version;
+                manifestInfo.size = size;
+                manifestInfo.md5 = checksum;
+                manifestInfo.loaded = true;
+                manifestInfo.fetchedAt = millis();
+                success = true;
+            }
+        } else {
+#if defined(AMS_REMOTE_DEBUG)
+            if (debugger->isActive(RemoteDebug::ERROR))
+#endif
+            debugger->printf_P(PSTR("Manifest parse error: %s\n"), err.c_str());
+        }
+    }
+
+    http.end();
+    if(!success && !manifestInfo.loaded) {
+#if defined(AMS_REMOTE_DEBUG)
+        if (debugger->isActive(RemoteDebug::WARNING))
+#endif
+        debugger->printf_P(PSTR("Unable to fetch manifest from %s (HTTP %d)\n"), url, status);
+    }
+    return manifestInfo.loaded;
+}
+#endif
+
 bool AmsFirmwareUpdater::writeUpdateStatus() {
     if(updateStatus.block_position - lastSaveBlocksWritten > 32) {
         updateStatusChanged = true;

scripts/package_firmware.py

@@ -0,0 +1,219 @@
+#!/usr/bin/env python3
+"""
+Package compiled PlatformIO firmware artifacts into a static layout suitable for
+publishing on GitHub Pages (or any static web host). For each target environment
+we copy the generated firmware binary, compute its checksum, and emit a manifest
+JSON document that the device firmware can consume when looking for OTA updates.
+
+Example structure produced under the output directory:
+
+    dist/
+      firmware/
+        esp32s2/
+          stable/
+            firmware-abcdef1.bin
+            manifest.json
+
+The manifest contains version, size, checksum, and download path information.
+"""
+
+from __future__ import annotations
+
+import argparse
+import hashlib
+import json
+import os
+from pathlib import Path
+from datetime import datetime, timezone
+from typing import Dict, Iterable, Optional
+
+DEFAULT_CHANNEL = "stable"
+DEFAULT_OUTPUT = Path("dist")
+
+# Mapping from PlatformIO environment names to chip identifiers used by the OTA
+# client. Update this map if new environments are added to platformio.ini.
+ENV_TO_CHIP: Dict[str, str] = {
+    "esp8266": "esp8266",
+    "esp32": "esp32",
+    "esp32s2": "esp32s2",
+    "esp32s3": "esp32s3",
+    "esp32c3": "esp32c3",
+    "esp32solo": "esp32solo",
+}
+
+
+def parse_args() -> argparse.Namespace:
+    parser = argparse.ArgumentParser(description="Package Firmware Artifacts")
+    parser.add_argument(
+        "--env",
+        dest="envs",
+        action="append",
+        help="PlatformIO environment(s) to package (defaults to all known)",
+    )
+    parser.add_argument(
+        "--channel",
+        default=DEFAULT_CHANNEL,
+        help="Firmware update channel name (default: stable)",
+    )
+    parser.add_argument(
+        "--output",
+        type=Path,
+        default=DEFAULT_OUTPUT,
+        help="Destination directory for packaged artifacts (default: dist)",
+    )
+    parser.add_argument(
+        "--build-dir",
+        type=Path,
+        default=Path(".pio") / "build",
+        help="Base directory containing PlatformIO build outputs",
+    )
+    parser.add_argument(
+        "--version",
+        type=str,
+        default=None,
+        help="Firmware version string (falls back to generated_version.h)",
+    )
+    parser.add_argument(
+        "--published-at",
+        type=str,
+        default=None,
+        help="ISO timestamp for the manifest (default: current UTC time)",
+    )
+    return parser.parse_args()
+
+
+def read_version(version_override: Optional[str]) -> str:
+    if version_override:
+        return version_override.strip()
+
+    header_path = Path("lib/FirmwareVersion/src/generated_version.h")
+    if not header_path.exists():
+        raise FileNotFoundError(
+            f"generated_version.h not found at {header_path}. Run PlatformIO build first."
+        )
+
+    for line in header_path.read_text().splitlines():
+        if "VERSION_STRING" in line:
+            parts = line.split('"')
+            if len(parts) >= 2:
+                return parts[1]
+    raise RuntimeError("Unable to extract VERSION_STRING from generated header")
+
+
+def compute_md5(path: Path) -> str:
+    hash_md5 = hashlib.md5()
+    with path.open("rb") as fh:
+        for chunk in iter(lambda: fh.read(65536), b""):
+            hash_md5.update(chunk)
+    return hash_md5.hexdigest()
+
+
+def package_environment(
+    env: str,
+    chip: str,
+    build_dir: Path,
+    channel: str,
+    version: str,
+    output_dir: Path,
+    published_at: str,
+) -> Optional[Dict[str, str]]:
+    firmware_path = build_dir / env / "firmware.bin"
+    if not firmware_path.exists():
+        print(f"WARN: Skipping {env}, firmware not found at {firmware_path}")
+        return None
+
+    md5_digest = compute_md5(firmware_path)
+    size_bytes = firmware_path.stat().st_size
+
+    channel_dir = output_dir / "firmware" / chip / channel
+    channel_dir.mkdir(parents=True, exist_ok=True)
+
+    binary_name = f"{chip}-{version}.bin"
+    binary_dest = channel_dir / binary_name
+    # Copy firmware binary (overwrite if it already exists)
+    binary_dest.write_bytes(firmware_path.read_bytes())
+
+    manifest_path = channel_dir / "manifest.json"
+    manifest = {
+        "version": version,
+        "channel": channel,
+        "chip": chip,
+        "size": size_bytes,
+        "md5": md5_digest,
+        "url": binary_name,
+        "published_at": published_at,
+        "env": env,
+    }
+    manifest_path.write_text(json.dumps(manifest, indent=2))
+
+    # Optional: bundle flashing zip if produced by scripts/mkzip.sh
+    zip_source = Path(f"{env}.zip")
+    zip_dest = None
+    if zip_source.exists():
+        releases_dir = output_dir / "releases"
+        releases_dir.mkdir(parents=True, exist_ok=True)
+        zip_dest = releases_dir / f"{chip}-{version}.zip"
+        zip_dest.write_bytes(zip_source.read_bytes())
+
+    return {
+        "env": env,
+        "chip": chip,
+        "channel": channel,
+        "manifest": manifest_path.relative_to(output_dir).as_posix(),
+        "binary": binary_dest.relative_to(output_dir).as_posix(),
+        "zip": zip_dest.relative_to(output_dir).as_posix() if zip_dest else None,
+        "md5": md5_digest,
+        "size": size_bytes,
+    }
+
+
+def write_index(output_dir: Path, summary: Iterable[Dict[str, str]], published_at: str, version: str) -> None:
+    summary_list = [item for item in summary if item]
+    if not summary_list:
+        return
+
+    index_path = output_dir / "firmware" / "index.json"
+    index_path.parent.mkdir(parents=True, exist_ok=True)
+    index = {
+        "generated_at": published_at,
+        "version": version,
+        "artifacts": summary_list,
+    }
+    index_path.write_text(json.dumps(index, indent=2))
+
+
+def main() -> None:
+    args = parse_args()
+    envs = args.envs if args.envs else list(ENV_TO_CHIP.keys())
+
+    unknown_envs = [env for env in envs if env not in ENV_TO_CHIP]
+    if unknown_envs:
+        raise ValueError(f"Unknown PlatformIO environment(s): {', '.join(unknown_envs)}")
+
+    version = read_version(args.version)
+    published_at = (
+        args.published_at
+        if args.published_at
+        else datetime.now(timezone.utc).isoformat(timespec="seconds")
+    )
+
+    summaries = []
+    for env in envs:
+        chip = ENV_TO_CHIP[env]
+        summary = package_environment(
+            env=env,
+            chip=chip,
+            build_dir=args.build_dir,
+            channel=args.channel,
+            version=version,
+            output_dir=args.output,
+            published_at=published_at,
+        )
+        if summary:
+            summaries.append(summary)
+
+    write_index(args.output, summaries, published_at, version)
+
+
+if __name__ == "__main__":
+    main()