diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index e8010ffc..f9c7116c 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -5,6 +5,11 @@ on: tags: - 'v*.*.*' +permissions: + contents: write + pages: write + id-token: write + jobs: build: @@ -211,3 +216,24 @@ jobs: asset_path: esp32c3.zip asset_name: ams2mqtt-esp32c3-${{ steps.release_tag.outputs.tag }}.zip asset_content_type: application/zip + + - name: Package firmware manifests + run: | + python scripts/package_firmware.py --output dist --channel stable --version ${{ steps.release_tag.outputs.tag }} + ls -R dist + + - name: Upload Pages artifact + uses: actions/upload-pages-artifact@v3 + with: + path: dist + + deploy: + needs: build + runs-on: ubuntu-latest + environment: + name: github-pages + url: ${{ steps.deploy.outputs.page_url }} + steps: + - name: Deploy to GitHub Pages + id: deploy + uses: actions/deploy-pages@v4 diff --git a/README.md b/README.md index 4fa8ae15..00b31e8f 100644 --- a/README.md +++ b/README.md @@ -8,6 +8,41 @@ Later development have added Energy usage graph for both day and month, as well Go to the [WiKi](https://github.com/UtilitechAS/amsreader-firmware/wiki) for information on how to get your own device! And find the latest prebuilt firmware file at the [release section](https://github.com/UtilitechAS/amsreader-firmware/releases). +## OTA updates from GitHub releases + +The firmware now supports downloading updates straight from GitHub Pages using a +lightweight manifest file. Each time you push a tag like `v1.2.3`, the +`.github/workflows/release.yml` pipeline will: + +1. Build every supported PlatformIO environment and publish `.bin`/`.zip` + assets on the release page (existing behaviour). +2. Run `scripts/package_firmware.py` to assemble a static structure under + `dist/firmware///` with the firmware binary, an MD5 checksum, + and a `manifest.json` pointing at the binary. +3. Deploy the contents of `dist/` to GitHub Pages, yielding public URLs such as + `https://.github.io/neas-amsreader-firmware-test/firmware/esp32s2/stable/manifest.json`. + +> ℹ️ Make sure GitHub Pages for the repository is configured to "GitHub +> Actions" under *Settings → Pages* the first time you run the workflow. + +To make the device follow those releases, set the default OTA endpoint before +flashing: + +```cpp +#define FIRMWARE_UPDATE_BASE_URL "https://.github.io/neas-amsreader-firmware-test" +#define FIRMWARE_UPDATE_CHANNEL "stable" +``` + +You can override the defaults either by editing +`lib/AmsFirmwareUpdater/include/UpgradeDefaults.h` or by adding corresponding +`-D` flags in `platformio.ini`. When a release is available, the device fetches +`manifest.json`, compares the `version` against its current firmware, and then +downloads the referenced binary in chunks. + +If you need parallel release tracks (for example `beta` versus `stable`), pass +`--channel beta` to `package_firmware.py` inside your automation and override +`FIRMWARE_UPDATE_CHANNEL` for the devices you want on that track. + ## Building this project with PlatformIO To build this project, you need [PlatformIO](https://platformio.org/) installed. diff --git a/lib/AmsFirmwareUpdater/include/AmsFirmwareUpdater.h b/lib/AmsFirmwareUpdater/include/AmsFirmwareUpdater.h index 5b4c8e90..9c327a50 100644 --- a/lib/AmsFirmwareUpdater/include/AmsFirmwareUpdater.h +++ b/lib/AmsFirmwareUpdater/include/AmsFirmwareUpdater.h @@ -4,6 +4,7 @@ #include "HwTools.h" #include "AmsData.h" #include "AmsConfiguration.h" +#include "UpgradeDefaults.h" #if defined(ESP32) #include "esp_flash_partitions.h" @@ -100,6 +101,20 @@ private: char nextVersion[10]; +#if FIRMWARE_UPDATE_USE_MANIFEST + struct FirmwareManifestInfo { + bool loaded = false; + String version; + uint32_t size = 0; + String downloadUrl; + String md5; + unsigned long fetchedAt = 0; + } manifestInfo; + + bool loadManifest(bool force = false); +#endif + + bool fetchNextVersion(); bool fetchVersionDetails(); bool fetchFirmwareChunk(HTTPClient& http); diff --git a/lib/AmsFirmwareUpdater/include/UpgradeDefaults.h b/lib/AmsFirmwareUpdater/include/UpgradeDefaults.h index c86eddec..8c3aca3b 100644 --- a/lib/AmsFirmwareUpdater/include/UpgradeDefaults.h +++ b/lib/AmsFirmwareUpdater/include/UpgradeDefaults.h @@ -8,7 +8,7 @@ // ---------------------------------------------------------------------------- #ifndef FIRMWARE_UPDATE_BASE_URL -#define FIRMWARE_UPDATE_BASE_URL "http://firmware.neas.no" +#define FIRMWARE_UPDATE_BASE_URL "https://github.com/EivindH06/neas-amsreader-firmware-test" #endif #ifndef FIRMWARE_UPDATE_CHANNEL @@ -18,3 +18,11 @@ #ifndef FIRMWARE_UPDATE_USER_AGENT #define FIRMWARE_UPDATE_USER_AGENT "NEAS-Firmware-Updater" #endif + +#ifndef FIRMWARE_UPDATE_USE_MANIFEST +#define FIRMWARE_UPDATE_USE_MANIFEST 1 +#endif + +#ifndef FIRMWARE_UPDATE_MANIFEST_NAME +#define FIRMWARE_UPDATE_MANIFEST_NAME "manifest.json" +#endif diff --git a/lib/AmsFirmwareUpdater/src/AmsFirmwareUpdater.cpp b/lib/AmsFirmwareUpdater/src/AmsFirmwareUpdater.cpp index 98549ade..26afdcc8 100644 --- a/lib/AmsFirmwareUpdater/src/AmsFirmwareUpdater.cpp +++ b/lib/AmsFirmwareUpdater/src/AmsFirmwareUpdater.cpp @@ -2,6 +2,7 @@ #include "AmsStorage.h" #include "FirmwareVersion.h" #include "UpgradeDefaults.h" +#include #if defined(ESP32) #include "esp_ota_ops.h" @@ -210,6 +211,23 @@ void AmsFirmwareUpdater::loop() { } bool AmsFirmwareUpdater::fetchNextVersion() { +#if FIRMWARE_UPDATE_USE_MANIFEST + if(!loadManifest(true)) { + return false; + } + if(manifestInfo.version.isEmpty()) { + return false; + } + + strncpy(nextVersion, manifestInfo.version.c_str(), sizeof(nextVersion) - 1); + nextVersion[sizeof(nextVersion) - 1] = '\0'; + + if(autoUpgrade && strcmp(updateStatus.toVersion, nextVersion) != 0) { + strcpy(updateStatus.toVersion, nextVersion); + updateStatus.size = 0; + } + return strlen(nextVersion) > 0; +#else HTTPClient http; const char * headerkeys[] = { "x-version" }; http.collectHeaders(headerkeys, 1); @@ -247,9 +265,26 @@ bool AmsFirmwareUpdater::fetchNextVersion() { http.end(); } return false; +#endif } bool AmsFirmwareUpdater::fetchVersionDetails() { +#if FIRMWARE_UPDATE_USE_MANIFEST + if(!loadManifest(false)) { + return false; + } + if(manifestInfo.version.isEmpty() || manifestInfo.size == 0) { + return false; + } + + updateStatus.size = manifestInfo.size; + if(manifestInfo.md5.length() > 0) { + md5 = manifestInfo.md5; + } else { + md5 = F("unknown"); + } + return true; +#else HTTPClient http; const char * headerkeys[] = { "x-size" }; http.collectHeaders(headerkeys, 1); @@ -299,9 +334,49 @@ bool AmsFirmwareUpdater::fetchVersionDetails() { http.end(); } return false; +#endif } bool AmsFirmwareUpdater::fetchFirmwareChunk(HTTPClient& http) { +#if FIRMWARE_UPDATE_USE_MANIFEST + if(!loadManifest(false)) { + return false; + } + if(manifestInfo.downloadUrl.isEmpty()) { + return false; + } + + uint32_t start = updateStatus.block_position * UPDATE_BUF_SIZE; + uint32_t end = start + (UPDATE_BUF_SIZE * 1); + char range[24]; + snprintf_P(range, 24, PSTR("bytes=%lu-%lu"), start, end); + + const char* url = manifestInfo.downloadUrl.c_str(); + +#if defined(ESP8266) + WiFiClient client; + client.setTimeout(5000); + if(http.begin(client, url)) { +#elif defined(ESP32) + if(http.begin(url)) { +#endif + http.useHTTP10(true); + http.setTimeout(30000); + http.setFollowRedirects(HTTPC_FORCE_FOLLOW_REDIRECTS); + http.setUserAgent(FIRMWARE_UPDATE_USER_AGENT); + http.addHeader(F("Cache-Control"), "no-cache"); + http.addHeader(F("x-AMS-version"), FirmwareVersion::VersionString); + http.addHeader(F("Range"), range); + int status = http.GET(); + if(status == HTTP_CODE_PARTIAL_CONTENT || status == HTTP_CODE_OK) { + if(md5.equals(F("unknown")) && manifestInfo.md5.length() > 0) { + md5 = manifestInfo.md5; + } + return true; + } + } + return false; +#else const char * headerkeys[] = { "x-MD5" }; http.collectHeaders(headerkeys, 1); @@ -334,8 +409,91 @@ bool AmsFirmwareUpdater::fetchFirmwareChunk(HTTPClient& http) { } } return false; +#endif } +#if FIRMWARE_UPDATE_USE_MANIFEST +bool AmsFirmwareUpdater::loadManifest(bool force) { + if(manifestInfo.loaded && !force) { + return true; + } + + HTTPClient http; + char url[256]; + snprintf(url, sizeof(url), "%s/firmware/%s/%s/%s", FIRMWARE_UPDATE_BASE_URL, chipType, FIRMWARE_UPDATE_CHANNEL, FIRMWARE_UPDATE_MANIFEST_NAME); + +#if defined(ESP8266) + WiFiClient client; + client.setTimeout(5000); + if(!http.begin(client, url)) { + return manifestInfo.loaded; + } +#elif defined(ESP32) + if(!http.begin(url)) { + return manifestInfo.loaded; + } +#endif + + http.useHTTP10(true); + http.setTimeout(30000); + http.setFollowRedirects(HTTPC_FORCE_FOLLOW_REDIRECTS); + http.setUserAgent(FIRMWARE_UPDATE_USER_AGENT); + http.addHeader(F("Cache-Control"), "no-cache"); + + bool success = false; + int status = http.GET(); + if(status == HTTP_CODE_OK) { + WiFiClient* stream = http.getStreamPtr(); + DynamicJsonDocument doc(2048); + DeserializationError err = deserializeJson(doc, *stream); + if(!err) { + String version = doc["version"].as(); + String download = doc["url"].as(); + if(download.length() == 0 && doc["download_url"].is()) { + download = doc["download_url"].as(); + } + uint32_t size = doc["size"] | 0; + String checksum = doc["md5"].as(); + + if(version.length() > 0 && download.length() > 0 && size > 0) { + String manifestUrl = url; + int lastSlash = manifestUrl.lastIndexOf('/'); + if(lastSlash >= 0) { + manifestUrl = manifestUrl.substring(0, lastSlash + 1); + } + + if(download.startsWith("http://") || download.startsWith("https://")) { + manifestInfo.downloadUrl = download; + } else { + manifestInfo.downloadUrl = manifestUrl + download; + } + + manifestInfo.version = version; + manifestInfo.size = size; + manifestInfo.md5 = checksum; + manifestInfo.loaded = true; + manifestInfo.fetchedAt = millis(); + success = true; + } + } else { +#if defined(AMS_REMOTE_DEBUG) + if (debugger->isActive(RemoteDebug::ERROR)) +#endif + debugger->printf_P(PSTR("Manifest parse error: %s\n"), err.c_str()); + } + } + + http.end(); + if(!success && !manifestInfo.loaded) { +#if defined(AMS_REMOTE_DEBUG) + if (debugger->isActive(RemoteDebug::WARNING)) +#endif + debugger->printf_P(PSTR("Unable to fetch manifest from %s (HTTP %d)\n"), url, status); + } + return manifestInfo.loaded; +} +#endif + bool AmsFirmwareUpdater::writeUpdateStatus() { if(updateStatus.block_position - lastSaveBlocksWritten > 32) { updateStatusChanged = true; diff --git a/scripts/package_firmware.py b/scripts/package_firmware.py new file mode 100644 index 00000000..85568ce0 --- /dev/null +++ b/scripts/package_firmware.py @@ -0,0 +1,219 @@ +#!/usr/bin/env python3 +""" +Package compiled PlatformIO firmware artifacts into a static layout suitable for +publishing on GitHub Pages (or any static web host). For each target environment +we copy the generated firmware binary, compute its checksum, and emit a manifest +JSON document that the device firmware can consume when looking for OTA updates. + +Example structure produced under the output directory: + + dist/ + firmware/ + esp32s2/ + stable/ + firmware-abcdef1.bin + manifest.json + +The manifest contains version, size, checksum, and download path information. +""" + +from __future__ import annotations + +import argparse +import hashlib +import json +import os +from pathlib import Path +from datetime import datetime, timezone +from typing import Dict, Iterable, Optional + +DEFAULT_CHANNEL = "stable" +DEFAULT_OUTPUT = Path("dist") + +# Mapping from PlatformIO environment names to chip identifiers used by the OTA +# client. Update this map if new environments are added to platformio.ini. +ENV_TO_CHIP: Dict[str, str] = { + "esp8266": "esp8266", + "esp32": "esp32", + "esp32s2": "esp32s2", + "esp32s3": "esp32s3", + "esp32c3": "esp32c3", + "esp32solo": "esp32solo", +} + + +def parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser(description="Package Firmware Artifacts") + parser.add_argument( + "--env", + dest="envs", + action="append", + help="PlatformIO environment(s) to package (defaults to all known)", + ) + parser.add_argument( + "--channel", + default=DEFAULT_CHANNEL, + help="Firmware update channel name (default: stable)", + ) + parser.add_argument( + "--output", + type=Path, + default=DEFAULT_OUTPUT, + help="Destination directory for packaged artifacts (default: dist)", + ) + parser.add_argument( + "--build-dir", + type=Path, + default=Path(".pio") / "build", + help="Base directory containing PlatformIO build outputs", + ) + parser.add_argument( + "--version", + type=str, + default=None, + help="Firmware version string (falls back to generated_version.h)", + ) + parser.add_argument( + "--published-at", + type=str, + default=None, + help="ISO timestamp for the manifest (default: current UTC time)", + ) + return parser.parse_args() + + +def read_version(version_override: Optional[str]) -> str: + if version_override: + return version_override.strip() + + header_path = Path("lib/FirmwareVersion/src/generated_version.h") + if not header_path.exists(): + raise FileNotFoundError( + f"generated_version.h not found at {header_path}. Run PlatformIO build first." + ) + + for line in header_path.read_text().splitlines(): + if "VERSION_STRING" in line: + parts = line.split('"') + if len(parts) >= 2: + return parts[1] + raise RuntimeError("Unable to extract VERSION_STRING from generated header") + + +def compute_md5(path: Path) -> str: + hash_md5 = hashlib.md5() + with path.open("rb") as fh: + for chunk in iter(lambda: fh.read(65536), b""): + hash_md5.update(chunk) + return hash_md5.hexdigest() + + +def package_environment( + env: str, + chip: str, + build_dir: Path, + channel: str, + version: str, + output_dir: Path, + published_at: str, +) -> Optional[Dict[str, str]]: + firmware_path = build_dir / env / "firmware.bin" + if not firmware_path.exists(): + print(f"WARN: Skipping {env}, firmware not found at {firmware_path}") + return None + + md5_digest = compute_md5(firmware_path) + size_bytes = firmware_path.stat().st_size + + channel_dir = output_dir / "firmware" / chip / channel + channel_dir.mkdir(parents=True, exist_ok=True) + + binary_name = f"{chip}-{version}.bin" + binary_dest = channel_dir / binary_name + # Copy firmware binary (overwrite if it already exists) + binary_dest.write_bytes(firmware_path.read_bytes()) + + manifest_path = channel_dir / "manifest.json" + manifest = { + "version": version, + "channel": channel, + "chip": chip, + "size": size_bytes, + "md5": md5_digest, + "url": binary_name, + "published_at": published_at, + "env": env, + } + manifest_path.write_text(json.dumps(manifest, indent=2)) + + # Optional: bundle flashing zip if produced by scripts/mkzip.sh + zip_source = Path(f"{env}.zip") + zip_dest = None + if zip_source.exists(): + releases_dir = output_dir / "releases" + releases_dir.mkdir(parents=True, exist_ok=True) + zip_dest = releases_dir / f"{chip}-{version}.zip" + zip_dest.write_bytes(zip_source.read_bytes()) + + return { + "env": env, + "chip": chip, + "channel": channel, + "manifest": manifest_path.relative_to(output_dir).as_posix(), + "binary": binary_dest.relative_to(output_dir).as_posix(), + "zip": zip_dest.relative_to(output_dir).as_posix() if zip_dest else None, + "md5": md5_digest, + "size": size_bytes, + } + + +def write_index(output_dir: Path, summary: Iterable[Dict[str, str]], published_at: str, version: str) -> None: + summary_list = [item for item in summary if item] + if not summary_list: + return + + index_path = output_dir / "firmware" / "index.json" + index_path.parent.mkdir(parents=True, exist_ok=True) + index = { + "generated_at": published_at, + "version": version, + "artifacts": summary_list, + } + index_path.write_text(json.dumps(index, indent=2)) + + +def main() -> None: + args = parse_args() + envs = args.envs if args.envs else list(ENV_TO_CHIP.keys()) + + unknown_envs = [env for env in envs if env not in ENV_TO_CHIP] + if unknown_envs: + raise ValueError(f"Unknown PlatformIO environment(s): {', '.join(unknown_envs)}") + + version = read_version(args.version) + published_at = ( + args.published_at + if args.published_at + else datetime.now(timezone.utc).isoformat(timespec="seconds") + ) + + summaries = [] + for env in envs: + chip = ENV_TO_CHIP[env] + summary = package_environment( + env=env, + chip=chip, + build_dir=args.build_dir, + channel=args.channel, + version=version, + output_dir=args.output, + published_at=published_at, + ) + if summary: + summaries.append(summary) + + write_index(args.output, summaries, published_at, version) + + +if __name__ == "__main__": + main()