From d9aea4dbac69147711f3f1fe73d37cbd7a48ef51 Mon Sep 17 00:00:00 2001 From: reubenmiller Date: Wed, 18 Nov 2026 12:15:17 +1000 Subject: [PATCH] feat: release tiny-edge's NuttX agent for the ESP32-C6, and take tiny-edge 0.7.0 Releases now carry tedge-agent's `nuttx` variant for the ESP32-C6: tiny-edge's NuttX agent (NuttX 13.1, booting through MCUboot), set up over USB with tedge-onboard, with the same assets as the other images: a factory image, the firmware update image, and a bundle scripts/flash.sh flashes. - release/devices.yml: `framework: nuttx` marks a variant built with tiny-edge's ports/nuttx/scripts/build.sh for the device's `nuttx_board`; the build's extra_conf are NuttX fragments. matrix.py validates it and gives it a matrix of its own (nuttx_matrix), whose group names the framework, so its artifact does not collide with the same app's Zephyr builds. _DIR (TINY_EDGE_DIR) names a west project where west is not installed. - scripts/release/build.sh builds it with the release firmware name and apps//VERSION; package_nuttx.py packages it from the build's NuttX configuration (slots, data partition, firmware name, device type thin-edge.io-nuttx, features). The update image may take at most 95 % of its slot (the agent takes 92 % of 1.5 MB). - The release workflow checks out the tiny-edge release west.yml pins and builds in its NuttX container. - scripts/flash.sh flashes a NuttX bundle; --erase-storage erases its LittleFS data partition. - DEVICES.md records the agent's runs on the board; README and the release notes describe the variant. - west.yml and tedge-agent-idf take tiny-edge 0.7.0 (the NuttX port; a download idle timeout and a format fix in the core). Co-Authored-By: Claude Opus 5.5 --- .github/workflows/release.yml | 60 ++++- DEVICES.md | 24 ++ README.md | 22 +- apps/tedge-agent-idf/README.md | 2 +- apps/tedge-agent-idf/main/idf_component.yml | 2 +- release/devices.yml | 15 ++ scripts/flash.sh | 12 +- scripts/release/build.sh | 46 +++- scripts/release/matrix.py | 50 +++- scripts/release/notes.py | 3 +- scripts/release/package_nuttx.py | 281 ++++++++++++++++++++ scripts/release/tests/test_matrix.py | 64 ++++- west.yml | 2 +- 13 files changed, 551 insertions(+), 32 deletions(-) create mode 100644 scripts/release/package_nuttx.py diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index fad7b9a..e5b1233 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -58,6 +58,7 @@ jobs: outputs: matrix: ${{ steps.matrix.outputs.matrix }} idf_matrix: ${{ steps.matrix.outputs.idf_matrix }} + nuttx_matrix: ${{ steps.matrix.outputs.nuttx_matrix }} version: ${{ steps.version.outputs.version }} steps: - uses: actions/checkout@v4 @@ -202,14 +203,67 @@ jobs: path: dist/ if-no-files-found: error + # tiny-edge's NuttX agent (`framework: nuttx` in release/devices.yml): the + # tiny-edge release west.yml pins, built in tiny-edge's own NuttX container + # with its ports/nuttx/scripts/build.sh, the same assets and metadata. + build-nuttx: + name: ${{ matrix.group }} + needs: matrix + if: ${{ fromJSON(needs.matrix.outputs.nuttx_matrix).include[0] != null }} + runs-on: ubuntu-24.04 + strategy: + fail-fast: false + matrix: ${{ fromJSON(needs.matrix.outputs.nuttx_matrix) }} + steps: + - uses: actions/checkout@v4 + with: + path: app + - name: The tiny-edge release west.yml pins + id: tiny-edge + run: | + rev=$(grep -A3 -- '- name: tiny-edge' app/west.yml | sed -n 's/^ *revision: *//p') + echo "revision=$rev" >> "$GITHUB_OUTPUT" + - uses: actions/checkout@v4 + with: + repository: thin-edge/tiny-edge + ref: ${{ steps.tiny-edge.outputs.revision }} + path: tiny-edge + - uses: docker/setup-buildx-action@v3 + - name: tiny-edge's NuttX container + uses: docker/build-push-action@v6 + with: + context: tiny-edge/ports/nuttx/docker + tags: tiny-edge-nuttx + load: true + cache-from: type=gha,scope=nuttx + cache-to: type=gha,scope=nuttx,mode=max + - name: Build and package + run: | + docker run --rm -v "$GITHUB_WORKSPACE":/ws -w /ws/app \ + -e TINY_EDGE_DIR=/ws/tiny-edge tiny-edge-nuttx bash -c ' + set -e + scripts/release/check-version.sh --apply "${{ github.ref_name }}" + for name in ${{ matrix.builds }}; do + echo "::group::$name" + scripts/release/build.sh "$name" --build-dir "/ws/build/$name" --out /ws/dist + echo "::endgroup::" + done' + - uses: actions/upload-artifact@v4 + with: + name: firmware-${{ matrix.group }} + path: dist/ + if-no-files-found: error + release: name: Publish the release - # build-idf is skipped when release/devices.yml lists no ESP-IDF build. + # build-idf and build-nuttx are skipped when release/devices.yml lists no + # build of theirs. if: >- startsWith(github.ref, 'refs/tags/v') && !cancelled() && needs.build.result == 'success' && - (needs.build-idf.result == 'success' || needs.build-idf.result == 'skipped') - needs: [matrix, build, build-idf] + (needs.build-idf.result == 'success' || needs.build-idf.result == 'skipped') && + (needs.build-nuttx.result == 'success' || needs.build-nuttx.result == 'skipped') + needs: [matrix, build, build-idf, build-nuttx] runs-on: ubuntu-24.04 permissions: contents: write diff --git a/DEVICES.md b/DEVICES.md index d96a4d9..52e4d27 100644 --- a/DEVICES.md +++ b/DEVICES.md @@ -342,6 +342,30 @@ parameters, and the cloud shell were tested with the same FreeRTOS port in tiny-edge's own ESP-IDF sample on this board (tiny-edge 0.4.0, 2026-11-11), not yet with this application. +### NuttX agent on the ESP32-C6 (2026-11-17) + +The `nuttx` variant of `tedge-agent` is tiny-edge's NuttX agent (NuttX 13.1, +tiny-edge's `ports/nuttx`), built with tiny-edge's `build.sh` and booting +through MCUboot. Its update image is 1,415,232 B, 92 % of its 1.5 MB slot +(the 4 MB layout's largest; tiny-edge's NuttX README). dram0 holds +271,904 of 452,112 B. + +On an ESP32-C6-DevKitC (eu-latest, `tedge-e8f60afb8198`), with no program +reading the USB port, from tiny-edge's NuttX change (`ports/nuttx/REPORT.md`): + +- **Set-up:** from an erased data partition with `tedge-onboard` over USB, + the board's Wi-Fi scan, and a wrong password that brought it back to its + stored network; certificate enrollment and renewal. +- **Cumulocity:** `c8y_Agent`, `tedge_OS` (`nuttx` 13.1.0), `c8y_Hardware`, + the `device` and health measurements, and type `thin-edge.io-nuttx`. +- **Operations:** a restart, log upload, the cloud shell, parameters that + kept their values across a restart, a remote access tunnel to the LAN, a + firmware update from Cumulocity that confirmed, the rollback of an image + that could not connect, and an operation interrupted by a reset, which was + failed after the reboot. +- **30 minutes** with telemetry and 52 tunnels: no gap in the measurements, + free heap steady at about 96 KB, at most 63 % of any thread's stack. + ### Identify measured (2026-10-06) `device-identify` adds `tedge identify [seconds]` (cloud shell) and the sw0 diff --git a/README.md b/README.md index 3758677..2d8ef86 100644 --- a/README.md +++ b/README.md @@ -15,13 +15,13 @@ Every [release](https://github.com/thin-edge/microcontroller-devices/releases) carries ready-to-flash images for these devices — no toolchain, no build: -| Device | Zephyr board | standalone | tedge-full | tedge-ota | tedge-onboard | -|---|---|---|---|---|---| -| ESP32-C6-DevKitC-1 (N4) | `esp32c6_devkitc/esp32c6/hpcore` | `modbus-server`, `opcua-server`, `snmp-agent` | `modbus-server`, `snmp-agent`, `tedge-agent` | `modbus-server`, `snmp-agent`, `tedge-agent` | `tedge-agent-idf` | -| ESP32-S3-DevKitC-1 (N16R8) | `esp32s3_devkitc/esp32s3/procpu` | `modbus-server`, `opcua-server`, `snmp-agent` | `modbus-server`, `snmp-agent`, `tedge-agent` | `modbus-server`, `snmp-agent`, `tedge-agent` | — | -| Adafruit QT Py ESP32-S3 (N4R2) | `adafruit_qt_py_esp32s3/esp32s3/procpu` | `modbus-server`, `opcua-server`, `snmp-agent` | `modbus-server`, `snmp-agent`, `tedge-agent` | `modbus-server`, `opcua-server`, `snmp-agent`, `tedge-agent` | — | -| ESP32-WROOM-32 DevKitC | `esp32_devkitc/esp32/procpu` | `modbus-server`, `opcua-server`, `snmp-agent` | — | `modbus-server`, `tedge-agent` | — | -| ESP32-CAM (ESP32 + 4 MB PSRAM) | `esp32_devkitc/esp32/procpu` | — | — | `snmp-agent`, `modbus-server`, `tedge-agent` | — | +| Device | Zephyr board | standalone | tedge-full | tedge-ota | tedge-onboard | nuttx | +|---|---|---|---|---|---|---| +| ESP32-C6-DevKitC-1 (N4) | `esp32c6_devkitc/esp32c6/hpcore` | `modbus-server`, `opcua-server`, `snmp-agent` | `modbus-server`, `snmp-agent`, `tedge-agent` | `modbus-server`, `snmp-agent`, `tedge-agent` | `tedge-agent-idf` | `tedge-agent` | +| ESP32-S3-DevKitC-1 (N16R8) | `esp32s3_devkitc/esp32s3/procpu` | `modbus-server`, `opcua-server`, `snmp-agent` | `modbus-server`, `snmp-agent`, `tedge-agent` | `modbus-server`, `snmp-agent`, `tedge-agent` | — | — | +| Adafruit QT Py ESP32-S3 (N4R2) | `adafruit_qt_py_esp32s3/esp32s3/procpu` | `modbus-server`, `opcua-server`, `snmp-agent` | `modbus-server`, `snmp-agent`, `tedge-agent` | `modbus-server`, `opcua-server`, `snmp-agent`, `tedge-agent` | — | — | +| ESP32-WROOM-32 DevKitC | `esp32_devkitc/esp32/procpu` | `modbus-server`, `opcua-server`, `snmp-agent` | — | `modbus-server`, `tedge-agent` | — | — | +| ESP32-CAM (ESP32 + 4 MB PSRAM) | `esp32_devkitc/esp32/procpu` | — | — | `snmp-agent`, `modbus-server`, `tedge-agent` | — | — | - **standalone** — the protocol server on its own (Modbus TCP, OPC-UA or @@ -39,13 +39,19 @@ carries ready-to-flash images for these devices — no toolchain, no build: [tedge-onboard](https://github.com/thin-edge/tiny-edge/tree/main/tools/onboard): it gives the device your computer's Wi-Fi network and registers it in your tenant. The images are not signed (ESP-IDF's bootloader, no MCUboot). +- **nuttx** — `tedge-agent` on [NuttX](https://nuttx.apache.org/): tiny-edge's + NuttX agent with every thin-edge.io feature, set up over USB with + `tedge-onboard` like `tedge-onboard` (on a DevKitC, the connector labelled + USB). It boots through MCUboot, so a firmware update that cannot connect is + rolled back. The images are not signed with a key: MCUboot checks their + hash. Each image comes as `---` with: | File | What it is for | |---|---| | `.factory.bin` | The first flash: everything in one image, written at `0x0` | -| `.app.bin` | Updates: `flash.sh --app-only`, or Cumulocity firmware update | +| `.app.bin` | Updates: `flash.sh --app-only`, or Cumulocity firmware update. For `nuttx` images it is the update image, which MCUboot boots on trial; `flash.sh --app-only` writes the bundle's confirmed image instead | | `.zip` | The three images with their offsets (`flash.json`) and instructions, for `scripts/flash.sh` | ### From your browser diff --git a/apps/tedge-agent-idf/README.md b/apps/tedge-agent-idf/README.md index 2f78338..9ff5cc1 100644 --- a/apps/tedge-agent-idf/README.md +++ b/apps/tedge-agent-idf/README.md @@ -75,6 +75,6 @@ tiny-edge Smart Functions and DTM definitions: slots and the rollback, and set-up is `tedge-onboard` over USB instead of Improv BLE or lab-ztp-provisioner. - No status LED or identify pattern yet. -- tiny-edge 0.6.0, so the client's parameter sets are `tiny_edge` and +- tiny-edge 0.6.0 and later, so the client's parameter sets are `tiny_edge` and `tiny_edge_remote_access`, not `zephyr_tedge`, and `tedge-onboard` registers the device with the image's type, `thin-edge.io-esp-idf-agent`. diff --git a/apps/tedge-agent-idf/main/idf_component.yml b/apps/tedge-agent-idf/main/idf_component.yml index 2f04f3d..505f64f 100644 --- a/apps/tedge-agent-idf/main/idf_component.yml +++ b/apps/tedge-agent-idf/main/idf_component.yml @@ -7,4 +7,4 @@ dependencies: tiny_edge: git: https://github.com/thin-edge/tiny-edge.git path: ports/freertos/esp-idf/tiny_edge - version: v0.6.0 + version: v0.7.0 diff --git a/release/devices.yml b/release/devices.yml index 2724140..a63690d 100644 --- a/release/devices.yml +++ b/release/devices.yml @@ -29,6 +29,12 @@ # `framework: esp-idf` marks an ESP-IDF application (apps/ built with # idf.py, the device's `chip` as the target): no Zephyr profile, board files # or tiny-edge board configuration; it brings its own sdkconfig.defaults. +# +# `framework: nuttx` marks tiny-edge's NuttX agent, built with tiny-edge's +# ports/nuttx/scripts/build.sh for the device's `nuttx_board` (a board +# tiny-edge's NuttX port has a configuration for). The build's `extra_conf` +# are NuttX configuration fragments, such as +# "{tiny-edge}/ports/nuttx/configs/esp32s3-psram-octal.conf". variants: standalone: description: the protocol server alone; Wi-Fi over Improv BLE @@ -45,6 +51,10 @@ variants: description: the device agent on ESP-IDF (FreeRTOS) with every thin-edge.io feature; Wi-Fi and Cumulocity over USB with tedge-onboard framework: esp-idf provisioner: tedge-onboard + nuttx: + description: the device agent on NuttX with every thin-edge.io feature; Wi-Fi and Cumulocity over USB with tedge-onboard + framework: nuttx + provisioner: tedge-onboard provisioners: improv: null @@ -60,6 +70,7 @@ devices: flash_size: 4MB esptool_before: usb-reset tedge_board_conf: "{tiny-edge}/ports/zephyr/profiles/boards/esp32c6-devkitc.conf" + nuttx_board: esp32c6-devkitc builds: - { app: modbus-server, variant: standalone, pr: true } - { app: opcua-server, variant: standalone } @@ -92,6 +103,10 @@ devices: variant: tedge-onboard pr: true measured: DEVICES.md#esp-idf-agent-on-the-esp32-c6-2026-11-12 + - app: tedge-agent + variant: nuttx + pr: true + measured: DEVICES.md#nuttx-agent-on-the-esp32-c6-2026-11-17 - id: esp32s3-devkitc name: ESP32-S3-DevKitC-1 (N16R8) diff --git a/scripts/flash.sh b/scripts/flash.sh index c9bc26f..da48730 100755 --- a/scripts/flash.sh +++ b/scripts/flash.sh @@ -27,14 +27,17 @@ # An ESP-IDF bundle (a `tedge-onboard` image: flash.json says # "framework": "esp-idf") is flashed the same way: everything, or with # --app-only the application into ota_0; --erase-storage erases its NVS -# partition (the Wi-Fi network and the Cumulocity certificate). +# partition (the Wi-Fi network and the Cumulocity certificate). A NuttX +# bundle ("framework": "nuttx", tiny-edge's NuttX agent) too: MCUboot and +# the confirmed image, or with --app-only the image into the primary slot; +# --erase-storage erases its LittleFS data partition. # # Local builds are signed with MCUboot's public development key: fine for # development, never for production devices. A bundle's flash.json says which # key signed it. set -euo pipefail -usage() { sed -n '4,29p' "$0" | sed 's/^# \{0,1\}//'; exit "${1:-0}"; } +usage() { sed -n '4,37p' "$0" | sed 's/^# \{0,1\}//'; exit "${1:-0}"; } [[ $# -ge 1 ]] || usage 1 build=$1; shift @@ -58,7 +61,7 @@ run() { [[ $dry -eq 1 ]] || "$@" } -if [[ -f $build/flash.json ]] && grep -q '"framework": "esp-idf"' "$build/flash.json"; then +if [[ -f $build/flash.json ]] && grep -qE '"framework": "(esp-idf|nuttx)"' "$build/flash.json"; then vars=$(python3 - "$build" <<'PY' import json, shlex, sys from pathlib import Path @@ -66,7 +69,8 @@ d = Path(sys.argv[1]) f = json.loads((d / "flash.json").read_text()) imgs = [(i["offset"], str(d / i["file"])) for i in f["images"]] app = next(i for i in f["images"] if i["name"] == "app") -nvs = f["partitions"].get("nvs", {}) +# The partition with the device's settings: NVS on ESP-IDF, LittleFS on NuttX. +nvs = f["partitions"].get("nvs") or f["partitions"].get("data") or {} for k, v in {"CHIP": f["chip"], "ESPTOOL_BEFORE": f["esptool_before"], "FLASH_ARGS": " ".join(f["flash_args"]), "ALL_IMAGES": " ".join(f"{o} {shlex.quote(p)}" for o, p in imgs), diff --git a/scripts/release/build.sh b/scripts/release/build.sh index 5cb32b7..a6f6955 100755 --- a/scripts/release/build.sh +++ b/scripts/release/build.sh @@ -21,6 +21,14 @@ # environment exported; scripts/release/package_idf.py packages it and fails # it when the application takes more than 80 % of its OTA slot. # +# A NuttX build (`framework: nuttx`) is tiny-edge's NuttX agent: it runs +# tiny-edge's ports/nuttx/scripts/build.sh for the device's `nuttx_board`, +# inside tiny-edge's NuttX container (ports/nuttx/docker), with the release's +# firmware name and apps//VERSION, and the build's `extra_conf` as +# NuttX fragments. TINY_EDGE_DIR names the tiny-edge checkout where west is +# not installed; NUTTX_WS, NuttX's workspace (default: in the build +# directory). scripts/release/package_nuttx.py packages it. +# # The size step (scripts/release/size.py) fails the build when an image is # over its flash budget or when any RAM region or image has grown past the # build's entry in release/size-baseline.json. A change that moves a build's @@ -28,7 +36,7 @@ # does every build) and commits it in the same pull request. set -euo pipefail -[[ $# -ge 1 ]] || { sed -n '4,23p' "$0" | sed 's/^# \{0,1\}//'; exit 1; } +[[ $# -ge 1 ]] || { sed -n '4,32p' "$0" | sed 's/^# \{0,1\}//'; exit 1; } name=$1; shift root=$(cd "$(dirname "$0")/../.." && pwd) build=$root/build-release/$name @@ -67,6 +75,42 @@ if [[ $FRAMEWORK == esp-idf ]]; then exit 0 fi +if [[ $FRAMEWORK == nuttx ]]; then + te=$(python3 - "$root" <<'PY' +import sys +sys.path.insert(0, sys.argv[1] + "/scripts/release") +import matrix +print(matrix.west_project_dir("tiny-edge") or "") +PY + ) + [[ -x $te/ports/nuttx/scripts/build.sh ]] || { + echo "tiny-edge's NuttX port not found (set TINY_EDGE_DIR)" >&2; exit 1; } + v=$root/apps/$APP/VERSION + field() { sed -n "s/^$1 *= *\([^ #]*\).*/\1/p" "$v"; } + version="$(field VERSION_MAJOR).$(field VERSION_MINOR).$(field PATCHLEVEL)" + extra=$(field EXTRAVERSION) + [[ -z $extra ]] || version="$version-$extra" + rm -rf "$build" + mkdir -p "$build" "$out" + # The release's firmware name, which an update must carry. + printf 'CONFIG_TINY_EDGE_FIRMWARE_NAME="%s"\n' "$FIRMWARE_NAME" \ + > "$build/release.conf" + args=(--agent --version "$version") + while IFS= read -r frag; do + case $frag in "") continue ;; /*) ;; *) frag=$root/$frag ;; esac + args+=(--extra "$frag") + done < <(tr ';' '\n' <<< "$CONF_FILES") + ws=${NUTTX_WS:-$build/nuttx} + NUTTX_WS=$ws "$te/ports/nuttx/scripts/build.sh" \ + "$NUTTX_BOARD" "${args[@]}" --extra "$build/release.conf" \ + --out "$build/out" + python3 "$root/scripts/release/package_nuttx.py" "$build/out" \ + --config "$ws/nuttx/.config" \ + --device "$DEVICE" --app "$APP" --variant "$VARIANT" --out "$out" \ + --version "$version" --tiny-edge "$te" + exit 0 +fi + abs() { # "a;/x/b" repository-relative or absolute -> "/root/a;/x/b" [[ -n $1 ]] || return 0 tr ';' '\n' <<< "$1" | sed "s#^\([^/]\)#$root/\1#" | paste -sd';' - diff --git a/scripts/release/matrix.py b/scripts/release/matrix.py index 89f392d..09d139f 100755 --- a/scripts/release/matrix.py +++ b/scripts/release/matrix.py @@ -3,7 +3,8 @@ """Validate release/devices.yml and turn it into the release build matrix. scripts/release/matrix.py # GitHub matrix JSON on stdout - scripts/release/matrix.py --github-output # "matrix=" and "idf_matrix=" + scripts/release/matrix.py --github-output # "matrix=", "idf_matrix=" + # and "nuttx_matrix=" # for $GITHUB_OUTPUT scripts/release/matrix.py --subset pr # only the builds marked `pr: true` scripts/release/matrix.py --markdown # supported-devices table @@ -20,6 +21,11 @@ or tiny-edge board configuration, and its builds form a matrix of their own (`idf_matrix`), which runs in Espressif's ESP-IDF container. +A variant with `framework: nuttx` is tiny-edge's NuttX agent, built with +tiny-edge's ports/nuttx/scripts/build.sh for the device's `nuttx_board`. Its +builds' `extra_conf` are NuttX configuration fragments, and they form +`nuttx_matrix`, which runs in tiny-edge's NuttX container. + Every problem found is reported, each naming the device and build it is in, and the exit status is non-zero; nothing is printed on stdout then, so a workflow step cannot build from a half-valid manifest. @@ -30,6 +36,7 @@ import argparse import functools import json +import os import re import subprocess import sys @@ -44,7 +51,9 @@ # A file from another west project is written "{project}/path", for example # "{tiny-edge}/ports/zephyr/profiles/full.conf". It is found with `west list`; # where west is not installed (the release workflow's matrix job) such a file -# is not checked. +# is not checked. _DIR in the environment (TINY_EDGE_DIR for +# tiny-edge) names the project's directory instead of west, as in the NuttX +# container. PROJECT_REF = re.compile(r"^\{([\w.-]+)\}/(.+)$") # tedge_identity's firmware_name buffer in tiny-edge is 48 bytes. @@ -52,7 +61,7 @@ ID_RE = re.compile(r"^[a-z0-9]+(-[a-z0-9]+)*$") FLASH_RE = re.compile(r"^(1|2|4|8|16|32)MB$") ESPTOOL_BEFORE = {"default-reset", "usb-reset", "no-reset"} -FRAMEWORKS = {"zephyr", "esp-idf"} +FRAMEWORKS = {"zephyr", "esp-idf", "nuttx"} class ManifestError(Exception): @@ -84,8 +93,11 @@ def firmware_name(self): return f"{self.app}-{self.variant}-{self.device['id']}" def conf_files(self): - """EXTRA_CONF_FILE for the application, repository-relative.""" + """EXTRA_CONF_FILE for the application, repository-relative; for + NuttX, the configuration fragments build.sh --extra adds.""" files = [] + if self.framework == "nuttx": + return list(self.extra_conf) if self.framework != "zephyr": return files if self.variant_def.get("profile"): @@ -107,6 +119,7 @@ def matrix_entry(self, manifest): "app": self.app, "variant": self.variant, "framework": self.framework, + "nuttx_board": self.device.get("nuttx_board", ""), "firmware_name": self.firmware_name, "conf_files": ";".join(self.conf_files()), "dtc_overlays": ";".join(self.device.get("dtc_overlays", [])), @@ -118,6 +131,9 @@ def matrix_entry(self, manifest): @functools.lru_cache(maxsize=None) def west_project_dir(name): """Absolute path of a west project, or None when west cannot say.""" + env = os.environ.get(name.upper().replace("-", "_") + "_DIR") + if env: + return Path(env) try: out = subprocess.run(["west", "list", "-f", "{abspath}", name], cwd=ROOT, capture_output=True, text=True, @@ -229,14 +245,17 @@ def need_file(where, rel): if variant not in variants: problems.append(f"{bwhere}: unknown variant") continue - idf = variants[variant].get("framework", "zephyr") == "esp-idf" + fw = variants[variant].get("framework", "zephyr") if not (root / "apps" / app / "CMakeLists.txt").is_file(): problems.append(f"{bwhere}: there is no apps/{app}") - elif idf and not (root / "apps" / app / "sdkconfig.defaults").is_file(): + elif fw == "esp-idf" and not (root / "apps" / app / "sdkconfig.defaults").is_file(): problems.append(f"{bwhere}: apps/{app} is not an ESP-IDF " "project (no sdkconfig.defaults)") - elif not idf and board and not (root / "apps" / app / "boards" / - f"{board_dir_name(board)}.conf").is_file(): + elif fw == "nuttx" and not dev.get("nuttx_board"): + problems.append(f"{bwhere}: a NuttX build needs the device's " + "nuttx_board (a tiny-edge NuttX board name)") + elif fw == "zephyr" and board and not (root / "apps" / app / "boards" / + f"{board_dir_name(board)}.conf").is_file(): problems.append(f"{bwhere}: apps/{app}/boards/" f"{board_dir_name(board)}.conf does not exist") for conf in b.get("extra_conf", []): @@ -280,11 +299,18 @@ def need_file(where, rel): def groups(builds): - """One matrix job per (device, app), in manifest order.""" + """One matrix job per (device, app, framework), in manifest order. + + The group names the job's artifact. The NuttX agent shares its app name + with the Zephyr tedge-agent builds, so its group names the framework. + """ out = {} for b in builds: - g = out.setdefault((b.device["id"], b.app), { - "group": f"{b.app}-{b.device['id']}", + name = f"{b.app}-{b.device['id']}" + if b.framework == "nuttx": + name = f"{b.app}-nuttx-{b.device['id']}" + g = out.setdefault((b.device["id"], b.app, b.framework), { + "group": name, "device": b.device["id"], "device_name": b.device["name"], "chip": b.device["chip"], @@ -351,9 +377,11 @@ def main(argv=None): builds = [b for b in builds if b.pr] zephyr = [b for b in builds if b.framework == "zephyr"] idf = [b for b in builds if b.framework == "esp-idf"] + nuttx = [b for b in builds if b.framework == "nuttx"] if args.github_output: print(f"matrix={json.dumps({'include': groups(zephyr)})}") print(f"idf_matrix={json.dumps({'include': groups(idf)})}") + print(f"nuttx_matrix={json.dumps({'include': groups(nuttx)})}") else: print(json.dumps({"include": groups(builds)})) return 0 diff --git a/scripts/release/notes.py b/scripts/release/notes.py index f43126b..fa901c0 100755 --- a/scripts/release/notes.py +++ b/scripts/release/notes.py @@ -78,7 +78,8 @@ def notes(manifest, metas, tag, repo): f"Wi-Fi over BLE]({readme}#provisioning-wi-fi-over-ble), including its " f"security notes.", "", - "`tedge-onboard` images (ESP-IDF) are set up over USB instead: run " + "The ESP-IDF (`tedge-onboard`) and NuttX (`nuttx`) agent images are " + "set up over USB instead: run " "[tedge-onboard](https://github.com/thin-edge/tiny-edge/tree/main/tools/onboard) " "on the computer the board is plugged into. It gives the device the " "computer's Wi-Fi network, registers it in your tenant, and passes it " diff --git a/scripts/release/package_nuttx.py b/scripts/release/package_nuttx.py new file mode 100644 index 0000000..bf11d37 --- /dev/null +++ b/scripts/release/package_nuttx.py @@ -0,0 +1,281 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: Apache-2.0 +"""Package one NuttX build (tiny-edge's ports/nuttx/scripts/build.sh) as +release assets. + + scripts/release/package_nuttx.py --config \\ + --device --app --variant --version \\ + --tiny-edge --out [--max-slot 95] + +The NuttX counterpart of package.py and package_idf.py, with the same +outputs, so the release notes, c8y-firmware.json and SHA256SUMS treat every +framework alike. is what tiny-edge's build.sh wrote: MCUboot, +nuttx.bin (padded and confirmed, for flashing) and nuttx-update.bin (for a +firmware update). With = ---: + + .factory.bin MCUboot and the confirmed image, merged into one image + that is written at 0x0 + .app.bin the firmware update: unpadded and not confirmed, so + MCUboot boots it once on trial and the agent confirms it + .zip / with MCUboot, the confirmed image, flash.json + and README.txt (scripts/flash.sh / flashes it) + .meta.json what the release notes and c8y-firmware.json are made + from; not a release asset + .size.json the update image against its slot + +The slots, the data partition, the firmware name, the device type and the +features come from the build's NuttX configuration. The build fails when the +update image takes more than --max-slot percent of its slot. Needs esptool +on PATH. +""" + +import argparse +import hashlib +import json +import re +import shutil +import subprocess +import sys +import zipfile +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).resolve().parent)) +import matrix # noqa: E402 + +SIGNING = { + "key": "none", + "fingerprint": None, + "statement": ("NuttX images (the NuttX agent) are not signed: MCUboot " + "checks each image's SHA-256 hash, not who built it. Use " + "MCUboot with a signing key for production devices."), +} + +FEATURES = ("TELEMETRY", "HEALTH", "RESTART", "FIRMWARE_UPDATE", + "REMOTE_ACCESS", "SHELL_COMMAND", "LOG_UPLOAD", "COREDUMP", + "PARAMETERS", "CERT_RENEWAL") + +README = """{name} +{rule} + +{device_name} — {app} ({variant}, tiny-edge's NuttX agent), version {version} + +Flash everything (first flash; erases the board, including the stored Wi-Fi +network and Cumulocity certificate): + + esptool --chip {chip} --before {before} erase-flash + esptool --chip {chip} --before {before} write-flash {writes} + +or, from a checkout of the repository: + + scripts/flash.sh {name} --erase-all + +or, without any tools, the single image {stem}.factory.bin at 0x0 from a +browser flasher such as https://web.esphome.io/ (Chrome or Edge). + +Replace only the application (keeps the Wi-Fi network and the certificate): + + esptool --chip {chip} --before {before} write-flash {app_off} app.signed.bin + +Then set it up over USB: run tedge-onboard on the computer the board is +plugged into (https://github.com/thin-edge/tiny-edge/tree/main/tools/onboard), +connected to the board's USB port (on a DevKitC, the connector labelled USB). +It gives the device the computer's Wi-Fi network, registers it in your +Cumulocity tenant, and passes it the tenant and a one-time password. + +Updates from Cumulocity install {stem}.app.bin, which MCUboot boots once on +trial: the agent confirms it once it has connected, or MCUboot goes back to +the image before. + +{signing} +""" + + +def sha256(path): + return hashlib.sha256(Path(path).read_bytes()).hexdigest() + + +def read_config(path): + """CONFIG_X=value -> {X: value}; strings unquoted, y -> True.""" + cfg = {} + for line in Path(path).read_text().splitlines(): + m = re.match(r"^CONFIG_(\w+)=(.*)$", line) + if not m: + continue + v = m.group(2) + if v == "y": + v = True + elif len(v) >= 2 and v[0] == v[-1] == '"': + v = v[1:-1] + cfg[m.group(1)] = v + return cfg + + +def option(cfg, name): + """An MCUboot layout option, named CONFIG_ESPRESSIF_* on the RISC-V ESP + chips and CONFIG_ESP32S3_* on the ESP32-S3.""" + for prefix in ("ESPRESSIF_", "ESP32S3_"): + if prefix + name in cfg: + return int(str(cfg[prefix + name]), 0) + raise SystemExit(f"the NuttX configuration has no {name} (built without " + "MCUboot?)") + + +def main(argv=None): + ap = argparse.ArgumentParser(description=__doc__.split("\n")[0]) + ap.add_argument("build") + ap.add_argument("--config", required=True, help="the build's NuttX .config") + ap.add_argument("--device", required=True) + ap.add_argument("--app", required=True) + ap.add_argument("--variant", required=True) + ap.add_argument("--version", required=True) + ap.add_argument("--tiny-edge", required=True, help="the tiny-edge checkout") + ap.add_argument("--out", required=True) + ap.add_argument("--manifest", default=matrix.DEFAULT_MANIFEST) + ap.add_argument("--esptool", default="esptool") + # The agent with every feature takes about 92 % of its 1.5 MB slot; the + # slot is as large as the 4 MB layout allows (tiny-edge's NuttX README). + ap.add_argument("--max-slot", type=float, default=95.0, + help="largest share of the slot the update image may take") + args = ap.parse_args(argv) + + manifest, builds = matrix.load(args.manifest) + build = next((b for b in builds if b.device["id"] == args.device and + b.app == args.app and b.variant == args.variant), None) + if build is None or build.framework != "nuttx": + print(f"{args.device} {args.app} {args.variant} is not a NuttX build " + "in the manifest", file=sys.stderr) + return 1 + dev = build.device + bdir = Path(args.build) + cfg = read_config(args.config) + + chip = cfg.get("ARCH_CHIP", "") + if chip != dev["chip"]: + print(f"the build is for {chip!r}, the device is {dev['chip']}", + file=sys.stderr) + return 1 + if cfg.get("TINY_EDGE_FIRMWARE_NAME") != build.firmware_name: + print(f"the image reports firmware name " + f"{cfg.get('TINY_EDGE_FIRMWARE_NAME')!r}, expected " + f"{build.firmware_name!r}", file=sys.stderr) + return 1 + if cfg.get("TINY_EDGE_FIRMWARE_VERSION") != args.version: + print(f"the image reports version " + f"{cfg.get('TINY_EDGE_FIRMWARE_VERSION')!r}, expected " + f"{args.version!r}", file=sys.stderr) + return 1 + # No release image carries site settings. + if cfg.get("TEDGE_C8Y_URL"): + print("CONFIG_TEDGE_C8Y_URL is set in this build: release images carry " + "no tenant", file=sys.stderr) + return 1 + + slot0 = option(cfg, "OTA_PRIMARY_SLOT_OFFSET") + slot1 = option(cfg, "OTA_SECONDARY_SLOT_OFFSET") + slot = option(cfg, "OTA_SLOT_SIZE") + scratch = option(cfg, "OTA_SCRATCH_OFFSET") + parts = { + "boot": {"offset": 0, "size": slot0}, + "slot0": {"offset": slot0, "size": slot}, + "slot1": {"offset": slot1, "size": slot}, + "scratch": {"offset": scratch, "size": option(cfg, "OTA_SCRATCH_SIZE")}, + "data": {"offset": option(cfg, "STORAGE_MTD_OFFSET"), + "size": option(cfg, "STORAGE_MTD_SIZE")}, + } + + mcuboot = bdir / f"mcuboot-{chip}.bin" + confirmed = bdir / "nuttx.bin" + update = bdir / "nuttx-update.bin" + for f in (mcuboot, confirmed, update): + if not f.is_file(): + print(f"{f} is missing: not a tiny-edge NuttX build with MCUboot", + file=sys.stderr) + return 1 + update_size = update.stat().st_size + percent = round(100.0 * update_size / slot, 1) + if percent > args.max_slot: + print(f"the update image is {update_size} B, {percent}% of its " + f"{slot} B slot (at most {args.max_slot}%)", file=sys.stderr) + return 1 + + stem = f"{build.firmware_name}-{args.version}" + out = Path(args.out) + out.mkdir(parents=True, exist_ok=True) + images = [("mcuboot", "mcuboot.bin", mcuboot, 0), + ("app", "app.signed.bin", confirmed, slot0)] + + factory = out / f"{stem}.factory.bin" + cmd = [args.esptool, "--chip", chip, "merge-bin", "--output", str(factory), + "--format", "raw"] + for _, _, src, off in images: + cmd += [f"{off:#x}", str(src)] + subprocess.run(cmd, check=True, stdout=subprocess.DEVNULL) + + app_bin = out / f"{stem}.app.bin" + shutil.copyfile(update, app_bin) + + before = dev.get("esptool_before", "default-reset") + flash = { + "schema": 1, + "framework": "nuttx", + "device": dev["id"], + "device_name": dev["name"], + "board": dev["nuttx_board"], + "app": args.app, + "variant": args.variant, + "version": args.version, + "firmware_name": build.firmware_name, + "chip": chip, + "flash_size": dev["flash_size"], + "esptool_before": before, + "flash_args": [], + "images": [{"name": n, "file": f, "offset": f"{off:#x}"} + for n, f, _, off in images], + "partitions": {n: {"offset": f"{p['offset']:#x}", "size": f"{p['size']:#x}"} + for n, p in parts.items()}, + "signing": SIGNING, + } + readme = README.format( + name=stem, rule="=" * len(stem), stem=stem, device_name=dev["name"], + app=args.app, variant=args.variant, version=args.version, chip=chip, + before=before, app_off=f"{slot0:#x}", + writes=" ".join(f"{off:#x} {f}" for _, f, _, off in images), + signing=SIGNING["statement"]) + + bundle = out / f"{stem}.zip" + with zipfile.ZipFile(bundle, "w", zipfile.ZIP_DEFLATED) as z: + for _, f, src, _ in images: + z.write(src, f"{stem}/{f}") + z.writestr(f"{stem}/flash.json", json.dumps(flash, indent=2) + "\n") + z.writestr(f"{stem}/README.txt", readme) + + tiny_edge = Path(args.tiny_edge) / "VERSION" + meta = { + **{k: flash[k] for k in ("device", "device_name", "board", "app", + "variant", "version", "firmware_name", "chip", + "flash_size")}, + "framework": "nuttx", + "stem": stem, + "tedge": True, + # tedge-onboard registers the device with the type the image reports, + # so the firmware repository entry reaches exactly these devices. + "device_type": cfg.get("TEDGE_DEVICE_TYPE") or None, + "features": [k.lower().replace("_", "-") for k in FEATURES + if cfg.get(f"TEDGE_{k}") is True], + "tedge_zephyr_version": (tiny_edge.read_text().strip() + if tiny_edge.is_file() else None), + "measured": build.measured, + "signing": SIGNING, + "assets": {p.name: sha256(p) for p in (factory, app_bin, bundle)}, + } + (out / f"{stem}.meta.json").write_text(json.dumps(meta, indent=2) + "\n") + size = {"image": update_size, "slot0": slot, "slot0_percent": percent, + "ram": []} + (out / f"{stem}.size.json").write_text(json.dumps(size, indent=2) + "\n") + for p in (factory, app_bin, bundle): + print(p) + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/scripts/release/tests/test_matrix.py b/scripts/release/tests/test_matrix.py index 8fea18b..1fa0db0 100644 --- a/scripts/release/tests/test_matrix.py +++ b/scripts/release/tests/test_matrix.py @@ -5,10 +5,12 @@ """ import json +import os import sys import tempfile import textwrap import unittest +import unittest.mock from pathlib import Path sys.path.insert(0, str(Path(__file__).resolve().parents[1])) @@ -237,9 +239,69 @@ def test_esp_idf_app_must_be_an_idf_project(self): def test_unknown_framework(self): self.idf_app() - p = self.problems(self.IDF.replace("framework: esp-idf", "framework: nuttx")) + p = self.problems(self.IDF.replace("framework: esp-idf", "framework: threadx")) self.assertIn("variant tedge-onboard: framework must be one of", p) + NUTTX = VALID.replace(""" tedge-ota: {profile: profiles/ota.conf, provisioner: ztp} +""", """ tedge-ota: {profile: profiles/ota.conf, provisioner: ztp} + nuttx: {framework: nuttx, provisioner: onboard} +""").replace(""" ztp: prov/ztp.conf +""", """ ztp: prov/ztp.conf + onboard: null +""").replace(""" tedge_board_conf: boards/c6.conf +""", """ tedge_board_conf: boards/c6.conf + nuttx_board: esp32c6-devkitc +""") + """ - app: modbus-server + variant: nuttx + extra_conf: [nuttx/psram.conf] + measured: DEVICES.md#measured-on-the-c6 +""" + + def test_nuttx_build(self): + """A NuttX build passes its fragments and board to tiny-edge's + build.sh, and lands in a matrix of its own, whose group is not the + same app's Zephyr group.""" + (self.root / "nuttx").mkdir() + (self.root / "nuttx/psram.conf").write_text("") + manifest, builds = self.load(self.NUTTX) + nx = [b for b in builds if b.variant == "nuttx"][0] + entry = nx.matrix_entry(manifest) + self.assertEqual(entry["framework"], "nuttx") + self.assertEqual(entry["nuttx_board"], "esp32c6-devkitc") + self.assertEqual(entry["conf_files"], "nuttx/psram.conf") + self.assertEqual(entry["firmware_name"], "modbus-server-nuttx-c6") + + from contextlib import redirect_stdout + import io + out = io.StringIO() + with redirect_stdout(out): + rc = matrix.main(["--manifest", str(self.root / "devices.yml"), + "--root", str(self.root), "--github-output"]) + self.assertEqual(rc, 0) + lines = dict(l.split("=", 1) for l in out.getvalue().splitlines()) + zephyr = json.loads(lines["matrix"])["include"] + nuttx = json.loads(lines["nuttx_matrix"])["include"] + self.assertEqual([j["group"] for j in zephyr], ["modbus-server-c6"]) + self.assertEqual([j["group"] for j in nuttx], ["modbus-server-nuttx-c6"]) + self.assertEqual([j["builds"] for j in nuttx], ["modbus-server-nuttx-c6"]) + + def test_nuttx_build_needs_a_board(self): + (self.root / "nuttx").mkdir() + (self.root / "nuttx/psram.conf").write_text("") + p = self.problems(self.NUTTX.replace(" nuttx_board: esp32c6-devkitc\n", "")) + self.assertIn("a NuttX build needs the device's nuttx_board", p) + + def test_project_dir_from_the_environment(self): + """Where west is not installed (the NuttX container), _DIR + names a west project's directory.""" + matrix.west_project_dir.cache_clear() + try: + with unittest.mock.patch.dict(os.environ, {"TINY_EDGE_DIR": "/x/te"}): + self.assertEqual(matrix.resolve(self.root, "{tiny-edge}/VERSION"), + Path("/x/te/VERSION")) + finally: + matrix.west_project_dir.cache_clear() + def test_repository_manifest_is_valid(self): manifest, builds = matrix.load() self.assertTrue(builds) diff --git a/west.yml b/west.yml index 4333ae9..0085a09 100644 --- a/west.yml +++ b/west.yml @@ -51,7 +51,7 @@ manifest: # CONFIG_TEDGE=y. Pin a release tag; move the pin to pick up a new one. - name: tiny-edge remote: thin-edge - revision: v0.6.0 + revision: v0.7.0 path: modules/lib/tiny-edge self: