From d37191164f739bf7ab4e6656254e8e1e3998ae9f Mon Sep 17 00:00:00 2001 From: John Vandenberg Date: Tue, 3 Nov 2026 07:38:23 +0800 Subject: [PATCH] Mixed hardware scenario --- .mise/config.rust.toml | 8 +- .mise/config.toml | 2 +- CHANGELOG.md | 40 +- CHECKS.md | 10 +- Cargo.lock | 27 +- Cargo.toml | 6 +- README.md | 28 +- config/cargo-features.toml | 34 + config/conftest/policy/cargo/cargo.rego | 36 +- config/conftest/policy/cargo/cargo_test.rego | 31 + config/jscpd-baseline.json | 20 +- services/firmware-modules/src/lib.rs | 2 +- services/firmware-modules/tests/generate.rs | 9 +- services/modules/Cargo.toml | 63 +- services/modules/src/lib.rs | 21 + .../src/picoserve.rs} | 28 +- .../tests/picoserve.rs} | 133 +--- services/storage/Cargo.toml | 79 ++- services/storage/src/lib.rs | 27 + services/storage/src/picoserve.rs | 177 ++++++ services/storage/tests/picoserve.rs | 163 +++++ services/ws-server-firmware/Cargo.toml | 24 +- services/ws-server-firmware/build.rs | 29 +- .../ws-server-firmware/src/board/esp32c6.rs | 3 +- services/ws-server-firmware/src/board/hub.rs | 72 +-- services/ws-server-firmware/src/board/mod.rs | 12 +- .../ws-server-firmware/src/board/pico2w.rs | 4 +- .../ws-server-firmware/src/board/platform.rs | 63 ++ .../ws-server-firmware/src/board/remote.rs | 169 +++++ services/ws-server-firmware/src/host.rs | 91 ++- services/ws-server-firmware/src/local.rs | 199 ++++++ services/ws-server-firmware/src/main.rs | 18 +- services/ws-server-firmware/src/runner.rs | 238 +++---- services/ws-server-firmware/src/server.rs | 87 +++ .../tests/serves_modules.rs | 53 +- services/ws-server/Cargo.toml | 100 +-- services/ws-server/src/lib.rs | 18 +- services/ws-server/src/picoserve_server.rs | 54 -- services/ws-server/src/routes.rs | 43 +- services/ws-server/tests/health.rs | 2 +- services/ws-test-server/Cargo.toml | 4 +- services/ws-wasi-runner/Cargo.toml | 13 +- services/ws-wasi-runner/src/board/buckets.rs | 46 ++ services/ws-wasi-runner/src/board/host.rs | 76 ++- services/ws-wasi-runner/src/board/mod.rs | 7 +- services/ws-wasi-runner/src/board/net.rs | 582 ++++++++++++++++++ .../ws-wasi-runner/src/board/net_tokio.rs | 82 +++ services/ws-wasi-runner/src/board/platform.rs | 32 +- services/ws-wasi-runner/src/board/run.rs | 9 +- .../ws-wasi-runner/src/board/test_host.rs | 51 +- services/ws-wasi-runner/tests/board.rs | 20 +- services/ws-wasi-runner/tests/board_net.rs | 239 +++++++ services/ws/Cargo.toml | 10 +- services/ws/src/health.rs | 28 + services/ws/src/lib.rs | 5 +- utilities/cli/src/deployment_types/dekit.rs | 26 +- .../cli/src/deployment_types/firmware.rs | 345 +++++++++-- .../cli/src/deployment_types/hub_sharing.rs | 12 +- utilities/cli/src/deployment_types/mise.rs | 39 +- utilities/cli/src/deployment_types/mod.rs | 3 +- .../src/deployment_types/process_compose.rs | 25 +- utilities/cli/src/error.rs | 8 + utilities/cli/src/input.rs | 52 ++ utilities/cli/src/lib.rs | 185 ++++-- utilities/cli/src/placement.rs | 22 +- utilities/cli/tests/firmware.rs | 209 ++++++- utilities/cli/tests/input.rs | 26 +- utilities/int-gen/src/openapi.rs | 2 +- verification/local/input/mixed-hardware.yaml | 34 + verification/local/output/default/README.md | 2 +- .../facility-security-scenario/README.md | 2 +- .../local/output/math1-multi-hub/README.md | 9 +- .../local/output/math1-multi/README.md | 9 +- .../local/output/mixed-hardware/.gitignore | 7 + .../local/output/mixed-hardware/README.md | 162 +++++ .../local/output/mixed-hardware/dekit.yaml | 11 + .../local/output/mixed-hardware/mise.toml | 99 +++ .../mixed-hardware/process-compose.yaml | 9 + .../local/output/mixed-hardware/secrets.env | 5 + .../local/output/wasi-math1/README.md | 2 +- .../published/output/math1-multi/README.md | 9 +- 81 files changed, 3878 insertions(+), 863 deletions(-) create mode 100644 config/cargo-features.toml rename services/{ws-server/src/picoserve_modules.rs => modules/src/picoserve.rs} (90%) rename services/{ws-server/tests/picoserve_server.rs => modules/tests/picoserve.rs} (57%) create mode 100644 services/storage/src/picoserve.rs create mode 100644 services/storage/tests/picoserve.rs create mode 100644 services/ws-server-firmware/src/board/platform.rs create mode 100644 services/ws-server-firmware/src/board/remote.rs create mode 100644 services/ws-server-firmware/src/local.rs create mode 100644 services/ws-server-firmware/src/server.rs delete mode 100644 services/ws-server/src/picoserve_server.rs create mode 100644 services/ws-wasi-runner/src/board/buckets.rs create mode 100644 services/ws-wasi-runner/src/board/net.rs create mode 100644 services/ws-wasi-runner/src/board/net_tokio.rs create mode 100644 services/ws-wasi-runner/tests/board_net.rs create mode 100644 services/ws/src/health.rs create mode 100644 verification/local/input/mixed-hardware.yaml create mode 100644 verification/local/output/mixed-hardware/.gitignore create mode 100644 verification/local/output/mixed-hardware/README.md create mode 100644 verification/local/output/mixed-hardware/dekit.yaml create mode 100644 verification/local/output/mixed-hardware/mise.toml create mode 100644 verification/local/output/mixed-hardware/process-compose.yaml create mode 100644 verification/local/output/mixed-hardware/secrets.env diff --git a/.mise/config.rust.toml b/.mise/config.rust.toml index 9ae15828..02db1ca6 100644 --- a/.mise/config.rust.toml +++ b/.mise/config.rust.toml @@ -105,8 +105,8 @@ shell = "{{ vars.task_shell }}" [tasks.build-no-std] description = "Build the no_std crates for a bare-metal target" run = """ -features=et-ws-service/picoserve-hub,et-ws-server/picoserve-server,et-ws-wasi-runner/board -set -- -p et-org -p edge-toolkit -p et-ws-service -p et-ws-server -p et-ws-wasi-runner +features=et-modules-service/board,et-storage-service/board,et-ws-service/board,et-ws-wasi-runner/board +set -- -p et-org -p edge-toolkit -p et-modules-service -p et-storage-service -p et-ws-service -p et-ws-wasi-runner cargo build --lib --target thumbv8m.main-none-eabihf --no-default-features --features "$features" "$@" """ shell = "{{ vars.task_shell }}" @@ -115,8 +115,8 @@ shell = "{{ vars.task_shell }}" [tasks.test-no-std] description = "Test the no_std crates built without std" run = """ -features=et-ws-service/picoserve-hub,et-ws-server/picoserve-server -set -- -p edge-toolkit -p et-ws-service -p et-ws-server +features=et-modules-service/board,et-storage-service/board,et-ws-service/board +set -- -p edge-toolkit -p et-modules-service -p et-storage-service -p et-ws-service cargo nextest run --config-file config/nextest.toml --no-default-features --features "$features" "$@" """ shell = "{{ vars.task_shell }}" diff --git a/.mise/config.toml b/.mise/config.toml index 552700b1..3808cdca 100644 --- a/.mise/config.toml +++ b/.mise/config.toml @@ -1343,7 +1343,7 @@ description = "Run conftest OPA/Rego policies over the TOML config + lock files" # combined TOML input. run = """ ns_list="cross cargo mise mise_lock pyproject checksums advisories" -named="Cargo.lock config/upstream-cache/data.toml config/deny.toml config/pyrefly.toml" +named="Cargo.lock config/cargo-features.toml config/upstream-cache/data.toml config/deny.toml config/pyrefly.toml" # Both lists rely on word-splitting when they expand. # $ns expands to repeated `--namespace ` pairs; $named is a space-separated path list. # shellcheck disable=SC2086 diff --git a/CHANGELOG.md b/CHANGELOG.md index 34fccbbc..3ed93394 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -19,7 +19,7 @@ Changelog 1.1.0. frames through the registry and hands back what to send where, leaving the transport, ids and clock to the embedder. Its features `std` (the actix-web hub), `nats` (the backplane) and `otel` (spans and metrics) are separate. Without `nats`, `WsConfig` has no `backplane`. -- `et-ws-service`'s `picoserve-hub` feature, on by default, adds `picoserve_hub::Hub`: a `no_std` hub serving +- `et-ws-service`'s `board` feature, on by default, adds `picoserve_hub::Hub`: a `no_std` hub serving `GET /ws` over HTTP and WebSocket through picoserve on any socket the embedder accepts, such as an `embassy-net` `TcpSocket`. It is a local hub, with no backplane, admin API, idle timeout or registry file. A crate awaiting `Hub::serve` needs `#![recursion_limit = "256"]`. @@ -27,16 +27,23 @@ Changelog 1.1.0. `LocalLink` routes what it sends as a connection's frames are, queues what others send it, and disconnects the agent when dropped. It is how a board's own runner reaches the hub beside it. Its `send_text` and `send_binary` refuse a frame larger than the hub reads from a socket with `FrameTooLarge`. -- `et-ws-server` builds as `no_std` + `alloc` without its new `std` feature. Its `picoserve-server` feature, on by - default, adds `picoserve_server::serve`: the hub's `/ws` with `/health` and `/favicon.ico` beside it, over picoserve, - and the modules a `picoserve_modules::Modules` builds into the binary under `/modules` and `/`, as the `std` server - serves them from disk. A module listed as served elsewhere is redirected to a base URL instead. -- The `et-ws-server-firmware` firmware runs et-ws-server on a Raspberry Pi Pico 2 W, an ESP32-C6 or a Pimoroni Pico +- `et-modules-service` and `et-storage-service` build as `no_std` + `alloc` without their new default `std` feature, + and each serves through picoserve under its new `board` feature. `et_modules_service::picoserve::ModuleRoutes` + serves the modules a `Modules` builds into the binary under `/modules` and `/`, as the `std` service serves them + from disk, redirecting a module listed as served elsewhere to a base URL. `et_storage_service::picoserve`'s + `StorageRoutes`, mounted at its `path()`, serves agent storage at `/storage/{agent_id}/{filename}`, held to the `std` + service's ownership rule against an `AgentRegistry` and kept in the embedder's `Storage`, each file at most + `MAX_STORED` bytes. Both are used with `default-features = false` from the workspace, so a consumer of the `std` + service now names the `std` feature. +- The `et-ws-server-firmware` firmware runs the hub on a Raspberry Pi Pico 2 W, an ESP32-C6 or a Pimoroni Pico Plus 2 W, over the board's Wi-Fi. `MISE_ENV=rust mise run build-firmware-pico2w` fetches the Pico's radio blobs and builds it for the Pico, `build-firmware-esp32c6` for the ESP32-C6, and `build-firmware-pico-plus2w` for the Pico Plus 2 W, each for the network `WIFI_NETWORK` and `WIFI_PASSWORD` name, embedding the modules `MODULES_PATHS` names and serving `MODULES_ROOT` at `/`. The Pico Plus 2 W makes its 8 MiB of PSRAM the heap, and also runs each wasi runner `RUNNER_GUESTS` names beside the hub, its guest compiled into the firmware and its websocket a link into the hub. + Every board serves agent storage from memory, and the guests' storage buckets are that same storage, so an agent on + another machine reads what a guest wrote. Built with `HUB_ADDRESS` naming a hub elsewhere, `address:port`, the + firmware serves nothing and runs its wasi runners against that hub instead, over the board's own sockets. - `et-cli generate-deployment --output-type firmware-pico2` writes a `mise.firmware-pico2.toml`, `firmware-esp32c6` a `mise.firmware-esp32c6.toml`, and `firmware-pico-plus2` a `mise.firmware-pico-plus2.toml`, whose `firmware-flash` task builds that firmware with the scenario's modules in it and loads it onto a board, with picotool or espflash. @@ -46,14 +53,19 @@ Changelog 1.1.0. published artifacts is refused on every board. Its `firmware-host-test` task serves the same modules from the host, and runs the same runners, as the firmware's host build does, and fetches every module it lists, with no board needed. +- A scenario's machine names the hardware it is with `device:`. A board (`pico2`, `esp32c6` or `pico-plus2`) as the + first machine runs the hub as that board's firmware, serving every module of the scenario, with the runners its + firmware hosts beside it, while each other machine runs its share as the format's tasks and reaches the board's hub + at its `address`. A board on any other machine runs only the runners placed on it, built as firmware that serves + nothing and reaches the server's hub at `HUB_ADDRESS`. The `mise.toml` gains `firmware-build-` and + `firmware-flash-` tasks for each board, each built in a target directory of its own, a + `firmware-host-test-` for a board running the hub, and no collector when the server is a board. Only + `mise`, `dekit` and `process-compose` deliver such a scenario. - `et-firmware-modules` generates the source a firmware's build script writes to build ws-modules into its image, from the same `MODULES_*` variables, so every board's firmware embeds modules the same way. Its `board` writes what the board a target picks bakes in -- the Wi-Fi network, and the Pico 2 W's radio blobs and memory layout -- and returns the directives that link it. Its `guests` compiles each runner's guest for the board, refusing one that imports what a board does not host, or guests needing more memory than the board holds for them. -- `et-ws-wasi-runner` builds as `no_std` + `alloc` without its new default `std` feature, and its `board` module - runs a guest on a microcontroller: compiled ahead of time with `engine::precompile` for the board's Pulley target, - and hosted through the firmware's `board::Platform` and `board::Transport`. - Native REST requests retry transient HTTP 408/429/500/502/503/504 responses as well as transport failures, using a bounded backoff policy for replayable idempotent operations. Permission errors, non-idempotent methods and streaming request bodies are not retried. Storage-call timings include the retry/backoff interval; retry attempts are logged. @@ -65,7 +77,10 @@ Changelog 1.1.0. - `et-ws-wasi-runner` builds as `no_std` + `alloc` without its new default `std` feature, and with its new `board` feature its `board` module runs a guest on a microcontroller: compiled ahead of time with `engine::precompile` (the `precompile` feature) for the board's Pulley target, and hosted through the firmware's `board::Platform` and - `board::Transport`. Every guest on a board shares its storage in `board::Buckets`, and draws its memories and + `board::Transport`. The transport is the hub as the guest reaches it, its storage included: a board running the hub + answers from `board::Buckets`, and `board::net::NetTransport` reaches a hub on another machine over the board's own + sockets, its websocket framed by `edge-ws` and its storage over HTTP, while a task of the board's runs + `board::net::drive`; `board::net_tokio` gives it tokio's sockets on a host. Every guest draws its memories and tables from one `board::MemoryBudget`, which refuses growth past what the board holds for them. - An opt-in `et-recovery-supervisor` service observes application progress and coordinates bounded Kubernetes recovery. - The Python SDK provides recovery lifecycle messages and integrity-checked checkpoint storage helpers. @@ -122,9 +137,8 @@ Changelog 1.1.0. ### Changed -- **Breaking:** `et-ws-server`'s actix server, its binary and its environment config are behind the new default `std` - feature. A build with `default-features = false` and none of the features that imply `std` -- `modules`, `otlp`, - `qr`, `storage`, `tls`, `websockify` -- now gets the `no_std` library and no binary; add `std` to keep the server. +- **Breaking:** `et_ws_server::routes::HealthResponse`, the answer to `GET /health`, is now + `et_ws_service::health::HealthResponse`, since a board's hub gives the same answer; import it from there. - **Breaking:** `et-ws-wasi-runner`'s host runner, its binary and `run_module` are behind the new default `std` feature, which `nn`, `webgpu` and `coverage` imply. A build with `default-features = false` and none of them now gets the `no_std` board runner and no binary; add `std` to keep the host runner. `server_message_to_wit` moved from diff --git a/CHECKS.md b/CHECKS.md index 7eeef5fe..8b5005b5 100644 --- a/CHECKS.md +++ b/CHECKS.md @@ -1,11 +1,11 @@ # Checks -- **Generated:** 2026-11-02 +- **Generated:** 2026-11-03 - **Checks:** 82 - **Tools:** 48 - **Shell-script checks:** 10 - **Checks with a repair pass:** 22 -- **Rules:** 361 custom local rules, or non-default strict linter settings +- **Rules:** 363 custom local rules, or non-default strict linter settings - **Meta rules:** 20 of those rules are rules about rules Every check is listed across every `MISE_ENV`, filed under the tool it runs. Each entry names the env whose config @@ -566,8 +566,8 @@ Rules, from `Cargo.toml`: - `conftest-check-policy-tests` (`default` env; run by `conftest-check`) -- Run the Rego unit tests beside the conftest policies (conftest verify). Config: `config/conftest/policy`. - `conftest-check-toml` (`default` env; run by `conftest-check`) -- Run conftest OPA/Rego policies over the TOML - config + lock files. Config: `config/conftest/policy`, `config/deny.toml`, `config/pyrefly.toml`, - `config/upstream-cache/data.toml`. + config + lock files. Config: `config/cargo-features.toml`, `config/conftest/policy`, `config/deny.toml`, + `config/pyrefly.toml`, `config/upstream-cache/data.toml`. - `conftest-check-yaml` (`default` env; run by `conftest-check`) -- Run conftest OPA/Rego policies over the GitHub Actions YAML, and the backslash ban over all YAML. Config: `config/conftest/policy`. @@ -602,6 +602,8 @@ Rules, from `config/conftest/policy`: - crates.io rejects an upload whose manifest carries no description, so every publishable crate has one. - An empty `features = []` on a dependency is pointless noise -- drop it. - A member crate's feature must not share its name with one of its dependencies. + - A member crate's feature must be one config/cargo-features.toml lists. + - config/cargo-features.toml lists no feature that no crate declares any longer. - No manifest may depend on a `cargo_*` / `cargo-*` crate (`cargo_metadata`, `cargo_toml`, ...). - Crates log through `tracing`, so none may depend on the `log` facade directly. - tokio's `full` feature is banned on every dependency spec, the workspace table included. diff --git a/Cargo.lock b/Cargo.lock index 7d96b896..622de6e8 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4676,6 +4676,15 @@ dependencies = [ "wat", ] +[[package]] +name = "edge-ws" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b0540ed0af1b8e98c055b74d87aec4ebd22c1bddb447bb625f5a86559f268bf" +dependencies = [ + "embedded-io-async 0.7.0", +] + [[package]] name = "either" version = "1.19.0" @@ -5841,6 +5850,7 @@ version = "0.1.0" dependencies = [ "chrono", "cortex-m-rt", + "critical-section", "cyw43", "cyw43-pio", "defmt 1.2.0", @@ -5852,6 +5862,7 @@ dependencies = [ "embassy-rp", "embassy-time", "embedded-alloc", + "embedded-io-async 0.7.0", "esp-alloc", "esp-backtrace", "esp-bootloader-esp-idf", @@ -5860,7 +5871,8 @@ dependencies = [ "esp-radio", "esp-rtos", "et-firmware-modules", - "et-ws-server", + "et-modules-service", + "et-storage-service", "et-ws-service", "et-ws-wasi-runner", "fs-err", @@ -5882,12 +5894,14 @@ dependencies = [ "document-features", "edge-toolkit", "fs-err", + "picoserve", "schemars 1.2.3", "serde", "serde-inline-default", "serde_default", "serde_json", "tempfile", + "tokio", "utoipa", ] @@ -6004,6 +6018,7 @@ dependencies = [ "image", "log", "object_store", + "picoserve", "reqwest", "schemars 1.2.3", "serde", @@ -6012,6 +6027,7 @@ dependencies = [ "temp-env", "tempfile", "thiserror 2.0.21", + "tokio", "tracing", "url", "utoipa", @@ -6413,9 +6429,7 @@ dependencies = [ "et-websockify-service", "et-ws-service", "fs-err", - "futures-util", "local-ip-address", - "picoserve", "qr2term", "rcgen", "rustls", @@ -6424,11 +6438,8 @@ dependencies = [ "serde-env", "serde-inline-default", "serde_default", - "serde_json", "tempfile", "thiserror 2.0.21", - "tokio", - "tokio-tungstenite", "tracing", "tracing-actix-web", "tracing-subscriber", @@ -6571,8 +6582,12 @@ version = "0.3.0" dependencies = [ "chrono", "command-error", + "critical-section", "document-features", "edge-toolkit", + "edge-ws", + "embassy-sync 0.8.0", + "embedded-io-async 0.7.0", "et-org", "et-otlp", "et-rest-client", diff --git a/Cargo.toml b/Cargo.toml index e495240a..8c1c6103 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -90,6 +90,7 @@ cloudevents-sdk = "0.9" command-error = "0.8" const-hex = "1.19" cortex-m-rt = "0.7" +critical-section = "1" croner = "4" cyw43 = "0.7" cyw43-pio = "0.10" @@ -105,11 +106,14 @@ deno_resolver = "0.91" deno_runtime = { version = "0.268", features = ["transpile", "hmr"] } document-features = "0.2" edge-toolkit = { path = "libs/edge-toolkit", version = "0.4.0", default-features = false } +edge-ws = { version = "0.8", default-features = false, features = ["io"] } embassy-executor = "0.10" embassy-net = "0.9" embassy-rp = "0.10" +embassy-sync = "0.8" embassy-time = "0.5" embedded-alloc = { version = "0.7", default-features = false, features = ["llff"] } +embedded-io-async = "0.7" esp-alloc = "0.11" esp-backtrace = "0.20" esp-bootloader-esp-idf = "0.6" @@ -122,7 +126,7 @@ et-actix-web-config = { path = "libs/actix-web-config", version = "0.1.0" } et-cli = { path = "utilities/cli", version = "0.2.0" } et-firmware-modules = { path = "services/firmware-modules", version = "0.1.0" } et-hub-backplane = { path = "services/hub-backplane", version = "0.2.0" } -et-modules-service = { path = "services/modules", version = "0.3.0" } +et-modules-service = { path = "services/modules", version = "0.3.0", default-features = false } et-onnx = { path = "utilities/onnx", version = "0.2.0" } et-org = { path = "libs/org", version = "0.2.0" } et-otlp = { path = "libs/et-otlp", version = "0.3.0" } diff --git a/README.md b/README.md index f47a8339..2907a9bf 100644 --- a/README.md +++ b/README.md @@ -443,7 +443,9 @@ MISE_ENV=firmware-pico2 mise run firmware-flash A board has no filesystem, so each module from this repository, or from the scenario's `module_paths:`, is built into the firmware from its built package, and the modules have to be built first. The npm packages they use, such as onnxruntime-web, are bigger than a board's flash: those are redirected to jsDelivr instead. A scenario with more than -one machine or published artifacts is refused. +one machine or published artifacts is refused; one spanning a board and other machines names the board as a machine's +`device:` instead. The board serves agent storage from memory, files of up to 64 KiB, and its own runners' storage is +that same memory, so an agent elsewhere reads what a runner on the board stored. It is lost when the board restarts. The board's firmware decides which runners it can run. Only `firmware-pico-plus2`, for the Pimoroni Pico Plus 2 W, runs any: its 8 MiB of PSRAM holds the memory a guest needs, so it runs each `wasi` runner beside the hub, its guest @@ -467,6 +469,30 @@ machines: agents: ["math1-twin"] ``` +### Machines on different hardware + +A machine's `device:` names the hardware it is, and that hardware decides what the machine may run. Left out, the +machine runs its share as the format's own tasks, with mise and this repository's toolchain on it. Naming a board +(`pico2`, `esp32c6` or `pico-plus2`) makes the machine that board's firmware, which runs only the runners its firmware +hosts. As the first machine, the server, the board also runs the hub, serving every module of the scenario so the +runners elsewhere can fetch theirs, and its agents' storage from memory. On any other machine it serves nothing, and +its runners reach the server's hub at the server's `address`, so a board there needs a runner placed on it. Only +`mise`, `dekit` and `process-compose` deliver a scenario naming a board, since the others ship container images: + +```yaml +machines: + - name: "hub-board" + address: "et-hub-board.local" + device: "pico2" + - name: "runner-board" + address: "et-runner-board.local" + device: "pico-plus2" + agents: ["math1-trigger"] + - name: "edge" + address: "10.0.0.2" + agents: ["pyo3-math1-twin"] +``` + ### Several hubs `services.hub.machines:` runs a hub on each machine it lists, the server among them. The hubs join into one through diff --git a/config/cargo-features.toml b/config/cargo-features.toml new file mode 100644 index 00000000..db5a718e --- /dev/null +++ b/config/cargo-features.toml @@ -0,0 +1,34 @@ +# Every Cargo feature name a crate in this workspace may declare, and what it gives the crate's user. +# +# Maintained by hand, and only by a human. A feature name is part of a crate's public API, so a new one is a +# decision to make deliberately -- named for what it gives the user, such as where the crate runs or what it serves, +# never for a dependency it pulls in -- and the same name means the same thing in every crate. Reuse a name below +# before adding one, add one only with the meaning written beside it, and remove one no crate declares any longer. + +[features] +board = "Runs on a microcontroller board: `no_std` + `alloc`, served or driven over the board's own sockets." +coverage = "Internal: builds in what the repo's coverage workflow measures with, such as minicov or the `/cov` preopen." +cuda = "Runs `wasi:nn` inference on an NVIDIA GPU, through ONNX Runtime's CUDA provider." +default = "Cargo's own: the features a dependent gets unless it sets `default-features = false`." +docs = "Internal: renders the crate's feature list into its documentation, as docs.rs builds it." +env-schema = "Internal: derives JSON Schema for the crate's environment config, which et-int-gen renders into HELP.md." +modules = "Serves ws-module packages from disk under `/modules`, and the root module as the browser UI." +nats = "Shares agents between hubs through the NATS JetStream backplane." +nn = "Hosts `wasi:nn` for guests, running their models through ONNX Runtime." +openapi-spec = "Adds OpenAPI metadata to the crate's HTTP handlers, for et-int-gen to harvest into the published spec." +otel = "Records OpenTelemetry spans and metrics." +otlp = "Exports traces and logs over OTLP." +pico-plus2w = "Builds for the Pimoroni Pico Plus 2 W, on its RP2350B." +pico2w = "Builds for the Raspberry Pi Pico 2 W, on its RP2350A." +precompile = "Compiles guests ahead of time for a board's engine, as a firmware's build script does." +qr = "Prints the advertised URL as a terminal QR code at startup." +rustls-0_23 = "Builds actix-web with rustls 0.23, which adds the TLS handshake timeout setting." +schema-export = "Derives JSON Schema and AsyncAPI descriptions for the WebSocket protocol types." +std = "Builds against the standard library, with everything that needs an operating system." +storage = "Serves per-agent file storage under `/storage`." +tls = "Adds an HTTPS and WSS listener." +tls-self-signed = "Generates a self-signed `localhost` certificate when none is configured." +tracing = "Injects the W3C `traceparent` of the current span into every request." +tty-image = "Renders each stored image as a thumbnail on the server's terminal." +webgpu = "Hosts `wasi:webgpu` for guests, through the host's wgpu device." +websockify = "Relays `/websockify` WebSocket traffic to the hub's own HTTP port, for runtimes that cannot open TCP." diff --git a/config/conftest/policy/cargo/cargo.rego b/config/conftest/policy/cargo/cargo.rego index a8a7b9d0..755af723 100644 --- a/config/conftest/policy/cargo/cargo.rego +++ b/config/conftest/policy/cargo/cargo.rego @@ -327,7 +327,41 @@ deny contains msg if { file.path != "generated/rust-rest/Cargo.toml" some feat, _ in file.contents.features is_dep_name(file, feat) - msg := sprintf("%s: feature %q shares its name with a dependency; rename it", [file.path, feat]) + msg := sprintf("%s: feature %q shares its name with a dependency; %s", [file.path, feat, dep_feature_hint]) +} + +dep_feature_hint := "name it for what it gives the crate's user (where it runs, what it serves), not a dependency" + +# The feature names config/cargo-features.toml allows, which a human keeps. +allowed_features contains name if { + some file in input + file.path == "config/cargo-features.toml" + some name, _ in file.contents.features +} + +# Every feature name a member crate declares. +declared_features contains name if { + some file in input + is_member(file) + some name, _ in file.contents.features +} + +# A member crate's feature must be one config/cargo-features.toml lists. +deny contains msg if { + some file in input + is_member(file) + some feat, _ in file.contents.features + not allowed_features[feat] + msg := sprintf("%s: feature %q is not in config/cargo-features.toml; %s", [file.path, feat, unlisted_feature_hint]) +} + +unlisted_feature_hint := "reuse a listed name, or ask a maintainer to add this one with its meaning" + +# config/cargo-features.toml lists no feature that no crate declares any longer. +deny contains msg if { + some name in allowed_features + not declared_features[name] + msg := $"config/cargo-features.toml: feature \"{name}\" is declared by no crate; remove it" } # No manifest may depend on a `cargo_*` / `cargo-*` crate (`cargo_metadata`, `cargo_toml`, ...). diff --git a/config/conftest/policy/cargo/cargo_test.rego b/config/conftest/policy/cargo/cargo_test.rego index 1d789c3a..22538c26 100644 --- a/config/conftest/policy/cargo/cargo_test.rego +++ b/config/conftest/policy/cargo/cargo_test.rego @@ -292,3 +292,34 @@ test_generated_rest_client_is_exempt if { msgs := cargo.deny with input as member("generated/rust-rest/Cargo.toml", {"features": {"tracing": []}}) feature_docs_clean(msgs) } + +# A member declaring `features`, beside an allowlist naming `allowed`. +with_allowlist(features, allowed) := [ + {"path": "libs/a/Cargo.toml", "contents": {"features": features}}, + {"path": "config/cargo-features.toml", "contents": {"features": allowed}}, +] + +test_unlisted_feature_is_rejected if { + msgs := cargo.deny with input as with_allowlist({"board": []}, {"std": "x"}) + some msg in msgs + startswith(msg, "libs/a/Cargo.toml: feature \"board\" is not in config/cargo-features.toml; reuse a listed name") +} + +test_listed_feature_is_accepted if { + msgs := cargo.deny with input as with_allowlist({"board": []}, {"board": "x"}) + every msg in msgs { + not contains(msg, "cargo-features.toml") + } +} + +test_stale_allowlist_entry_is_rejected if { + msgs := cargo.deny with input as with_allowlist({"board": []}, {"board": "x", "std": "x"}) + "config/cargo-features.toml: feature \"std\" is declared by no crate; remove it" in msgs +} + +test_feature_named_for_a_dependency_is_rejected if { + contents := {"features": {"picoserve": ["dep:picoserve"]}, "dependencies": {"picoserve": {"optional": true}}} + msgs := cargo.deny with input as member("libs/a/Cargo.toml", contents) + some msg in msgs + contains(msg, "feature \"picoserve\" shares its name with a dependency; name it for what it gives") +} diff --git a/config/jscpd-baseline.json b/config/jscpd-baseline.json index c3553f23..c3d1425b 100644 --- a/config/jscpd-baseline.json +++ b/config/jscpd-baseline.json @@ -10,7 +10,6 @@ "03e4e07acff327e2": 1, "04199950b57990a0": 1, "04daef50421ed235": 1, - "05f89321b4dbba3a": 1, "0690b4ae1dc710a4": 1, "07378241fe562817": 1, "075c84a635548077": 1, @@ -43,8 +42,6 @@ "1a1e1611e4db4656": 1, "1a46175d923bf850": 3, "1a97e5c72a30152e": 1, - "1b8894933d283827": 1, - "1bd6f5989d1961e5": 1, "1c64414236b8476c": 1, "1c8cf13e10b67513": 1, "1ce17b70419170cb": 1, @@ -60,7 +57,6 @@ "264e155c8fdaf095": 1, "265b3a5011a42fc4": 1, "266cd9babed7be0c": 1, - "268a187843c419fa": 1, "26bec4de311d99cf": 1, "2c47cf795be8c0d9": 1, "2c61d8307488c819": 1, @@ -77,7 +73,6 @@ "34b2a7c1a1a932c1": 1, "34cd0de3c97e4264": 1, "35d7f304546de8a8": 1, - "373a54c3fd093400": 1, "378e3311bed8161a": 1, "389aea3e8662fb9b": 1, "39df8b0c7c0dccc1": 1, @@ -91,8 +86,8 @@ "3de59029c7b786b0": 1, "3e5aadc3e77078b6": 1, "3e6e51f257b8453d": 1, + "4259948cb662c568": 1, "4268c1cf96611f46": 1, - "439a73019944af1b": 1, "43f0e3fca21daafa": 1, "448a09ade148d8c5": 1, "4534795197b3c34c": 1, @@ -106,10 +101,10 @@ "506af004dd69d586": 1, "50b373907a5a8f78": 1, "50f5f12046dd3030": 1, - "51c917fb5e31c2b5": 1, "51d9869c79c352cd": 1, "521c8a862b4eacb3": 1, "533bdecb47a87756": 1, + "5356167d8b24069f": 1, "544d0bff2e6dd595": 1, "54b576854edda79b": 1, "54f0cec8830d66e4": 1, @@ -144,6 +139,7 @@ "635fb32c899e4df6": 1, "644491f2c34618c6": 1, "6523544c604ca05c": 1, + "65b46fbd1e1ee947": 1, "6609674842b23010": 1, "683f90ab64c941ba": 1, "688189c9d78d110e": 1, @@ -175,6 +171,7 @@ "781cc393068d5440": 1, "7ac63870d0087e3b": 1, "7aee849465ea91ab": 1, + "7ef0fbdc607ad59c": 1, "7f343488c1aacd37": 1, "7f700d1f7681c95e": 1, "7fa442e9e2160278": 1, @@ -192,14 +189,17 @@ "87566acb3f2550bf": 1, "879bfe322b3d676b": 1, "87f1eade46af4d27": 1, + "8815b916d90a59c6": 1, "88bca8b73d89f892": 1, "8996a093b7bb3b54": 1, "89bb6ca7b46aa5cd": 1, "8acea4476f04e4a0": 2, + "8b0371fedfb54c9c": 1, "8b5b2be949793716": 1, "8c912ef640675113": 1, "8d739440b2cad0b0": 1, "8f29fa9325c6a7fa": 1, + "8fb32f671142f0b4": 1, "8fcac3476268b171": 1, "928c39b341599636": 1, "92cbe2a88c46dd90": 6, @@ -212,6 +212,7 @@ "96fdb2ccaf1ea7d5": 1, "985af6ffa52d3359": 1, "9868e80d502c44d3": 1, + "98d7f31afdd9cdff": 1, "98dab08a54b05abf": 1, "98e1331244999da4": 1, "98f11864904716c0": 1, @@ -221,7 +222,6 @@ "9cb0f337a422df56": 1, "9e7af9bc8ddddb59": 1, "9e835c94361d6917": 1, - "9ec9ce10070abf0a": 1, "9ed30a2d34c205ae": 1, "a03ab32ade9684de": 1, "a0438c241cc4a042": 1, @@ -243,7 +243,6 @@ "a743e181c2f6424b": 1, "a896f65e46666bd0": 1, "a92c94299aa8f156": 1, - "a9b99a8f3b63f49d": 1, "a9f793157ec6fced": 1, "aa68ceead4a59e5f": 1, "aad9806c62fff62f": 1, @@ -271,6 +270,7 @@ "bab02c8fb19dca60": 1, "bbd85a05647fdf5e": 1, "bbda58a7774990a3": 1, + "bc23880652f9e228": 1, "bd0a27d6a9ee4b87": 1, "bd684492ef8fdd33": 1, "c01708e08c252d4a": 1, @@ -289,7 +289,6 @@ "c6ad80475453274a": 1, "c6e1e53b3f01fafb": 1, "c713bd192586f25f": 1, - "c7bf05bb2529b46b": 1, "c889a14a7d34fe32": 1, "c8aa41bf2df4989a": 1, "c8f96481b232f2ca": 1, @@ -318,7 +317,6 @@ "da7b27ae8c89cb73": 1, "da812f4e54fefbf1": 1, "db4fb9ed218e4d66": 1, - "dd101dfd2d94984f": 1, "dd162080de996936": 1, "dd41701c3663b9f3": 1, "ddca47a4caded0fd": 1, diff --git a/services/firmware-modules/src/lib.rs b/services/firmware-modules/src/lib.rs index 0239ba9a..af9a7ff2 100644 --- a/services/firmware-modules/src/lib.rs +++ b/services/firmware-modules/src/lib.rs @@ -30,7 +30,7 @@ const PACKAGE_DIR: &str = "pkg"; /// The manifest naming a module, and listing its files. const PACKAGE_JSON: &str = "package.json"; /// Where the types the source builds its modules from live, named in full so a build with none imports nothing. -const MODULES_API: &str = "et_ws_server::picoserve_modules"; +const MODULES_API: &str = "et_modules_service::picoserve"; /// The variable the module directories come from, which errors about them name. const MODULES_PATHS: &str = "MODULES_PATHS"; /// The variable naming the runners a board starts and the modules they run, which errors about them name. diff --git a/services/firmware-modules/tests/generate.rs b/services/firmware-modules/tests/generate.rs index e8069f2b..d47550cf 100644 --- a/services/firmware-modules/tests/generate.rs +++ b/services/firmware-modules/tests/generate.rs @@ -90,11 +90,10 @@ fn it_builds_in_what_each_package_publishes() { let source = &generated.source; assert!(source.contains(r#"Module::new("@scope/built", &["#)); assert!( - source.contains(r#"("assets/model.wasm", et_ws_server::picoserve_modules::ModuleFile::new("application/wasm""#) + source.contains(r#"("assets/model.wasm", et_modules_service::picoserve::ModuleFile::new("application/wasm""#) ); assert!( - source - .contains(r#"("index.html", et_ws_server::picoserve_modules::ModuleFile::new("text/html; charset=utf-8""#) + source.contains(r#"("index.html", et_modules_service::picoserve::ModuleFile::new("text/html; charset=utf-8""#) ); assert!(source.contains(r#"Module::new("plain", &["#)); assert!(source.contains(" \"@scope/built\",\n")); @@ -145,11 +144,11 @@ fn each_file_is_served_as_its_extension_says() { let source = generated(&root.path().join("m").display().to_string(), "", "", "").source; for (extension, content_type) in extensions { assert!(source.contains(&format!( - r#"("file.{extension}", et_ws_server::picoserve_modules::ModuleFile::new("{content_type}""# + r#"("file.{extension}", et_modules_service::picoserve::ModuleFile::new("{content_type}""# ))); } assert!( - source.contains(r#"("LICENSE", et_ws_server::picoserve_modules::ModuleFile::new("application/octet-stream""#) + source.contains(r#"("LICENSE", et_modules_service::picoserve::ModuleFile::new("application/octet-stream""#) ); } diff --git a/services/modules/Cargo.toml b/services/modules/Cargo.toml index 1c353b05..276871f3 100644 --- a/services/modules/Cargo.toml +++ b/services/modules/Cargo.toml @@ -14,30 +14,73 @@ rustdoc-args = ["--cfg", "docsrs"] [lib] doctest = false +[[test]] +name = "api_modules" +required-features = ["std"] + +[[test]] +name = "configure_missing_root" +required-features = ["std"] + +[[test]] +name = "declared_names" +required-features = ["std"] + +[[test]] +name = "get_module_file" +required-features = ["std"] + +[[test]] +name = "picoserve" +required-features = ["board"] + +[[test]] +name = "symlinks" +required-features = ["std"] + [features] +default = ["std"] +## The actix-web service that discovers module packages on disk and serves their files. +## Without it the crate is `no_std` + `alloc`, and serves modules built into the binary under `picoserve`. +std = [ + "dep:actix-files", + "dep:actix-web", + "dep:fs-err", + "dep:serde", + "dep:serde-inline-default", + "dep:serde_default", + "dep:serde_json", + "edge-toolkit/std", +] +## Serves ws-module packages built into the binary through picoserve, on a board with no filesystem. +board = ["dep:picoserve"] ## Adds OpenAPI metadata to the module-discovery handlers for `et-int-gen` to harvest into the published spec. -openapi-spec = ["dep:utoipa"] +openapi-spec = ["dep:utoipa", "std"] ## Internal: derives JSON Schema for `ModulesConfig`, which et-int-gen renders into the hub's HELP.md. -env-schema = ["dep:schemars"] +env-schema = ["dep:schemars", "std"] ## Internal: renders this feature list into the crate documentation, as docs.rs builds it. docs = ["dep:document-features"] [dependencies] -actix-files.workspace = true -actix-web.workspace = true +actix-files = { workspace = true, optional = true } +actix-web = { workspace = true, optional = true } document-features = { workspace = true, optional = true } -edge-toolkit = { workspace = true, features = ["std"] } -fs-err.workspace = true +edge-toolkit.workspace = true +fs-err = { workspace = true, optional = true } +picoserve = { workspace = true, features = ["json"], optional = true } schemars = { workspace = true, optional = true } -serde = { workspace = true, features = ["std"] } -serde-inline-default.workspace = true -serde_default.workspace = true -serde_json = { workspace = true, features = ["std"] } +serde = { workspace = true, features = ["std"], optional = true } +serde-inline-default = { workspace = true, optional = true } +serde_default = { workspace = true, optional = true } +serde_json = { workspace = true, features = ["std"], optional = true } utoipa = { workspace = true, optional = true } [dev-dependencies] actix-rt.workspace = true +picoserve = { workspace = true, features = ["tokio"] } +serde_json = { workspace = true, features = ["std"] } tempfile.workspace = true +tokio = { workspace = true, features = ["io-util", "macros", "net", "rt", "time"] } [lints] workspace = true diff --git a/services/modules/src/lib.rs b/services/modules/src/lib.rs index 30fbc52c..eeccf2d3 100644 --- a/services/modules/src/lib.rs +++ b/services/modules/src/lib.rs @@ -2,22 +2,39 @@ #![cfg_attr(feature = "docs", doc = "## Feature flags")] #![cfg_attr(feature = "docs", doc = document_features::document_features!())] #![cfg_attr(docsrs, feature(doc_cfg))] +#![cfg_attr(not(feature = "std"), no_std)] +#[cfg(all(feature = "board", not(feature = "std")))] +extern crate alloc; + +#[cfg(feature = "std")] use std::path::PathBuf; +#[cfg(feature = "std")] use actix_files::Files; +#[cfg(feature = "std")] use actix_web::web; +#[cfg(feature = "std")] use edge_toolkit::config::default_modules_folders; +#[cfg(feature = "std")] use fs_err as fs; +#[cfg(feature = "std")] use serde::Deserialize; +#[cfg(feature = "std")] use serde_default::DefaultFromSerde; +#[cfg(feature = "std")] use serde_inline_default::serde_inline_default; +#[cfg(feature = "board")] +pub mod picoserve; +#[cfg(feature = "std")] pub mod routes; +#[cfg(feature = "std")] pub use self::routes::list_modules_handler; /// Modules config. +#[cfg(feature = "std")] #[serde_inline_default] #[derive(Clone, Debug, DefaultFromSerde, Deserialize)] #[cfg_attr(feature = "env-schema", derive(schemars::JsonSchema))] @@ -55,6 +72,7 @@ pub struct ModulesConfig { pub mise_discover: bool, } +#[cfg(feature = "std")] impl ModulesConfig { /// Config that serves exactly `paths`, which is what a caller naming directories outright wants. #[must_use] @@ -68,6 +86,7 @@ impl ModulesConfig { } /// The package name a `package.json` declares, exactly as written. +#[cfg(feature = "std")] fn read_package_name(package_json: &std::path::Path) -> Option { let content = fs::read_to_string(package_json).ok()?; let value: serde_json::Value = serde_json::from_str(&content).ok()?; @@ -75,6 +94,7 @@ fn read_package_name(package_json: &std::path::Path) -> Option { } /// Scan all configured module paths and return a sorted list of `(name, pkg_dir)` pairs. +#[cfg(feature = "std")] #[must_use] pub fn list_modules(config: &ModulesConfig) -> Vec<(String, PathBuf)> { let mut modules: Vec<(String, PathBuf)> = Vec::new(); @@ -141,6 +161,7 @@ pub fn list_modules(config: &ModulesConfig) -> Vec<(String, PathBuf)> { /// Panics if `config.root` names a module none of the scanned `config.paths` provides -- naming a front page /// that isn't there is a config error, and it is fatal early so the operator sees it at startup rather than /// as a 404 much later. +#[cfg(feature = "std")] #[expect( clippy::panic, reason = "a root module named but absent is a config error; failing fast at startup is intentional" diff --git a/services/ws-server/src/picoserve_modules.rs b/services/modules/src/picoserve.rs similarity index 90% rename from services/ws-server/src/picoserve_modules.rs rename to services/modules/src/picoserve.rs index 1f338607..3a952a41 100644 --- a/services/ws-server/src/picoserve_modules.rs +++ b/services/modules/src/picoserve.rs @@ -1,10 +1,10 @@ -//! ws-module packages built into the binary, served through picoserve where the `std` server serves them from disk. +//! ws-module packages built into the binary, served through picoserve where the `std` service serves them from disk. //! -//! A board has no filesystem to scan, so its embedder hands [`serve`](crate::picoserve_server::serve) a [`Modules`] -//! whose every file is a `&'static [u8]`, usually through `include_bytes!` in generated code. Each answers a matching -//! `If-None-Match` with a 304, so a browser reloading the page does not fetch its wasm from the board again. The routes -//! are the `std` server's: `GET /modules/` lists the module names, `GET /modules/{name}/{path}` is one file of one -//! module, and the root module's files are served from `/`, `index.html` answering `/` itself. +//! A board has no filesystem to scan, so its embedder mounts [`ModuleRoutes`] over a [`Modules`] whose every file is a +//! `&'static [u8]`, usually through `include_bytes!` in generated code. Each answers a matching `If-None-Match` with a +//! 304, so a browser reloading the page does not fetch its wasm from the board again. The routes are the `std` +//! service's: `GET /modules/` lists the module names, `GET /modules/{name}/{path}` is one file of one module, and the +//! root module's files are served from `/`, `index.html` answering `/` itself. //! //! A package too big for the board's flash can be listed as served [`elsewhere`](Modules::elsewhere) instead: a request //! for one of its files is redirected to the same path under [`elsewhere_url`](Modules::elsewhere_url), such as a CDN @@ -20,7 +20,7 @@ use picoserve::response::{Content, IntoResponse as _, Json, Response, ResponseWr use picoserve::routing::{MethodNotAllowed, NotFound, PathRouter as _, PathRouterService, RequestHandler as _}; use picoserve::url_encoded::UrlEncodedString; -/// Where module files are served under, as the `std` server's modules service mounts them. +/// Where module files are served under, as the `std` service mounts them. const MODULES_PREFIX: &str = "/modules"; /// The file that answers a request for the root module's `/`. const INDEX_FILE: &str = "index.html"; @@ -147,7 +147,7 @@ impl Modules { } } - /// Every module name, embedded or not, sorted as the `std` server lists them. + /// Every module name, embedded or not, sorted as the `std` service lists them. fn names(&self) -> Vec<&'static str> { let mut names: Vec<&'static str> = self.embedded.iter().map(|module| module.name).collect(); names.extend_from_slice(self.elsewhere); @@ -183,16 +183,14 @@ impl Modules { } } -/// The routes [`Modules`] answers, as the fallback of a server's other routes: a request none of them matched. -pub(crate) struct ModuleRoutes<'modules>(&'modules Modules); +/// The routes [`Modules`] answers, mounted as the fallback of a server's other routes: a request none of them matched. +#[derive(Clone, Copy, Debug)] +pub struct ModuleRoutes<'modules>(&'modules Modules); impl<'modules> ModuleRoutes<'modules> { /// The routes `modules` answers. - #[expect( - clippy::single_call_fn, - reason = "the constructor that keeps the field private to this module, for the one server that mounts it" - )] - pub(crate) const fn new(modules: &'modules Modules) -> Self { + #[must_use] + pub const fn new(modules: &'modules Modules) -> Self { Self(modules) } } diff --git a/services/ws-server/tests/picoserve_server.rs b/services/modules/tests/picoserve.rs similarity index 57% rename from services/ws-server/tests/picoserve_server.rs rename to services/modules/tests/picoserve.rs index dc06fd48..3465a51d 100644 --- a/services/ws-server/tests/picoserve_server.rs +++ b/services/modules/tests/picoserve.rs @@ -1,27 +1,16 @@ -//! The `no_std` ws-server over real TCP sockets, with tokio standing in for an embedded executor and network stack: -//! `/health` and `/favicon.ico` answer as the `std` server's do, `/ws` is the hub, and the modules built into the -//! binary are served from `/modules` and `/`. +//! Modules built into the binary, served through picoserve over real TCP sockets, with tokio standing in for an +//! embedded executor and network stack: listed under `/modules/`, each file under `/modules/{name}/`, the root module's +//! from `/`, and a module served elsewhere redirected there. #![cfg(test)] -#![recursion_limit = "256"] use std::net::{Ipv4Addr, SocketAddr}; -use std::sync::Arc; use std::time::Duration; -use edge_toolkit::ws::{ClientMessage, ConnectStatus, ServerMessage}; -use edge_toolkit::ws_server::MessageRetention; -use et_ws_server::picoserve_modules::{Module, ModuleFile, Modules}; -use et_ws_server::picoserve_server::serve; -use et_ws_server::routes::HealthResponse; -use et_ws_service::hub::Stamps; -use et_ws_service::picoserve_hub::{Hub, config}; -use futures_util::{SinkExt as _, StreamExt as _}; -use picoserve::time::TokioTimer; +use et_modules_service::picoserve::{Module, ModuleFile, ModuleRoutes, Modules}; +use picoserve::{Config, Router, Server}; use tokio::io::{AsyncReadExt as _, AsyncWriteExt as _}; use tokio::net::{TcpListener, TcpStream}; use tokio::task::LocalSet; -use tokio_tungstenite::connect_async; -use tokio_tungstenite::tungstenite::Message; /// How long a test waits for an answer before failing rather than hanging. const PATIENCE: Duration = Duration::from_secs(5); @@ -49,58 +38,22 @@ static MODULES: Modules = Modules::new( "https://cdn.example/npm", ); -/// Ids `id-1`, `id-2`, ... in turn, and a fixed time. -#[derive(Default)] -struct Counter(u32); - -impl Stamps for Counter { - fn new_id(&mut self) -> String { - self.0 = self.0.saturating_add(1); - format!("id-{}", self.0) - } - - fn now(&mut self) -> String { - "2026-10-30T00:00:00Z".to_owned() - } -} - -/// The server of `modules` on a fresh loopback port, one task per connection as an embedder would, and its address. +/// `modules` served on a fresh loopback port, one task per connection as an embedder would, and its address. async fn served(modules: &'static Modules) -> SocketAddr { - let hub = Arc::new(Hub::new(Counter::default(), MessageRetention::All, 1024, 16)); let listener = TcpListener::bind((Ipv4Addr::LOCALHOST, 0)).await.unwrap(); let address = listener.local_addr().unwrap(); let _accepting = tokio::task::spawn_local(async move { + let app = Router::from_service(ModuleRoutes::new(modules)); + let config = Config::const_default(); loop { - let (socket, peer) = listener.accept().await.unwrap(); - let hub = Arc::clone(&hub); - let _serving = tokio::task::spawn_local(async move { - let mut http_buffer = [0; 2048]; - let _served = serve( - &hub, - modules, - socket, - peer.ip(), - TokioTimer, - &config(), - &mut http_buffer, - ) - .await; - }); + let (socket, _peer) = listener.accept().await.unwrap(); + let mut http_buffer = [0; 2048]; + let _served = Server::new_tokio(&app, &config, &mut http_buffer).serve(socket).await; } }); address } -/// The whole response to a bodiless `GET path`, headers and body, as text. -async fn get(address: SocketAddr, path: &str) -> String { - request(address, "GET", path).await -} - -/// The whole response to a bodiless `method path`, headers and body, as text. -async fn request(address: SocketAddr, method: &str, path: &str) -> String { - send(address, &format!("{method} {path} HTTP/1.1\r\nHost: hub\r\n")).await -} - /// The whole response to `head`, a request line and headers to which the closing ones are added. async fn send(address: SocketAddr, head: &str) -> String { let mut socket = TcpStream::connect(address).await.unwrap(); @@ -114,39 +67,14 @@ async fn send(address: SocketAddr, head: &str) -> String { response } -#[tokio::test] -async fn it_serves_health_favicon_and_the_hub() { - // Boxed because its size depends on how the build unifies features: over 16 KiB with the whole workspace, under it - // alone, so an expectation of `clippy::large_futures` could hold in only one of the two. - LocalSet::new() - .run_until(Box::pin(async { - let address = served(&MODULES).await; +/// The whole response to a bodiless `method path`. +async fn request(address: SocketAddr, method: &str, path: &str) -> String { + send(address, &format!("{method} {path} HTTP/1.1\r\nHost: hub\r\n")).await +} - let health = get(address, "/health").await; - assert!(health.starts_with("HTTP/1.1 200")); - assert!(health.contains("Content-Type: application/json")); - let (_head, body) = health.split_once("\r\n\r\n").unwrap(); - let body: HealthResponse = serde_json::from_str(body).unwrap(); - assert_eq!((body.status.as_str(), body.service.as_str()), ("healthy", "ws-server")); - - assert!(get(address, "/favicon.ico").await.starts_with("HTTP/1.1 204")); - assert!(get(address, "/elsewhere").await.starts_with("HTTP/1.1 404")); - - let (mut agent, _response) = connect_async(format!("ws://{address}/ws")).await.unwrap(); - let connect = ClientMessage::Connect { agent_id: None }; - let frame = Message::text(serde_json::to_string(&connect).unwrap()); - agent.send(frame).await.unwrap(); - let ack = tokio::time::timeout(PATIENCE, agent.next()) - .await - .unwrap() - .unwrap() - .unwrap(); - assert!(matches!( - ServerMessage::from_text_frame(ack.to_text().unwrap()).unwrap(), - ServerMessage::ConnectAck { status: ConnectStatus::Assigned, agent_id } if agent_id == "id-1" - )); - })) - .await; +/// The whole response to a `GET path`. +async fn get(address: SocketAddr, path: &str) -> String { + request(address, "GET", path).await } /// The body of `response`, after its head. @@ -186,22 +114,18 @@ async fn it_serves_the_embedded_modules() { assert_eq!(body(&changed), "\0asm"); let encoded = get(address, "/modules/%40scope/math/package.json").await; assert_eq!(body(&encoded), "{}"); - assert!( - get(address, "/modules/@scope/math/missing.js") - .await - .starts_with("HTTP/1.1 404") - ); - assert!( - get(address, "/modules/@scope/mathematics/x") - .await - .starts_with("HTTP/1.1 404") - ); - assert!(get(address, "/modulesx").await.starts_with("HTTP/1.1 404")); + for missing in [ + "/modules/@scope/math/missing.js", + "/modules/@scope/mathematics/x", + "/modulesx", + "/modules/bigger/x.js", + ] { + assert!(get(address, missing).await.starts_with("HTTP/1.1 404")); + } let redirect = get(address, "/modules/big/dist/big.js").await; assert!(redirect.starts_with("HTTP/1.1 307")); assert!(redirect.contains("Location: https://cdn.example/npm/big/dist/big.js\r\n")); - assert!(get(address, "/modules/bigger/x.js").await.starts_with("HTTP/1.1 404")); assert!(request(address, "POST", "/app.js").await.starts_with("HTTP/1.1 405")); assert!(request(address, "POST", "/missing").await.starts_with("HTTP/1.1 404")); @@ -228,3 +152,8 @@ async fn modules_assembled_at_run_time_serve_as_built_in_ones_do() { })) .await; } + +#[test] +fn no_modules_serve_nothing() { + assert!(Modules::NONE.embedded.is_empty() && Modules::NONE.root.is_empty()); +} diff --git a/services/storage/Cargo.toml b/services/storage/Cargo.toml index 99e2c959..f75bc332 100644 --- a/services/storage/Cargo.toml +++ b/services/storage/Cargo.toml @@ -19,40 +19,81 @@ ignored = ["log", "serde_json"] [lib] doctest = false +[[test]] +name = "blocked" +required-features = ["std"] + +[[test]] +name = "image_logging" +required-features = ["std"] + +[[test]] +name = "picoserve" +required-features = ["board"] + +[[test]] +name = "put" +required-features = ["std"] + +[[test]] +name = "s3_backend" +required-features = ["std"] + [[test]] name = "tty_render" required-features = ["tty-image"] [features] -default = ["tty-image"] +default = ["std", "tty-image"] +## The actix-web service, storing each agent's files in any `object_store` backend. +## Without it the crate is `no_std` + `alloc`, and serves storage the embedder keeps under `picoserve`. +std = [ + "dep:actix-web", + "dep:actix-web-thiserror", + "dep:et-hub-backplane", + "dep:fs-err", + "dep:futures-util", + "dep:log", + "dep:object_store", + "dep:serde", + "dep:serde_default", + "dep:serde_json", + "dep:thiserror", + "dep:tracing", + "dep:url", + "edge-toolkit/std", +] +## Serves agent storage through picoserve, kept wherever the embedder keeps it, on a board with no filesystem. +board = ["dep:picoserve"] ## Adds OpenAPI metadata to the `/storage` handlers for `et-int-gen` to harvest into the published spec. -openapi-spec = ["dep:utoipa"] +openapi-spec = ["dep:utoipa", "std"] ## Renders each stored image as a thumbnail on the server's terminal, e.g. what pic-viewer uploads. ## Without it a stored image is still logged by name and size, and the `image` decoder stays out of the build. -tty-image = ["dep:image"] +tty-image = ["dep:image", "std"] ## Internal: derives JSON Schema for `StorageConfig`, which et-int-gen renders into the hub's HELP.md. -env-schema = ["dep:schemars"] +env-schema = ["dep:schemars", "std"] ## Internal: renders this feature list into the crate documentation, as docs.rs builds it. docs = ["dep:document-features"] [dependencies] -actix-web.workspace = true -actix-web-thiserror.workspace = true +actix-web = { workspace = true, optional = true } +actix-web-thiserror = { workspace = true, optional = true } document-features = { workspace = true, optional = true } -edge-toolkit = { workspace = true, features = ["std"] } -et-hub-backplane.workspace = true -fs-err.workspace = true -futures-util.workspace = true +edge-toolkit.workspace = true +et-hub-backplane = { workspace = true, optional = true } +fs-err = { workspace = true, optional = true } +futures-util = { workspace = true, optional = true } image = { workspace = true, optional = true } -log.workspace = true -object_store.workspace = true +log = { workspace = true, optional = true } +object_store = { workspace = true, optional = true } +picoserve = { workspace = true, optional = true } schemars = { workspace = true, optional = true } -serde = { workspace = true, features = ["std"] } -serde_default.workspace = true -serde_json = { workspace = true, features = ["std"] } -thiserror = { workspace = true, features = ["std"] } -tracing = { workspace = true, features = ["std"] } -url.workspace = true +serde = { workspace = true, features = ["std"], optional = true } +serde_default = { workspace = true, optional = true } +serde_json = { workspace = true, features = ["std"], optional = true } +thiserror = { workspace = true, features = ["std"], optional = true } +tracing = { workspace = true, features = ["std"], optional = true } +url = { workspace = true, optional = true } utoipa = { workspace = true, optional = true } [dev-dependencies] @@ -61,9 +102,11 @@ actix-rt.workspace = true command-error.workspace = true et-test-helpers.workspace = true object_store.workspace = true +picoserve = { workspace = true, features = ["tokio"] } reqwest.workspace = true temp-env = { workspace = true, features = ["async_closure"] } tempfile.workspace = true +tokio = { workspace = true, features = ["io-util", "macros", "net", "rt", "time"] } [lints] workspace = true diff --git a/services/storage/src/lib.rs b/services/storage/src/lib.rs index 9649b9a1..5a86440c 100644 --- a/services/storage/src/lib.rs +++ b/services/storage/src/lib.rs @@ -2,25 +2,43 @@ #![cfg_attr(feature = "docs", doc = "## Feature flags")] #![cfg_attr(feature = "docs", doc = document_features::document_features!())] #![cfg_attr(docsrs, feature(doc_cfg))] +#![cfg_attr(not(feature = "std"), no_std)] +#[cfg(all(feature = "board", not(feature = "std")))] +extern crate alloc; + +#[cfg(feature = "std")] use std::path::PathBuf; +#[cfg(feature = "std")] use std::sync::{Arc, PoisonError}; +#[cfg(feature = "std")] use actix_web::web; +#[cfg(feature = "std")] use actix_web_thiserror::ResponseError; +#[cfg(feature = "std")] use object_store::ObjectStore; +#[cfg(feature = "std")] use object_store::local::LocalFileSystem; +#[cfg(feature = "std")] use serde::Deserialize; +#[cfg(feature = "std")] use serde_default::DefaultFromSerde; +#[cfg(feature = "std")] use thiserror::Error; +#[cfg(feature = "board")] +pub mod picoserve; +#[cfg(feature = "std")] pub mod routes; #[cfg(feature = "tty-image")] pub mod tty_image; +#[cfg(feature = "std")] pub use self::routes::{get_file, head_file, put_file}; /// Default storage directory. +#[cfg(feature = "std")] #[must_use] pub fn default_storage_folder() -> PathBuf { let project_root = edge_toolkit::config::get_project_root(); @@ -31,6 +49,7 @@ pub fn default_storage_folder() -> PathBuf { /// /// This is the default baked into [`StorageConfig::url`], so the local-disk backend is expressed the same way /// as any other -- as a URL -- rather than being inferred later from an absent field. +#[cfg(feature = "std")] #[must_use] pub fn default_storage_url() -> String { file_url(&default_storage_folder()) @@ -41,12 +60,14 @@ pub fn default_storage_url() -> String { /// `Url::from_directory_path` only fails for a non-absolute path, and every caller here passes an absolute one /// (the project root, or a tempdir). The fallback keeps this infallible rather than forcing a `Result` through /// serde's default hook, and is not reachable for an absolute input on any platform. +#[cfg(feature = "std")] #[must_use] pub fn file_url(path: &std::path::Path) -> String { url::Url::from_directory_path(path).map_or_else(|()| format!("file://{}", path.display()), String::from) } /// Storage config. +#[cfg(feature = "std")] #[derive(Clone, Debug, DefaultFromSerde, Deserialize)] #[cfg_attr(feature = "env-schema", derive(schemars::JsonSchema))] #[non_exhaustive] @@ -64,6 +85,7 @@ pub struct StorageConfig { pub url: String, } +#[cfg(feature = "std")] impl StorageConfig { /// Build a config for an explicit backend URL. #[must_use] @@ -79,8 +101,10 @@ impl StorageConfig { } /// Shared handle to the configured backend, held in actix app data and by the routes. +#[cfg(feature = "std")] pub type SharedStore = Arc; +#[cfg(feature = "std")] #[derive(Debug, Error, ResponseError)] #[non_exhaustive] pub enum StorageError { @@ -134,6 +158,7 @@ pub enum StorageError { // `PoisonError` is generic over the guard type; a generic `From` impl // lets `?` drop the `T` and surface the variant directly. +#[cfg(feature = "std")] impl From> for StorageError { fn from(_source: PoisonError) -> Self { Self::AgentRegistryPoisoned @@ -146,6 +171,7 @@ impl From> for StorageError { /// environment variables (a mounted Kubernetes secret, say) instead of being embedded in the URL. `object_store` /// errors clearly if the URL names a backend whose feature is not compiled in -- currently only `aws`, plus the /// always-present `file` and `memory`. +#[cfg(feature = "std")] pub fn build_store(config: &StorageConfig) -> Result { let parsed = url::Url::parse(&config.url)?; @@ -179,6 +205,7 @@ pub fn build_store(config: &StorageConfig) -> Result /// Panics if `config` describes a backend that cannot be constructed (an unparsable URL, a scheme whose /// backend is not compiled in, or an unusable local path). That is a misconfigured deployment rather than a /// runtime condition, so it fails at startup instead of turning every later request into a 500. +#[cfg(feature = "std")] #[expect( clippy::panic, reason = "an unbuildable backend is a misconfigured deployment; fail at startup, not on every request" diff --git a/services/storage/src/picoserve.rs b/services/storage/src/picoserve.rs new file mode 100644 index 00000000..47629abc --- /dev/null +++ b/services/storage/src/picoserve.rs @@ -0,0 +1,177 @@ +//! Agent storage through picoserve, as the `std` service answers `/storage/{agent_id}/{filename}`, kept wherever the +//! embedder's [`Storage`] keeps it. +//! +//! A board has no filesystem, so the embedder hands in the store, usually the memory its own runners' storage buckets +//! are in, so that an agent on another machine reads what a runner on the board wrote, and the other way round. It +//! mounts [`StorageRoutes`] at [`path`] on its router. + +#[cfg(not(feature = "std"))] +use alloc::{string::String, vec, vec::Vec}; +use core::net::IpAddr; +use core::str::FromStr; + +use edge_toolkit::ws_server::AgentRegistry; +use picoserve::ResponseSent; +use picoserve::io::Read; +use picoserve::request::Request; +use picoserve::response::{IntoResponse as _, Response, ResponseWriter, StatusCode}; +use picoserve::routing::{MethodHandlerService, ParsePathSegment, parse_path_segment}; + +/// The largest file an agent may store, which bounds what one upload takes of a board's heap. +pub const MAX_STORED: usize = 64 * 1024; + +/// Where a board keeps agents' storage: a bucket per agent id, each a map from filename to contents. +/// +/// Shared by every connection the server answers at once, so a store guards its own contents. +pub trait Storage { + /// What `agent_id`'s bucket holds under `filename`. + /// + /// # Errors + /// + /// [`NotStored`] when the bucket holds nothing under that name. + fn get(&self, agent_id: &str, filename: &str) -> Result, NotStored>; + + /// Hold `contents` under `filename` in `agent_id`'s bucket, replacing what was there. + fn set(&self, agent_id: &str, filename: &str, contents: Vec); +} + +/// That a bucket holds nothing under the name asked for. +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +#[expect( + clippy::exhaustive_structs, + reason = "a marker with nothing to add, which every Storage outside this crate has to construct" +)] +pub struct NotStored; + +/// One decoded segment of `/storage/{agent_id}/{filename}`. +/// +/// Never empty, never holding a `/`, and never only dots, which would step out of its bucket. +#[derive(Clone, Debug, Eq, PartialEq)] +pub struct Segment(String); + +/// That a path segment names no agent or file. +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +#[non_exhaustive] +pub struct NotASegment; + +impl FromStr for Segment { + type Err = NotASegment; + + /// `segment`, already decoded, unless it names nothing a bucket holds. + fn from_str(segment: &str) -> Result { + if segment.trim_matches('.').is_empty() || segment.contains('/') { + return Err(NotASegment); + } + Ok(Self(String::from(segment))) + } +} + +/// Where [`StorageRoutes`] are mounted: `/storage/{agent_id}/{filename}`, each part a [`Segment`]. +#[must_use] +pub fn path() -> (&'static str, ParsePathSegment, ParsePathSegment) { + ("/storage", parse_path_segment(), parse_path_segment()) +} + +/// The storage one connection from `client_ip` reaches, its owners as `registry` says. +pub struct StorageRoutes<'routes, S, K> { + registry: &'routes AgentRegistry, + storage: &'routes K, + client_ip: IpAddr, +} + +impl<'routes, S, K> StorageRoutes<'routes, S, K> { + /// `storage`, reached from `client_ip`, its owners as `registry` says. + #[must_use] + pub const fn new(registry: &'routes AgentRegistry, storage: &'routes K, client_ip: IpAddr) -> Self { + Self { + registry, + storage, + client_ip, + } + } +} + +impl StorageRoutes<'_, S, K> { + /// What a `PUT` of `length` bytes into `agent_id`'s bucket is refused with, if the client may not make it. + /// + /// As the `std` service holds it: a blocked address may not, and otherwise only the agent that owns the bucket, + /// while connected, from the address it connected from. + fn refusal(&self, agent_id: &str, length: usize) -> Result<(), StatusCode> { + if self.registry.blocklist.is_blocked(self.client_ip) { + return Err(StatusCode::FORBIDDEN); + } + let Ok(owner_ip) = self.registry.agent_ip(agent_id) else { + return Err(StatusCode::NOT_FOUND); + }; + let connected = self.registry.agent_session(agent_id).is_some(); + if !connected || owner_ip.to_canonical() != self.client_ip.to_canonical() { + return Err(StatusCode::FORBIDDEN); + } + if length > MAX_STORED { + return Err(StatusCode::PAYLOAD_TOO_LARGE); + } + Ok(()) + } +} + +impl MethodHandlerService for StorageRoutes<'_, S, K> +where + S: Clone + Send + 'static, + K: Storage, +{ + /// Answer `method` for the file `filename` in `agent_id`'s bucket. + /// + /// A `GET` or `HEAD` reads it, and a `PUT` stores the request's body as it. + #[expect( + clippy::future_not_send, + reason = "picoserve's futures hold Cells; its executors run each connection on one thread" + )] + async fn call_method_handler_service( + &self, + _state: &State, + (Segment(agent_id), Segment(filename)): (Segment, Segment), + method: &str, + mut request: Request<'_, R>, + response_writer: W, + ) -> Result + where + R: Read, + W: ResponseWriter, + { + let status = if method.eq_ignore_ascii_case("get") || method.eq_ignore_ascii_case("head") { + match self.storage.get(&agent_id, &filename) { + Ok(contents) => { + let connection = request.body_connection.finalize().await?; + // A `HEAD` only asks whether the file is there, so its body is left out. + let body = if method.eq_ignore_ascii_case("head") { + &[] + } else { + contents.as_slice() + }; + return Response::ok(body).write_to(connection, response_writer).await; + } + Err(NotStored) => StatusCode::NOT_FOUND, + } + } else if method.eq_ignore_ascii_case("put") { + let length = request.body_connection.content_length(); + match self.refusal(&agent_id, length) { + Ok(()) => { + let mut contents = vec![0; length]; + let read = request.body_connection.body().reader().read_exact(&mut contents).await; + if read.is_ok() { + self.storage.set(&agent_id, &filename, contents); + StatusCode::OK + } else { + StatusCode::BAD_REQUEST + } + } + Err(status) => status, + } + } else { + StatusCode::METHOD_NOT_ALLOWED + }; + status + .write_to(request.body_connection.finalize().await?, response_writer) + .await + } +} diff --git a/services/storage/tests/picoserve.rs b/services/storage/tests/picoserve.rs new file mode 100644 index 00000000..5e391112 --- /dev/null +++ b/services/storage/tests/picoserve.rs @@ -0,0 +1,163 @@ +//! Agent storage through picoserve over real TCP sockets, with tokio standing in for an embedded executor and network +//! stack: what an agent stores any client reads back, and only the owner, connected, may store. +#![cfg(test)] + +use std::collections::BTreeMap; +use std::net::{IpAddr, Ipv4Addr, SocketAddr}; +use std::sync::{Arc, Mutex}; +use std::time::Duration; + +use edge_toolkit::ws_server::AgentRegistry; +use et_storage_service::picoserve::{MAX_STORED, NotStored, Storage, StorageRoutes, path}; +use picoserve::{Config, Router, Server}; +use tokio::io::{AsyncReadExt as _, AsyncWriteExt as _}; +use tokio::net::{TcpListener, TcpStream}; +use tokio::task::LocalSet; + +/// How long a test waits for an answer before failing rather than hanging. +const PATIENCE: Duration = Duration::from_secs(5); +/// The agent the tests store files for, connected from this host. +const AGENT: &str = "id-1"; + +/// Agents' storage in memory, as a board keeps it. +#[derive(Default)] +struct Memory(Mutex>>); + +impl Storage for Memory { + fn get(&self, agent_id: &str, filename: &str) -> Result, NotStored> { + let key = (agent_id.to_owned(), filename.to_owned()); + let Ok(stored) = self.0.lock() else { + return Err(NotStored); + }; + stored.get(&key).cloned().ok_or(NotStored) + } + + fn set(&self, agent_id: &str, filename: &str, contents: Vec) { + let _previous = self + .0 + .lock() + .unwrap() + .insert((agent_id.to_owned(), filename.to_owned()), contents); + } +} + +/// Storage on a fresh loopback port, one task per connection as an embedder would, and its address. +/// +/// Also the registry its owners are checked against, [`AGENT`] connected in it. +async fn serving() -> (SocketAddr, Arc>) { + let registry = Arc::new(AgentRegistry::default()); + let localhost = IpAddr::V4(Ipv4Addr::LOCALHOST); + let _connected = registry.connect_agent(Some(AGENT.to_owned()), AGENT.to_owned(), localhost, ()); + let storage = Arc::new(Memory::default()); + let listener = TcpListener::bind((Ipv4Addr::LOCALHOST, 0)).await.unwrap(); + let address = listener.local_addr().unwrap(); + let accepting = Arc::clone(®istry); + let _accepting = tokio::task::spawn_local(async move { + loop { + let (socket, peer) = listener.accept().await.unwrap(); + let registry = Arc::clone(&accepting); + let storage = Arc::clone(&storage); + let _serving = tokio::task::spawn_local(async move { + let routes = StorageRoutes::new(&*registry, &*storage, peer.ip()); + let app = Router::new().route_service(path(), routes); + let config = Config::const_default(); + let mut http_buffer = [0; 2048]; + let _served = Server::new_tokio(&app, &config, &mut http_buffer).serve(socket).await; + }); + } + }); + (address, registry) +} + +/// The whole response to `head`, a request line and headers to which the closing ones are added, and then `body`. +async fn send(address: SocketAddr, head: &str, body: &[u8]) -> String { + let mut socket = TcpStream::connect(address).await.unwrap(); + let request = format!("{head}Connection: close\r\n\r\n"); + socket.write_all(request.as_bytes()).await.unwrap(); + socket.write_all(body).await.unwrap(); + let mut response = String::default(); + let _read = tokio::time::timeout(PATIENCE, socket.read_to_string(&mut response)) + .await + .unwrap() + .unwrap(); + response +} + +/// The whole response to a bodiless `method path`. +async fn request(address: SocketAddr, method: &str, path: &str) -> String { + send(address, &format!("{method} {path} HTTP/1.1\r\nHost: hub\r\n"), b"").await +} + +/// The whole response to a `PUT path` of `contents`. +async fn put(address: SocketAddr, path: &str, contents: &[u8]) -> String { + let head = format!( + "PUT {path} HTTP/1.1\r\nHost: hub\r\nContent-Length: {}\r\n", + contents.len() + ); + send(address, &head, contents).await +} + +/// The body of `response`, after its head. +fn body(response: &str) -> &str { + response.split_once("\r\n\r\n").unwrap().1 +} + +#[tokio::test] +async fn an_agent_stores_files_any_client_reads_back() { + LocalSet::new() + .run_until(Box::pin(async { + let (address, registry) = serving().await; + let path = format!("/storage/{AGENT}/model.json"); + + assert!(request(address, "GET", &path).await.starts_with("HTTP/1.1 404")); + assert!(put(address, &path, b"{\"w\":1}").await.starts_with("HTTP/1.1 200")); + assert_eq!(body(&request(address, "GET", &path).await), "{\"w\":1}"); + let head = request(address, "HEAD", &path).await; + assert!(head.starts_with("HTTP/1.1 200")); + assert_eq!(body(&head), ""); + // Replaced by the next upload, and decoded as the `std` service decodes it. + assert!(put(address, &path, b"2").await.starts_with("HTTP/1.1 200")); + let encoded = request(address, "GET", &format!("/storage/{AGENT}/model%2Ejson")).await; + assert_eq!(body(&encoded), "2"); + + let too_large = vec![0; MAX_STORED + 1]; + let refused = put(address, &format!("/storage/{AGENT}/big"), &too_large).await; + assert!(refused.starts_with("HTTP/1.1 413")); + assert!( + put(address, "/storage/nobody/x", b"1") + .await + .starts_with("HTTP/1.1 404") + ); + assert!(request(address, "DELETE", &path).await.starts_with("HTTP/1.1 405")); + + // Only while the agent is connected. + registry.mark_disconnected(AGENT); + assert!(put(address, &path, b"3").await.starts_with("HTTP/1.1 403")); + assert_eq!(body(&request(address, "GET", &path).await), "2"); + + let localhost = IpAddr::V4(Ipv4Addr::LOCALHOST); + let _again = registry.connect_agent(Some(AGENT.to_owned()), AGENT.to_owned(), localhost, ()); + let _blocked = registry.blocklist.block(localhost); + assert!(put(address, &path, b"4").await.starts_with("HTTP/1.1 403")); + })) + .await; +} + +#[tokio::test] +async fn a_path_naming_no_stored_file_is_not_found() { + LocalSet::new() + .run_until(Box::pin(async { + let (address, _registry) = serving().await; + for path in [ + "/storage/id-1", + "/storage/id-1/a/b", + "/storage/id-1/%2F", + "/storage/id-1/..", + "/storage//a", + "/storage/id-1/%zz", + ] { + assert!(request(address, "GET", path).await.starts_with("HTTP/1.1 404")); + } + })) + .await; +} diff --git a/services/ws-server-firmware/Cargo.toml b/services/ws-server-firmware/Cargo.toml index 00264e7d..a30691ca 100644 --- a/services/ws-server-firmware/Cargo.toml +++ b/services/ws-server-firmware/Cargo.toml @@ -1,7 +1,7 @@ [package] name = "et-int-ws-server-firmware" publish = false -description = "et-ws-server as firmware for the Raspberry Pi Pico 2 W and the ESP32-C6" +description = "The ws hub, its modules and storage, as firmware for the Pico 2 W, Pico Plus 2 W and ESP32-C6" version = "0.1.0" edition.workspace = true license.workspace = true @@ -35,18 +35,29 @@ docs = ["dep:document-features"] chrono = { workspace = true, features = ["alloc"] } document-features = { workspace = true, optional = true } edge-toolkit.workspace = true -et-ws-server = { workspace = true, features = ["picoserve-server"] } -et-ws-service = { workspace = true, features = ["picoserve-hub"] } +et-modules-service = { workspace = true, features = ["board"] } +et-storage-service = { workspace = true, features = ["board"] } +et-ws-service = { workspace = true, features = ["board"] } et-ws-wasi-runner = { workspace = true, features = ["board"] } -picoserve.workspace = true +picoserve = { workspace = true, features = ["json"] } serde_json = { workspace = true, features = ["alloc"] } # What every board shares once its radio is up: embassy's network stack, executor and clock, and the defmt log. [target.'cfg(target_os = "none")'.dependencies] defmt.workspace = true embassy-executor = { workspace = true, features = ["defmt"] } -embassy-net = { workspace = true, features = ["defmt", "dhcpv4", "medium-ethernet", "proto-ipv4", "tcp"] } +# `mdns` resolves a hub named by its `.local` host name, as a scenario's machine addresses often are. +embassy-net = { workspace = true, features = [ + "defmt", + "dhcpv4", + "dns", + "mdns", + "medium-ethernet", + "proto-ipv4", + "tcp", +] } embassy-time = { workspace = true, features = ["defmt"] } +embedded-io-async.workspace = true picoserve = { workspace = true, features = ["embassy"] } static_cell.workspace = true @@ -74,6 +85,9 @@ esp-rtos = { workspace = true, features = ["defmt", "embassy", "esp-alloc", "esp # Anywhere else, the host's: tokio, and a clock that knows the date. [target.'cfg(not(target_os = "none"))'.dependencies] chrono = { workspace = true, features = ["std"] } +critical-section = { workspace = true, features = ["std"] } +# The guests' sockets to a hub elsewhere are tokio's. +et-ws-wasi-runner = { workspace = true, features = ["board", "std"] } picoserve = { workspace = true, features = ["tokio"] } tokio = { workspace = true, features = ["net", "rt", "time"] } diff --git a/services/ws-server-firmware/build.rs b/services/ws-server-firmware/build.rs index 734571e0..5c8cd12b 100644 --- a/services/ws-server-firmware/build.rs +++ b/services/ws-server-firmware/build.rs @@ -5,7 +5,7 @@ use std::env; use std::error::Error; -use std::path::PathBuf; +use std::path::{Path, PathBuf}; use et_firmware_modules::{BOARD_VARIABLES, RUNNER_GUESTS, board, generate, guests}; use fs_err as fs; @@ -34,6 +34,10 @@ const MODULES_ROOT: &str = "MODULES_ROOT"; const MODULES_ELSEWHERE: &str = "MODULES_ELSEWHERE"; /// Where a request for a module in `MODULES_ELSEWHERE` is redirected, its path under `/modules` appended. const MODULES_ELSEWHERE_URL: &str = "MODULES_ELSEWHERE_URL"; +/// The `address:port` of a hub on another machine, which makes the firmware a runner of its guests and no hub. +const HUB_ADDRESS: &str = "HUB_ADDRESS"; +/// The cfg a build for a hub elsewhere is compiled with. +const REMOTE_HUB: &str = "remote_hub"; /// The operating system cargo is building for, `none` for a board. const CARGO_CFG_TARGET_OS: &str = "CARGO_CFG_TARGET_OS"; /// The architecture cargo is building for, which picks the board. @@ -47,11 +51,12 @@ fn main() -> Result<(), Box> { for variable in BOARD_VARIABLES .into_iter() .chain(MODULE_VARIABLES) - .chain([RUNNER_GUESTS]) + .chain([RUNNER_GUESTS, HUB_ADDRESS]) { println!("cargo:rerun-if-env-changed={variable}"); } let out_dir = PathBuf::from(env::var(OUT_DIR)?); + write_hub_address(&out_dir)?; // The modules are served on the host too, so what a board would serve can be explored without one. let modules = generate( &optional(MODULES_PATHS), @@ -99,6 +104,26 @@ fn main() -> Result<(), Box> { Ok(()) } +/// Write `hub_address.rs`, naming [`HUB_ADDRESS`], and compile the firmware as a runner for that hub when it is set. +/// +/// With a hub elsewhere the firmware serves nothing, and its guests reach that hub instead of one beside them. +#[expect( + clippy::single_call_fn, + reason = "the hub step of main, named apart from the modules" +)] +fn write_hub_address(out_dir: &Path) -> Result<(), Box> { + println!("cargo::rustc-check-cfg=cfg({REMOTE_HUB})"); + let hub_address = optional(HUB_ADDRESS); + if !hub_address.is_empty() { + println!("cargo::rustc-cfg={REMOTE_HUB}"); + } + let source = format!( + "/// The hub this firmware's guests reach, `address:port`.\nconst HUB_ADDRESS: &str = {hub_address:?};\n" + ); + fs::write(out_dir.join("hub_address.rs"), source)?; + Ok(()) +} + /// The variable `name`, empty when it is unset, since every module variable may be. fn optional(name: &str) -> String { env::var(name).unwrap_or_default() diff --git a/services/ws-server-firmware/src/board/esp32c6.rs b/services/ws-server-firmware/src/board/esp32c6.rs index 5066a44f..38365f1e 100644 --- a/services/ws-server-firmware/src/board/esp32c6.rs +++ b/services/ws-server-firmware/src/board/esp32c6.rs @@ -21,7 +21,8 @@ use esp_radio::wifi::sta::StationConfig; use esp_radio::wifi::{AuthenticationMethodConfig, Config, ControllerConfig, Interface, WifiController}; use {esp_backtrace as _, esp_println as _}; -use super::hub::{HEAP_SIZE, serve, stack_resources}; +use super::platform::HEAP_SIZE; +use super::role::{serve, stack_resources}; include!(concat!(env!("OUT_DIR"), "/wifi.rs")); diff --git a/services/ws-server-firmware/src/board/hub.rs b/services/ws-server-firmware/src/board/hub.rs index b5f9833d..b9ddc667 100644 --- a/services/ws-server-firmware/src/board/hub.rs +++ b/services/ws-server-firmware/src/board/hub.rs @@ -17,70 +17,28 @@ use edge_toolkit::ws_server::MessageRetention; use embassy_executor::Spawner; use embassy_net::tcp::TcpSocket; use embassy_net::{Stack, StackResources}; -use embassy_time::{Instant, Timer}; -use et_ws_server::picoserve_server::serve as serve_connection; +use embassy_time::Instant; use et_ws_service::hub::Stamps; use et_ws_service::picoserve_hub::{self, Hub}; -use et_ws_wasi_runner::bindings::wasi::logging::logging; -use et_ws_wasi_runner::board::{Console, Platform}; use picoserve::time::EmbassyTimer; use static_cell::StaticCell; +use super::platform::{BoardPlatform, spawn_runner}; +use crate::local::BoardStorage; use crate::runner::{GUEST_POOL, GUESTS, Shared, run_guest}; +use crate::server::{Served, serve as serve_connection}; use crate::{HTTP_BUFFER, MAX_FRAME_SIZE, MAX_QUEUED, MODULES, PORT}; /// How many connections the hub serves at once: one task, and one socket's buffers, each. const CONNECTIONS: usize = 4; /// Each connection's TCP receive and transmit buffer. const TCP_BUFFER: usize = 4 * 1024; -/// The heap the hub's registry, routing and frame buffers allocate from, on a board whose heap is its SRAM. -#[cfg(not(feature = "pico-plus2w"))] -pub(super) const HEAP_SIZE: usize = 160 * 1024; -/// Nanoseconds in a microsecond: a guest's clock counts the one, the board's the other. -const NANOS_PER_MICRO: u64 = 1000; - -/// The board's clock and its defmt log, which its guests' clock and log are. -#[derive(Clone, Copy)] -struct BoardPlatform; - -impl Platform for BoardPlatform { - /// The board's uptime, in the nanoseconds a guest's clock counts. - fn now(&self) -> u64 { - Instant::now().as_micros().saturating_mul(NANOS_PER_MICRO) - } - - /// Wait on the board's timer until its uptime reaches `deadline`. - async fn sleep_until(&self, deadline: u64) { - Timer::at(Instant::from_micros(deadline.div_euclid(NANOS_PER_MICRO))).await; - } - - /// Log the guest's entry at the defmt level nearest its own. - #[expect( - clippy::missing_asserts_for_indexing, - reason = "the indexing is inside defmt's error! expansion, which an attribute on the call does not reach" - )] - fn log(&self, level: logging::Level, context: &str, message: &str) { - match level { - logging::Level::Critical | logging::Level::Error => defmt::error!("{}: {}", context, message), - logging::Level::Warn => warn!("{}: {}", context, message), - logging::Level::Info => info!("{}: {}", context, message), - logging::Level::Debug | logging::Level::Trace => defmt::debug!("{}: {}", context, message), - } - } - - /// Log what the guest wrote to either standard stream, the board's one console being its defmt log. - fn write(&self, _console: Console, bytes: &[u8]) { - info!( - "{=str}", - core::str::from_utf8(bytes).unwrap_or("(output that is not UTF-8)") - ); - } -} /// Run one guest the build compiled in beside the hub, until it ends. #[embassy_executor::task(pool_size = GUEST_POOL)] async fn guest_task(hub: &'static Hub, name: &'static str, component: &'static [u8], shared: Shared) { - run_guest(hub, name, component, BoardPlatform, shared).await; + let transport = || shared.hub_transport(hub, BoardPlatform); + run_guest(name, component, BoardPlatform, transport, shared.memory()).await; } /// Ids unique across boots -- a random word drawn at boot, then a count -- and the time since boot. @@ -111,7 +69,7 @@ pub(super) fn stack_resources() -> &'static mut StackResources<{ CONNECTIONS + 1 /// Accept connections on [`PORT`] one at a time, serving each until it closes. #[embassy_executor::task(pool_size = CONNECTIONS)] -async fn connection_task(stack: Stack<'static>, hub: &'static Hub) -> ! { +async fn connection_task(stack: Stack<'static>, hub: &'static Hub, storage: &'static BoardStorage) -> ! { let mut rx_buffer = [0; TCP_BUFFER]; let mut tx_buffer = [0; TCP_BUFFER]; let mut http_buffer = [0; HTTP_BUFFER]; @@ -126,12 +84,10 @@ async fn connection_task(stack: Stack<'static>, hub: &'static Hub) .remote_endpoint() .map_or(IpAddr::V4(Ipv4Addr::UNSPECIFIED), |remote| remote.addr.into()); let served = serve_connection( - hub, - &MODULES, + Served::new(hub, &MODULES, storage, &config), socket, client_ip, EmbassyTimer, - &config, &mut http_buffer, ); if let Err(error) = served.await { @@ -147,6 +103,7 @@ async fn connection_task(stack: Stack<'static>, hub: &'static Hub) /// Once `stack` has an address, serve the hub on it, its ids drawn from `boot`, a random word. pub(super) async fn serve(spawner: Spawner, stack: Stack<'static>, boot: u32) { static HUB: StaticCell> = StaticCell::new(); + static STORAGE: StaticCell = StaticCell::new(); stack.wait_config_up().await; if let Some(config) = stack.config_v4() { info!("Serving ws://{}:{}/ws", Debug2Format(&config.address.address()), PORT); @@ -154,17 +111,16 @@ pub(super) async fn serve(spawner: Spawner, stack: Stack<'static>, boot: u32) { let stamps = BoardStamps { boot, issued: 0 }; let hub = Hub::new(stamps, MessageRetention::LatestPerSender, MAX_FRAME_SIZE, MAX_QUEUED); let hub: &'static Hub = HUB.init(hub); + // The guests' storage is the hub's, so an agent on another machine reads what a guest here wrote. + let shared = Shared::new(); + let storage: &'static BoardStorage = STORAGE.init(shared.storage()); for _ in 0..CONNECTIONS { - match connection_task(stack, hub) { + match connection_task(stack, hub, storage) { Ok(task) => spawner.spawn(task), Err(error) => warn!("Failed to start a connection task: {}", Debug2Format(&error)), } } - let shared = Shared::new(); for (name, component) in GUESTS { - match guest_task(hub, name, component, shared.clone()) { - Ok(task) => spawner.spawn(task), - Err(error) => warn!("Failed to start runner {=str}: {}", name, Debug2Format(&error)), - } + spawn_runner(spawner, guest_task(hub, name, component, shared.clone()), name); } } diff --git a/services/ws-server-firmware/src/board/mod.rs b/services/ws-server-firmware/src/board/mod.rs index f404c95b..4002ac17 100644 --- a/services/ws-server-firmware/src/board/mod.rs +++ b/services/ws-server-firmware/src/board/mod.rs @@ -1,11 +1,21 @@ //! The firmware on a board: each board's module brings up its radio and joins the network named at build time, and -//! [`hub`] serves the hub on it. +//! its role serves on it: [`hub`] the hub and the guests beside it, or for a build with a hub elsewhere, `remote` the +//! guests alone, reaching that hub. #[cfg(all(target_arch = "arm", not(any(feature = "pico-plus2w", feature = "pico2w"))))] compile_error!("a bare-metal Arm build is for a board: enable the `pico2w` or the `pico-plus2w` feature"); #[cfg(target_arch = "riscv32")] mod esp32c6; +#[cfg(not(remote_hub))] mod hub; #[cfg(target_arch = "arm")] mod pico2w; +mod platform; +#[cfg(remote_hub)] +mod remote; + +#[cfg(not(remote_hub))] +use self::hub as role; +#[cfg(remote_hub)] +use self::remote as role; diff --git a/services/ws-server-firmware/src/board/pico2w.rs b/services/ws-server-firmware/src/board/pico2w.rs index 997bea3b..a2207b7c 100644 --- a/services/ws-server-firmware/src/board/pico2w.rs +++ b/services/ws-server-firmware/src/board/pico2w.rs @@ -26,8 +26,8 @@ use static_cell::StaticCell; use {defmt_rtt as _, panic_probe as _}; #[cfg(not(feature = "pico-plus2w"))] -use super::hub::HEAP_SIZE; -use super::hub::{serve, stack_resources}; +use super::platform::HEAP_SIZE; +use super::role::{serve, stack_resources}; include!(concat!(env!("OUT_DIR"), "/wifi.rs")); include!(concat!(env!("OUT_DIR"), "/cyw43.rs")); diff --git a/services/ws-server-firmware/src/board/platform.rs b/services/ws-server-firmware/src/board/platform.rs new file mode 100644 index 00000000..a638d20c --- /dev/null +++ b/services/ws-server-firmware/src/board/platform.rs @@ -0,0 +1,63 @@ +//! What every board hands its guests, whatever hub they reach: its clock and its log, and the heap it starts with. + +use defmt::{Debug2Format, info, warn}; +use embassy_executor::{SpawnError, SpawnToken, Spawner}; +use embassy_time::{Instant, Timer}; +use et_ws_wasi_runner::bindings::wasi::logging::logging; +use et_ws_wasi_runner::board::{Console, Platform}; + +/// The heap the hub's registry, routing and frame buffers allocate from, on a board whose heap is its SRAM. +#[cfg(not(feature = "pico-plus2w"))] +pub(super) const HEAP_SIZE: usize = 160 * 1024; +/// Nanoseconds in a microsecond: a guest's clock counts the one, the board's the other. +const NANOS_PER_MICRO: u64 = 1000; + +/// Start the task running the guest `name`, or log why it could not be. +#[expect( + clippy::single_call_fn, + reason = "shared by the board's two roles, each of which a build has only one of" +)] +pub(super) fn spawn_runner(spawner: Spawner, task: Result, SpawnError>, name: &str) { + match task { + Ok(task) => spawner.spawn(task), + Err(error) => warn!("Failed to start runner {=str}: {}", name, Debug2Format(&error)), + } +} + +/// The board's clock and its defmt log, which its guests' clock and log are. +#[derive(Clone, Copy)] +pub(super) struct BoardPlatform; + +impl Platform for BoardPlatform { + /// The board's uptime, in the nanoseconds a guest's clock counts. + fn now(&self) -> u64 { + Instant::now().as_micros().saturating_mul(NANOS_PER_MICRO) + } + + /// Wait on the board's timer until its uptime reaches `deadline`. + async fn sleep_until(&self, deadline: u64) { + Timer::at(Instant::from_micros(deadline.div_euclid(NANOS_PER_MICRO))).await; + } + + /// Log the guest's entry at the defmt level nearest its own. + #[expect( + clippy::missing_asserts_for_indexing, + reason = "the indexing is inside defmt's error! expansion, which an attribute on the call does not reach" + )] + fn log(&self, level: logging::Level, context: &str, message: &str) { + match level { + logging::Level::Critical | logging::Level::Error => defmt::error!("{}: {}", context, message), + logging::Level::Warn => warn!("{}: {}", context, message), + logging::Level::Info => info!("{}: {}", context, message), + logging::Level::Debug | logging::Level::Trace => defmt::debug!("{}: {}", context, message), + } + } + + /// Log what the guest wrote to either standard stream, the board's one console being its defmt log. + fn write(&self, _console: Console, bytes: &[u8]) { + info!( + "{=str}", + core::str::from_utf8(bytes).unwrap_or("(output that is not UTF-8)") + ); + } +} diff --git a/services/ws-server-firmware/src/board/remote.rs b/services/ws-server-firmware/src/board/remote.rs new file mode 100644 index 00000000..28599dca --- /dev/null +++ b/services/ws-server-firmware/src/board/remote.rs @@ -0,0 +1,169 @@ +//! A board whose guests reach a hub on another machine: it serves nothing, and drives each guest's sockets to the hub +//! at [`HUB_ADDRESS`]. + +#![expect( + clippy::future_not_send, + clippy::single_call_fn, + reason = "embassy's executor runs every task on the one core; each board calls each step once" +)] + +use alloc::format; +use alloc::string::String; + +use defmt::{Debug2Format, info, warn}; +use embassy_executor::Spawner; +use embassy_net::dns::DnsQueryType; +use embassy_net::tcp::{Error, TcpReader, TcpSocket, TcpWriter}; +use embassy_net::{IpEndpoint, Stack, StackResources}; +use embedded_io_async::{ErrorType, Read, Write}; +use et_ws_wasi_runner::board::net::{Connector, NetError, NetLink, NetTransport, Split, drive}; + +use super::platform::{BoardPlatform, spawn_runner}; +use crate::HUB_ADDRESS; +use crate::runner::{GUEST_POOL, GUESTS, Shared, run_guest}; + +/// Each socket's TCP receive and transmit buffer. +const TCP_BUFFER: usize = 4 * 1024; +/// The network stack's sockets: two for each guest, its websocket's and its storage requests', and DHCP and DNS. +const SOCKETS: usize = 2 * GUEST_POOL + 2; + +/// Each guest's link to the task driving its sockets. +static LINKS: [NetLink; GUEST_POOL] = [const { NetLink::idle() }; GUEST_POOL]; + +/// The network stack's resources. +pub(super) fn stack_resources() -> &'static mut StackResources { + static RESOURCES: static_cell::StaticCell> = static_cell::StaticCell::new(); + RESOURCES.init(StackResources::new()) +} + +/// A socket of the board's to the hub. +struct BoardSocket<'socket>(TcpSocket<'socket>); + +impl ErrorType for BoardSocket<'_> { + type Error = Error; +} + +impl Read for BoardSocket<'_> { + async fn read(&mut self, buf: &mut [u8]) -> Result { + self.0.read(buf).await + } +} + +impl Write for BoardSocket<'_> { + async fn write(&mut self, buf: &[u8]) -> Result { + self.0.write(buf).await + } + + async fn flush(&mut self) -> Result<(), Error> { + self.0.flush().await + } +} + +impl Split for BoardSocket<'_> { + type Reader<'half> + = TcpReader<'half> + where + Self: 'half; + type Writer<'half> + = TcpWriter<'half> + where + Self: 'half; + + fn split(&mut self) -> (TcpReader<'_>, TcpWriter<'_>) { + self.0.split() + } +} + +/// Opens a socket to the hub on `stack`, through buffers of its own, each time it is asked. +struct BoardConnector { + stack: Stack<'static>, + rx_buffer: [u8; TCP_BUFFER], + tx_buffer: [u8; TCP_BUFFER], +} + +impl BoardConnector { + /// A connector on `stack`. + const fn new(stack: Stack<'static>) -> Self { + Self { + stack, + rx_buffer: [0; TCP_BUFFER], + tx_buffer: [0; TCP_BUFFER], + } + } +} + +impl Connector for BoardConnector { + type Socket<'socket> = BoardSocket<'socket>; + + /// A socket to the hub at [`HUB_ADDRESS`], its host looked up first. + async fn connect(&mut self) -> Result, NetError> { + let endpoint = hub_endpoint(self.stack).await?; + let mut socket = TcpSocket::new(self.stack, &mut self.rx_buffer, &mut self.tx_buffer); + match socket.connect(endpoint).await { + Ok(()) => Ok(BoardSocket(socket)), + Err(error) => Err(NetError::Connect(format!("{error:?}"))), + } + } +} + +/// Where [`HUB_ADDRESS`] is, its host looked up on `stack` unless it is an address already. +async fn hub_endpoint(stack: Stack<'static>) -> Result { + let unreachable = |reason: String| Err(NetError::Connect(reason)); + let Some((host, port)) = HUB_ADDRESS.rsplit_once(':') else { + return unreachable(format!("{HUB_ADDRESS} names no port")); + }; + let Ok(port) = port.parse() else { + return unreachable(format!("{HUB_ADDRESS} names no port")); + }; + match stack.dns_query(host, DnsQueryType::A).await { + Ok(addresses) => addresses.first().map_or_else( + || unreachable(format!("{host} resolves to no address")), + |address| Ok(IpEndpoint::new(*address, port)), + ), + Err(error) => unreachable(format!("{error:?}")), + } +} + +/// Drive one guest's sockets to the hub, for as long as the board runs. +#[embassy_executor::task(pool_size = GUEST_POOL)] +async fn net_task(stack: Stack<'static>, link: &'static NetLink) -> ! { + #[expect( + clippy::large_futures, + reason = "the sockets' buffers are meant to be in it: a task's future is the static memory embassy keeps it in" + )] + drive( + link, + BoardConnector::new(stack), + BoardConnector::new(stack), + HUB_ADDRESS, + BoardPlatform, + ) + .await +} + +/// Run one guest the build compiled in against the hub, until it ends. +#[embassy_executor::task(pool_size = GUEST_POOL)] +async fn guest_task(link: &'static NetLink, name: &'static str, component: &'static [u8], shared: Shared) { + let transport = || NetTransport::new(link, BoardPlatform); + run_guest(name, component, BoardPlatform, transport, shared.memory()).await; +} + +/// Once `stack` has an address, run each guest the build compiled in against the hub at [`HUB_ADDRESS`]. +/// +/// `_boot`, the random word a hub draws its ids from, is not needed: the hub elsewhere assigns them. +pub(super) async fn serve(spawner: Spawner, stack: Stack<'static>, _boot: u32) { + stack.wait_config_up().await; + info!("Joining ws://{=str}/ws", HUB_ADDRESS); + let shared = Shared::new(); + for ((name, component), link) in GUESTS.iter().zip(&LINKS) { + match net_task(stack, link) { + Ok(task) => spawner.spawn(task), + Err(error) => warn!( + "Failed to start runner {=str}'s sockets: {}", + name, + Debug2Format(&error) + ), + } + spawn_runner(spawner, guest_task(link, name, component, shared.clone()), name); + } +} diff --git a/services/ws-server-firmware/src/host.rs b/services/ws-server-firmware/src/host.rs index 43a7cebe..746f080e 100644 --- a/services/ws-server-firmware/src/host.rs +++ b/services/ws-server-firmware/src/host.rs @@ -3,26 +3,47 @@ //! The hub, its settings, the modules built in and the guests its runners run are the board's own; only the network //! stack, the executor and the clock differ. It listens on the address its one argument names, or on the board's port //! on every interface, and says where on its first line of output, so a caller can ask for port 0 and learn the one -//! it got. +//! it got. A build for a hub elsewhere serves nothing, and runs its guests against that hub as the board would. use std::error::Error; use std::io::Write as _; +#[cfg(not(remote_hub))] use std::net::SocketAddr; use std::sync::LazyLock; -use std::time::{Duration, Instant, SystemTime}; +#[cfg(not(remote_hub))] +use std::time::SystemTime; +use std::time::{Duration, Instant}; +#[cfg(not(remote_hub))] use chrono::{DateTime, Utc}; +// The host's critical section, which the guests' links lock under, as a board's HAL provides on the board. +use critical_section as _; +#[cfg(not(remote_hub))] use edge_toolkit::ws_server::MessageRetention; -use et_ws_server::picoserve_server::serve; +#[cfg(not(remote_hub))] use et_ws_service::hub::Stamps; +#[cfg(not(remote_hub))] use et_ws_service::picoserve_hub::{self, Hub}; use et_ws_wasi_runner::bindings::wasi::logging::logging::Level; +#[cfg(remote_hub)] +use et_ws_wasi_runner::board::net::{NetLink, NetTransport, drive}; +#[cfg(remote_hub)] +use et_ws_wasi_runner::board::net_tokio::TokioConnector; use et_ws_wasi_runner::board::{Console, Platform}; +#[cfg(not(remote_hub))] use picoserve::time::TokioTimer; +#[cfg(not(remote_hub))] use tokio::net::TcpListener; use tokio::task::LocalSet; +#[cfg(remote_hub)] +use crate::HUB_ADDRESS; +#[cfg(not(remote_hub))] +use crate::local::BoardStorage; use crate::runner::{GUESTS, Shared, run_guest}; +#[cfg(not(remote_hub))] +use crate::server::{Served, serve}; +#[cfg(not(remote_hub))] use crate::{HTTP_BUFFER, MAX_FRAME_SIZE, MAX_QUEUED, MODULES, PORT}; /// When the process started, which the guests' clock counts from. @@ -72,9 +93,11 @@ impl Platform for HostPlatform { } /// Ids `host-1`, `host-2`, ... in turn, and the host's clock, which unlike the board's knows the date. +#[cfg(not(remote_hub))] #[derive(Default)] struct HostStamps(u32); +#[cfg(not(remote_hub))] impl Stamps for HostStamps { fn new_id(&mut self) -> String { self.0 = self.0.wrapping_add(1); @@ -86,24 +109,61 @@ impl Stamps for HostStamps { } } -/// Serve the hub on the address the first argument names, or on [`PORT`] on every interface, until killed. +/// Run what the board would until killed: serve the hub, or with a hub elsewhere, run the guests against it. pub(crate) fn main() -> Result<(), Box> { - let address: SocketAddr = std::env::args() - .nth(1) - .unwrap_or_else(|| format!("0.0.0.0:{PORT}")) - .parse()?; let runtime = tokio::runtime::Builder::new_current_thread().enable_all().build()?; // One thread, as the board's executor has: picoserve's connections are not `Send`. - LocalSet::new().block_on(&runtime, Box::pin(accept(address))) + LocalSet::new().block_on(&runtime, Box::pin(start())) +} + +/// Run each guest the build compiled in against the hub at [`HUB_ADDRESS`], until every one has ended. +/// +/// Says where on its first line of output, as the hub's own build does. +#[cfg(remote_hub)] +#[expect(clippy::single_call_fn, reason = "a step apart from the runtime's setup")] +async fn start() -> Result<(), Box> { + let address = tokio::net::lookup_host(HUB_ADDRESS) + .await? + .next() + .ok_or("HUB_ADDRESS names no address")?; + { + let mut stdout = std::io::stdout().lock(); + writeln!(stdout, "Joining ws://{HUB_ADDRESS}/ws with {} runners", GUESTS.len())?; + stdout.flush()?; + } + let shared = Shared::new(); + let mut running = Vec::new(); + for (name, component) in GUESTS { + // Leaked, as the board's live in statics: the guest and the task driving its sockets hold it to the end. + let link: &'static NetLink = Box::leak(Box::new(NetLink::idle())); + let dial = TokioConnector::new(address); + let _driving = tokio::task::spawn_local(async move { + drive(link, dial, dial, HUB_ADDRESS, HostPlatform).await; + }); + let transport = move || NetTransport::new(link, HostPlatform); + let guest = run_guest(name, component, HostPlatform, transport, shared.memory()); + running.push(tokio::task::spawn_local(guest)); + } + for guest in running { + guest.await?; + } + Ok(()) } -/// Accept connections on `address`, serving each from its own task with the one hub they share. +/// Serve the hub on the address the first argument names, or on [`PORT`] on every interface. +/// +/// Accepts connections there, serving each from its own task with the one hub they share. +#[cfg(not(remote_hub))] #[expect( clippy::future_not_send, clippy::single_call_fn, reason = "picoserve's futures hold Cells, run on one thread as on the board; a step apart from the runtime's setup" )] -async fn accept(address: SocketAddr) -> Result<(), Box> { +async fn start() -> Result<(), Box> { + let address: SocketAddr = std::env::args() + .nth(1) + .unwrap_or_else(|| format!("0.0.0.0:{PORT}")) + .parse()?; let listener = TcpListener::bind(address).await?; let mut stdout = std::io::stdout().lock(); // With how many runners it starts, so a caller knows how many agents of its own to expect. @@ -122,8 +182,12 @@ async fn accept(address: SocketAddr) -> Result<(), Box> { MAX_QUEUED, ))); let shared = Shared::new(); + let storage: &'static BoardStorage = Box::leak(Box::new(shared.storage())); for (name, component) in GUESTS { - let _running = tokio::task::spawn_local(run_guest(hub, name, component, HostPlatform, shared.clone())); + let guest_shared = shared.clone(); + let transport = move || guest_shared.hub_transport(hub, HostPlatform); + let guest = run_guest(name, component, HostPlatform, transport, shared.memory()); + let _running = tokio::task::spawn_local(guest); } loop { let (socket, peer) = listener.accept().await?; @@ -131,8 +195,9 @@ async fn accept(address: SocketAddr) -> Result<(), Box> { // Each connection's own, as each of the board's connection tasks holds one. let config = picoserve_hub::config(); let mut http_buffer = [0; HTTP_BUFFER]; + let served = Served::new(hub, &MODULES, storage, &config); // A connection that ends badly is the client's to report; the hub carries on serving the others. - let _served = serve(hub, &MODULES, socket, peer.ip(), TokioTimer, &config, &mut http_buffer).await; + let _served = serve(served, socket, peer.ip(), TokioTimer, &mut http_buffer).await; }); } } diff --git a/services/ws-server-firmware/src/local.rs b/services/ws-server-firmware/src/local.rs new file mode 100644 index 00000000..cad23c48 --- /dev/null +++ b/services/ws-server-firmware/src/local.rs @@ -0,0 +1,199 @@ +//! The hub beside the board's guests: each guest's websocket a link into it in the same process rather than a socket, +//! and its storage the memory the hub serves agents' storage from. + +#[cfg(target_os = "none")] +use alloc::string::String; +#[cfg(target_os = "none")] +use alloc::vec::Vec; +use core::future::poll_fn; +use core::net::{IpAddr, Ipv4Addr}; +use core::pin::pin; +use core::task::Poll; + +use et_storage_service::picoserve::{NotStored, Storage}; +use et_ws_service::hub::{SessionMessage, Stamps}; +use et_ws_service::picoserve_hub::{FrameTooLarge, Hub, LocalLink}; +use et_ws_wasi_runner::board::{Absent, Buckets, Platform, Transport}; +use et_ws_wasi_runner::convert::Frame; + +/// The address a runner's agent is registered from: the board itself. +const LOOPBACK: IpAddr = IpAddr::V4(Ipv4Addr::LOCALHOST); +/// Nanoseconds in a millisecond, the unit a guest's `recv` timeout comes in. +const NANOS_PER_MILLI: u64 = 1_000_000; + +/// Why a guest's link to the hub refused a call; nothing between them can break, so only the guest's own use. +#[derive(Debug)] +pub(crate) enum LinkError { + /// The link was used while closed. + NotOpen, + /// A frame larger than the hub reads from a socket, which it holds the board's own agents to too. + TooLarge(FrameTooLarge), +} + +impl From for LinkError { + fn from(refused: FrameTooLarge) -> Self { + Self::TooLarge(refused) + } +} + +impl core::fmt::Display for LinkError { + /// Why the link refused, as the guest is told it. + fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { + match self { + Self::NotOpen => f.write_str("the link to the hub is not open"), + Self::TooLarge(refused) => refused.fmt(f), + } + } +} + +/// Whether a guest's link to the hub is open, and the link when it is. +enum State { + Closed, + Open(LocalLink<'static, S>), +} + +/// A guest's websocket to the hub it runs beside: a link in process, opened on the guest's `connect`. +/// +/// Its waits are on the guest's own clock, `platform`'s. It needs no keep-alive, since nothing idles out a link. +pub(crate) struct HubTransport { + hub: &'static Hub, + platform: P, + state: State, + buckets: Buckets, +} + +impl HubTransport { + /// A transport into `hub`, its storage `buckets`, waiting on `platform`'s clock, not yet open. + #[expect(clippy::single_call_fn, reason = "the constructor, called where a guest is started")] + pub(crate) const fn new(hub: &'static Hub, platform: P, buckets: Buckets) -> Self { + Self { + hub, + platform, + state: State::Closed, + buckets, + } + } +} + +#[expect( + clippy::unused_async_trait_impl, + reason = "the Transport trait declares every method async, and a link in process never waits to open or send" +)] +impl Transport for HubTransport +where + S: Stamps + Send + 'static, + P: Platform, +{ + type Error = LinkError; + + /// Join the hub as an agent of the board's own, which never fails, since nothing lies between them. + async fn connect(&mut self) -> Result<(), LinkError> { + self.state = State::Open(self.hub.local(LOOPBACK)); + Ok(()) + } + + /// Route `frame` through the hub, as a connection's frame would be, refusing one larger than a connection's may be. + async fn send(&mut self, frame: Frame) -> Result<(), LinkError> { + let State::Open(link) = &mut self.state else { + return Err(LinkError::NotOpen); + }; + match frame { + Frame::Text(text) => link.send_text(&text)?, + Frame::Binary(bytes) => link.send_binary(bytes.into())?, + // Every kind of frame a guest sends is one of the two above. + _ => {} + } + Ok(()) + } + + /// The next message for the guest, as the frame a socket would have carried, if one comes within `timeout_ms`. + async fn recv(&mut self, timeout_ms: u32) -> Result, LinkError> { + let State::Open(link) = &mut self.state else { + return Err(LinkError::NotOpen); + }; + let deadline = self + .platform + .now() + .saturating_add(u64::from(timeout_ms).saturating_mul(NANOS_PER_MILLI)); + let Ok(message) = first(link.next(), self.platform.sleep_until(deadline)).await else { + return Ok(None); + }; + Ok(Some(match message { + SessionMessage::Json(message) => Frame::Text(serde_json::to_string(&message).unwrap_or_default()), + SessionMessage::Text(text) => Frame::Text(text), + SessionMessage::Binary(bytes) => Frame::Binary(bytes.to_vec()), + _ => Frame::Text(String::default()), + })) + } + + /// Leave the hub, which drops the link and so disconnects the guest's agent. + async fn close(&mut self) { + self.state = State::Closed; + } + + /// What the board's storage holds under `key` in `bucket`, which is the hub's beside it. + async fn load(&mut self, bucket: &str, key: &str) -> Result>, LinkError> { + Ok(self.buckets.get(bucket, key).ok()) + } + + /// Store `value` in the board's storage, which the hub beside it serves. + async fn store(&mut self, bucket: &str, key: &str, value: Vec) -> Result<(), LinkError> { + self.buckets.set(bucket, key, value); + Ok(()) + } +} + +/// That a wait's deadline passed before what it waited for arrived. +struct Elapsed; + +/// What `wanted` resolves to, or [`Elapsed`] once `deadline` has passed first. +#[expect( + clippy::single_call_fn, + reason = "the timeout of recv, a race of two futures kept apart from the frames" +)] +async fn first(wanted: W, deadline: D) -> Result +where + W: Future, + D: Future, +{ + let mut wanted = pin!(wanted); + let mut deadline = pin!(deadline); + poll_fn(|context| { + if let Poll::Ready(output) = wanted.as_mut().poll(context) { + return Poll::Ready(Ok(output)); + } + deadline.as_mut().poll(context).map(|()| Err(Elapsed)) + }) + .await +} + +/// The board's storage as the hub serves it. +/// +/// So an agent anywhere reads what a guest wrote, and a guest what it uploaded. +pub(crate) struct BoardStorage(Buckets); + +impl BoardStorage { + /// `buckets`, as the hub serves them. + #[expect( + clippy::single_call_fn, + reason = "the constructor, called where the board's storage is shared" + )] + pub(crate) const fn new(buckets: Buckets) -> Self { + Self(buckets) + } +} + +impl Storage for BoardStorage { + /// What a guest, or an agent through the hub, stored under `filename` in `agent_id`'s bucket. + fn get(&self, agent_id: &str, filename: &str) -> Result, NotStored> { + match self.0.get(agent_id, filename) { + Ok(contents) => Ok(contents), + Err(Absent { .. }) => Err(NotStored), + } + } + + /// Store what an agent uploaded through the hub, where the board's guests read it too. + fn set(&self, agent_id: &str, filename: &str, contents: Vec) { + self.0.set(agent_id, filename, contents); + } +} diff --git a/services/ws-server-firmware/src/main.rs b/services/ws-server-firmware/src/main.rs index 56f85733..0de58778 100644 --- a/services/ws-server-firmware/src/main.rs +++ b/services/ws-server-firmware/src/main.rs @@ -1,5 +1,5 @@ -//! et-ws-server as firmware: the hub's `/ws`, `/health`, `/favicon.ico` and the modules built into it, on port 8080, -//! over a board's own Wi-Fi. +//! The hub as firmware: its `/ws`, `/health`, `/favicon.ico`, agent storage and the modules built into it, on port +//! 8080, over a board's own Wi-Fi. //! //! The target picks the board: bare-metal Arm is the Raspberry Pi Pico 2 W, or the Pimoroni Pico Plus 2 W with its //! feature, and bare-metal RISC-V the ESP32-C6. The Pico Plus 2 W also runs the wasi runners the build compiled in @@ -12,24 +12,38 @@ extern crate alloc; +#[cfg(not(remote_hub))] use edge_toolkit::ports::Services; #[cfg(target_os = "none")] mod board; #[cfg(not(target_os = "none"))] mod host; +#[cfg(not(remote_hub))] +mod local; mod runner; +#[cfg(not(remote_hub))] +mod server; #[cfg(not(target_os = "none"))] use self::host::main; /// The port the hub serves on, as the `std` server's plain-HTTP listener does. +#[cfg(not(remote_hub))] const PORT: u16 = Services::InsecureWebSocketServer.port(); /// The largest WebSocket frame the hub reads; each connection holds a buffer this size on the heap. +#[cfg(not(remote_hub))] const MAX_FRAME_SIZE: usize = 8 * 1024; /// How many messages for one connection the hub holds while it writes earlier ones, each up to `MAX_FRAME_SIZE`. +#[cfg(not(remote_hub))] const MAX_QUEUED: usize = 2; /// Each connection's buffer for the HTTP request head. +#[cfg(not(remote_hub))] const HTTP_BUFFER: usize = 2048; +// The hub's modules, which a build for a hub elsewhere serves none of. +#[cfg(not(remote_hub))] include!(concat!(env!("OUT_DIR"), "/modules.rs")); +// The hub elsewhere a build for one reaches. +#[cfg(remote_hub)] +include!(concat!(env!("OUT_DIR"), "/hub_address.rs")); diff --git a/services/ws-server-firmware/src/runner.rs b/services/ws-server-firmware/src/runner.rs index 2232bba6..f5fe235c 100644 --- a/services/ws-server-firmware/src/runner.rs +++ b/services/ws-server-firmware/src/runner.rs @@ -1,23 +1,23 @@ -//! The wasi runners the firmware starts beside its hub: each guest the build compiled in, run on the board's engine, -//! its websocket a link into the hub in the same process rather than a socket. +//! The wasi runners the firmware starts: each guest the build compiled in, run on the board's engine. //! -//! The board and the host share all of it but the clock and the log, which each hands in as a [`Platform`]. +//! Each reaches its hub through the [`Transport`] it is handed: the hub beside it, or one on another machine. The board +//! and the host share all of it but the clock and the log, which each hands in as a [`Platform`]. #[cfg(target_os = "none")] -use alloc::string::{String, ToString as _}; -use core::future::poll_fn; -use core::net::{IpAddr, Ipv4Addr}; -use core::pin::pin; +use alloc::string::ToString as _; use core::ptr::null_mut; use core::sync::atomic::{AtomicPtr, Ordering}; -use core::task::Poll; -use et_ws_service::hub::{SessionMessage, Stamps}; -use et_ws_service::picoserve_hub::{FrameTooLarge, Hub, LocalLink}; -use et_ws_wasi_runner::board::{BoardError, Buckets, MemoryBudget, Platform, Transport, run}; -use et_ws_wasi_runner::convert::Frame; +#[cfg(not(remote_hub))] +use et_ws_service::picoserve_hub::Hub; +#[cfg(not(remote_hub))] +use et_ws_wasi_runner::board::Buckets; +use et_ws_wasi_runner::board::{BoardError, MemoryBudget, Platform, Transport, run}; use et_ws_wasi_runner::engine::board_engine; +#[cfg(not(remote_hub))] +use crate::local::{BoardStorage, HubTransport}; + include!(concat!(env!("OUT_DIR"), "/guests.rs")); /// How many guest tasks a board's executor holds room for. @@ -28,10 +28,9 @@ include!(concat!(env!("OUT_DIR"), "/guests.rs")); expect(dead_code, reason = "the host spawns its guests without a pool") )] pub(crate) const GUEST_POOL: usize = if GUESTS.is_empty() { 1 } else { GUESTS.len() }; -/// The address a runner's agent is registered from: the board itself. -const LOOPBACK: IpAddr = IpAddr::V4(Ipv4Addr::LOCALHOST); -/// Nanoseconds in a millisecond, the unit a guest's `recv` timeout comes in. -const NANOS_PER_MILLI: u64 = 1_000_000; + +/// How long, in nanoseconds of the board's clock, a guest whose run failed waits before it is run again. +const RERUN_DELAY: u64 = 5_000_000_000; /// The pointers wasmtime keeps for the call it is in, which with one thread are statics rather than a thread's. /// @@ -62,145 +61,13 @@ extern "C" fn wasmtime_tls_set(slot: usize, pointer: *mut u8) { } } -/// Why a guest's link to the hub refused a call; nothing between them can break, so only the guest's own use. -#[derive(Debug)] -pub(crate) enum LinkError { - /// The link was used while closed. - NotOpen, - /// A frame larger than the hub reads from a socket, which it holds the board's own agents to too. - TooLarge(FrameTooLarge), -} - -impl From for LinkError { - fn from(refused: FrameTooLarge) -> Self { - Self::TooLarge(refused) - } -} - -impl core::fmt::Display for LinkError { - fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { - match self { - Self::NotOpen => f.write_str("the link to the hub is not open"), - Self::TooLarge(refused) => refused.fmt(f), - } - } -} - -/// Whether a guest's link to the hub is open, and the link when it is. -enum State { - Closed, - Open(LocalLink<'static, S>), -} - -/// A guest's websocket to the hub it runs beside: a link in process, opened on the guest's `connect`. -/// -/// Its waits are on the guest's own clock, `platform`'s. It needs no keep-alive, since nothing idles out a link. -pub(crate) struct HubTransport { - hub: &'static Hub, - platform: P, - state: State, -} - -impl HubTransport { - /// A transport into `hub`, waiting on `platform`'s clock, not yet open. - #[expect( - clippy::single_call_fn, - reason = "the transport's constructor, called where a guest is started" - )] - pub(crate) const fn new(hub: &'static Hub, platform: P) -> Self { - Self { - hub, - platform, - state: State::Closed, - } - } -} - -#[expect( - clippy::unused_async_trait_impl, - reason = "the Transport trait declares every method async, and a link in process never waits to open or send" -)] -impl Transport for HubTransport -where - S: Stamps + Send + 'static, - P: Platform, -{ - type Error = LinkError; - - /// Join the hub as an agent of the board's own, which never fails, since nothing lies between them. - async fn connect(&mut self) -> Result<(), LinkError> { - self.state = State::Open(self.hub.local(LOOPBACK)); - Ok(()) - } - - /// Route `frame` through the hub, as a connection's frame would be, refusing one larger than a connection's may be. - async fn send(&mut self, frame: Frame) -> Result<(), LinkError> { - let State::Open(link) = &mut self.state else { - return Err(LinkError::NotOpen); - }; - match frame { - Frame::Text(text) => link.send_text(&text)?, - Frame::Binary(bytes) => link.send_binary(bytes.into())?, - // Every kind of frame a guest sends is one of the two above. - _ => {} - } - Ok(()) - } - - /// The next message for the guest, as the frame a socket would have carried, if one comes within `timeout_ms`. - async fn recv(&mut self, timeout_ms: u32) -> Result, LinkError> { - let State::Open(link) = &mut self.state else { - return Err(LinkError::NotOpen); - }; - let deadline = self - .platform - .now() - .saturating_add(u64::from(timeout_ms).saturating_mul(NANOS_PER_MILLI)); - let Ok(message) = first(link.next(), self.platform.sleep_until(deadline)).await else { - return Ok(None); - }; - Ok(Some(match message { - SessionMessage::Json(message) => Frame::Text(serde_json::to_string(&message).unwrap_or_default()), - SessionMessage::Text(text) => Frame::Text(text), - SessionMessage::Binary(bytes) => Frame::Binary(bytes.to_vec()), - _ => Frame::Text(String::default()), - })) - } - - /// Leave the hub, which drops the link and so disconnects the guest's agent. - async fn close(&mut self) { - self.state = State::Closed; - } -} - -/// That a wait's deadline passed before what it waited for arrived. -struct Elapsed; - -/// What `wanted` resolves to, or [`Elapsed`] once `deadline` has passed first. -#[expect( - clippy::single_call_fn, - reason = "the timeout of recv, a race of two futures kept apart from the frames" -)] -async fn first(wanted: W, deadline: D) -> Result -where - W: Future, - D: Future, -{ - let mut wanted = pin!(wanted); - let mut deadline = pin!(deadline); - poll_fn(|context| { - if let Poll::Ready(output) = wanted.as_mut().poll(context) { - return Poll::Ready(Ok(output)); - } - deadline.as_mut().poll(context).map(|()| Err(Elapsed)) - }) - .await -} - -/// What every guest on the board shares: its storage, and the memory the build holds for the guests. +/// What every guest on the board shares: the storage of a hub beside them, and the memory the build holds for them. #[derive(Clone)] pub(crate) struct Shared { + /// The storage of the hub beside the guests. + #[cfg(not(remote_hub))] buckets: Buckets, + /// The memory the build holds for the guests' memories and tables. memory: MemoryBudget, } @@ -212,35 +79,72 @@ impl Shared { )] pub(crate) fn new() -> Self { Self { + #[cfg(not(remote_hub))] buckets: Buckets::default(), memory: MemoryBudget::new(usize::try_from(GUEST_MEMORY).unwrap_or(usize::MAX)), } } + + /// The memory the build holds for the guests, which each guest draws on. + pub(crate) fn memory(&self) -> MemoryBudget { + self.memory.clone() + } + + /// The guests' storage, for the hub beside them to serve. + #[cfg(not(remote_hub))] + pub(crate) fn storage(&self) -> BoardStorage { + BoardStorage::new(self.buckets.clone()) + } + + /// A guest's transport into `hub`, its storage the board's, waiting on `platform`'s clock, not yet open. + #[cfg(not(remote_hub))] + pub(crate) fn hub_transport(&self, hub: &'static Hub, platform: P) -> HubTransport { + HubTransport::new(hub, platform, self.buckets.clone()) + } } -/// Run the guest `name` until it ends, against `hub`, on `platform`, with what it shares with the board's other guests. +/// Run the guest `name` on `platform` until it finishes. /// -/// `component` is the guest as the build compiled it in. How it ended is logged rather than returned, since nothing -/// waits on a runner: the hub carries on without it. +/// It reaches its hub through a transport from `transport`, and keeps its memories in `memory`. `component` is the +/// guest as the build compiled it in. A run the guest itself fails, as it does when its hub drops the connection under +/// it, is run again on a fresh transport after [`RERUN_DELAY`], as a supervisor restarts a runner process; one that +/// cannot load the guest is not, since it would fail the same way every time. How each run ended is logged rather than +/// returned, since nothing waits on a runner: the hub carries on without it. #[expect( clippy::single_call_fn, reason = "shared by the board's guest task and the host's, each of which a build has only one of" )] -pub(crate) async fn run_guest(hub: &'static Hub, name: &str, component: &[u8], platform: P, shared: Shared) -where - S: Stamps + Send + Sync + 'static, +pub(crate) async fn run_guest( + name: &str, + component: &[u8], + platform: P, + mut transport: impl FnMut() -> T, + memory: MemoryBudget, +) where P: Platform + Clone + 'static, + T: Transport + 'static, { - let transport = HubTransport::new(hub, platform.clone()); - let Shared { buckets, memory } = shared; - let outcome = match board_engine() { - Ok(engine) => run(&engine, component, platform.clone(), transport, buckets, memory).await, - Err(error) => Err(BoardError::Wasmtime(error)), - }; - let ended = match outcome { - Ok(()) => "finished".to_string(), - Err(error) => error.to_string(), + let engine = match board_engine() { + Ok(engine) => engine, + Err(error) => return report(&platform, name, &BoardError::Wasmtime(error).to_string()), }; + loop { + match run(&engine, component, platform.clone(), transport(), memory.clone()).await { + Ok(()) => return report(&platform, name, "finished"), + Err(error @ BoardError::Guest(_)) => { + report(&platform, name, &alloc::format!("{error}; running it again")); + platform.sleep_until(platform.now().saturating_add(RERUN_DELAY)).await; + } + Err(error) => return report(&platform, name, &error.to_string()), + } + } +} + +/// Log how a run of the guest `name` ended, on `platform`'s stderr. +fn report

(platform: &P, name: &str, ended: &str) +where + P: Platform, +{ platform.write( et_ws_wasi_runner::board::Console::Stderr, alloc::format!("runner {name}: {ended}\n").as_bytes(), diff --git a/services/ws-server-firmware/src/server.rs b/services/ws-server-firmware/src/server.rs new file mode 100644 index 00000000..da41c2a2 --- /dev/null +++ b/services/ws-server-firmware/src/server.rs @@ -0,0 +1,87 @@ +//! The hub as the firmware serves it: its `/ws`, with `/health`, `/favicon.ico`, agent storage and the modules built +//! in beside it, through picoserve. +//! +//! As with [`Hub::serve`], the board or the host owns the network stack and the executor, accepts each connection, and +//! runs [`serve`] for it with the one [`Hub`] every connection shares. + +use core::net::IpAddr; + +use et_modules_service::picoserve::{ModuleRoutes, Modules}; +use et_storage_service::picoserve::{Storage, StorageRoutes, path}; +use et_ws_service::health::HealthResponse; +use et_ws_service::hub::Stamps; +use et_ws_service::picoserve_hub::Hub; +use picoserve::io::Socket; +use picoserve::response::{Json, StatusCode}; +use picoserve::routing::get; +use picoserve::{Config, Router, Server, Timer}; + +/// What every connection is served from. +/// +/// That is the hub, the modules built in, where agents' storage is kept, and the server's settings. +#[derive(Clone, Copy)] +pub(crate) struct Served<'served, T, S> { + hub: &'served Hub, + modules: &'served Modules, + storage: &'served S, + config: &'served Config, +} + +impl<'served, T, S> Served<'served, T, S> { + /// Serve `modules` and `storage` beside `hub`, as `config` says, usually [`et_ws_service::picoserve_hub::config`]. + #[expect( + clippy::single_call_fn, + reason = "the constructor, called where each connection is served, on the board or on the host" + )] + pub(crate) const fn new( + hub: &'served Hub, + modules: &'served Modules, + storage: &'served S, + config: &'served Config, + ) -> Self { + Self { + hub, + modules, + storage, + config, + } + } +} + +/// Serve one accepted connection from `client_ip` until it closes. +/// +/// `GET /health` answers what the `std` server's does, `GET /favicon.ico` answers 204, and `GET /ws` is the hub's, +/// as [`Hub::routes`] describes. `/storage/{agent_id}/{filename}` is the storage the `std` server serves, kept in +/// `served.storage`. Every other request is one of the modules', or answered 404. `http_buffer` holds the request +/// head. +#[expect( + clippy::future_not_send, + clippy::single_call_fn, + reason = "picoserve's futures hold Cells, run on one thread; the board and the host each call it once" +)] +pub(crate) async fn serve( + served: Served<'_, T, K>, + socket: S, + client_ip: IpAddr, + timer: M, + http_buffer: &mut [u8], +) -> Result<(), picoserve::Error> +where + T: Stamps, + M: Timer, + S: Socket, + K: Storage, +{ + let storage = StorageRoutes::new(served.hub.registry(), served.storage, client_ip); + let app = served.hub.routes( + Router::from_service(ModuleRoutes::new(served.modules)) + .route_service(path(), storage) + .route("/favicon.ico", get(async || StatusCode::NO_CONTENT)) + .route("/health", get(async || Json(HealthResponse::healthy()))), + client_ip, + ); + let _disconnection = Server::custom(&app, timer, served.config, http_buffer) + .serve(socket) + .await?; + Ok(()) +} diff --git a/services/ws-server-firmware/tests/serves_modules.rs b/services/ws-server-firmware/tests/serves_modules.rs index 7e155c19..1a9459e8 100644 --- a/services/ws-server-firmware/tests/serves_modules.rs +++ b/services/ws-server-firmware/tests/serves_modules.rs @@ -13,6 +13,9 @@ use std::process::{Child, Stdio}; use edge_toolkit::ws::{ClientMessage, ServerMessage}; use tungstenite::Message; +/// A WebSocket to the hub. +type Socket = tungstenite::WebSocket>; + /// The host binary, serving on a port of its choosing, killed when dropped. struct Server { child: Child, @@ -47,11 +50,34 @@ impl Server { } } + /// An agent connected to the hub, and the id it was given. + fn connected(&self) -> (Socket, String) { + let (mut socket, _response) = tungstenite::connect(format!("ws://{}/ws", self.address)).unwrap(); + let ServerMessage::ConnectAck { agent_id, .. } = + exchange(&mut socket, &ClientMessage::Connect { agent_id: None }) + else { + panic!("the connect was not acknowledged"); + }; + (socket, agent_id) + } + /// The response to a `GET path` carrying `headers`, each a whole `Name: value\r\n` line. fn get(&self, path: &str, headers: &str) -> Response { + self.request(&format!("GET {path}"), headers, b"") + } + + /// The response to a `PUT path` of `body`. + fn put(&self, path: &str, body: &[u8]) -> Response { + let length = format!("Content-Length: {}\r\n", body.len()); + self.request(&format!("PUT {path}"), &length, body) + } + + /// The response to `method_path`, a method and a path, carrying `headers` and then `body`. + fn request(&self, method_path: &str, headers: &str, body: &[u8]) -> Response { let mut socket = TcpStream::connect(self.address).unwrap(); - let request = format!("GET {path} HTTP/1.1\r\nHost: board\r\n{headers}Connection: close\r\n\r\n"); + let request = format!("{method_path} HTTP/1.1\r\nHost: board\r\n{headers}Connection: close\r\n\r\n"); socket.write_all(request.as_bytes()).unwrap(); + socket.write_all(body).unwrap(); let mut raw = Vec::new(); let _read = socket.read_to_end(&mut raw).unwrap(); let split = raw.windows(4).position(|window| window == b"\r\n\r\n").unwrap(); @@ -142,10 +168,7 @@ fn it_serves_every_module_it_lists() { /// Send `message` to the hub over `socket`, and decode its reply. /// /// What other agents send in the meantime is passed over: a runner the build compiled in may be broadcasting. -fn exchange( - socket: &mut tungstenite::WebSocket>, - message: &ClientMessage, -) -> ServerMessage { +fn exchange(socket: &mut Socket, message: &ClientMessage) -> ServerMessage { socket .send(Message::text(serde_json::to_string(message).unwrap())) .unwrap(); @@ -161,11 +184,7 @@ fn exchange( fn every_runner_the_build_compiled_in_joins_the_hub() { let server = Server::start(); let runners = server.runners; - let (mut socket, _response) = tungstenite::connect(format!("ws://{}/ws", server.address)).unwrap(); - let ServerMessage::ConnectAck { agent_id, .. } = exchange(&mut socket, &ClientMessage::Connect { agent_id: None }) - else { - panic!("the connect was not acknowledged"); - }; + let (mut socket, agent_id) = server.connected(); let mut joined = || { let ServerMessage::ListAgentsResponse { agents } = exchange(&mut socket, &ClientMessage::ListAgents) else { panic!("the agents were not listed"); @@ -183,6 +202,20 @@ fn every_runner_the_build_compiled_in_joins_the_hub() { assert_eq!(others, runners); } +#[test] +fn an_agent_stores_files_the_hub_serves_back() { + // The storage a runner elsewhere reads what the board's own guests wrote from, and writes what they read. + let server = Server::start(); + let (_socket, agent_id) = server.connected(); + let path = format!("/storage/{agent_id}/input.json"); + assert_eq!(server.get(&path, "").status, 404); + assert_eq!(server.put(&path, b"[1, 2]").status, 200); + let stored = server.get(&path, ""); + assert_eq!(stored.status, 200); + assert_eq!(stored.body, b"[1, 2]"); + assert_eq!(server.put("/storage/nobody/input.json", b"[]").status, 404); +} + #[test] fn an_agent_is_given_an_id_and_told_the_time() { // The host's own ids and clock, which the board's hub stamps every agent and message with in its place. diff --git a/services/ws-server/Cargo.toml b/services/ws-server/Cargo.toml index 8a64136f..ef755630 100644 --- a/services/ws-server/Cargo.toml +++ b/services/ws-server/Cargo.toml @@ -18,80 +18,32 @@ doctest = false doctest = false name = "et-ws-server" path = "src/main.rs" -required-features = ["std"] [[test]] name = "configure_app" required-features = ["modules", "storage"] -[[test]] -name = "health" -required-features = ["std"] - -[[test]] -name = "http_config" -required-features = ["std"] - -[[test]] -name = "net" -required-features = ["std"] - -[[test]] -name = "picoserve_server" -required-features = ["picoserve-server"] - [[test]] name = "tls" required-features = ["tls-self-signed"] [features] #! Every service the hub can mount beyond `/ws` and `/health` is a feature, all on by default. -#! A deployment that needs only the hub sets `default-features = false` and opts back into `std` and the services it -#! serves. +#! A deployment that needs only the hub sets `default-features = false` and opts back into the services it serves. -default = [ - "modules", - "otlp", - "picoserve-server", - "qr", - "std", - "storage", - "tls", - "tls-self-signed", - "tty-image", - "websockify", -] -## The actix-web server and its binary, configured from the environment, with the NATS backplane. -## Without it the crate is `no_std` + `alloc`, and serves through picoserve under `picoserve-server`. -std = [ - "dep:actix-web", - "dep:et-actix-web-config", - "dep:local-ip-address", - "dep:serde-env", - "dep:serde_default", - "dep:thiserror", - "dep:tracing-actix-web", - "dep:tracing-subscriber", - "edge-toolkit/std", - "et-ws-service/nats", - "et-ws-service/std", - "serde/std", - "tracing/std", -] -## A `no_std` server for `/ws`, `/health`, `/favicon.ico` and built-in modules over picoserve, on any socket. -picoserve-server = ["dep:picoserve", "et-ws-service/picoserve-hub"] +default = ["modules", "otlp", "qr", "storage", "tls", "tls-self-signed", "tty-image", "websockify"] ## Serves ws-module packages from disk under `/modules`, and the root module as the browser UI at `/`. -modules = ["dep:et-modules-service", "std"] +modules = ["dep:et-modules-service"] ## Exports traces and logs over OTLP when `OTLP_*` is configured. ## Each request span is parented on the caller's W3C `traceparent`. -otlp = ["dep:et-otlp", "et-ws-service/otel", "std", "tracing-actix-web/opentelemetry_0_32"] +otlp = ["dep:et-otlp", "et-ws-service/otel", "tracing-actix-web/opentelemetry_0_32"] ## Prints the advertised URL as a terminal QR code at startup, for phones joining the hub. -qr = ["dep:qr2term", "std"] +qr = ["dep:qr2term"] ## Per-agent file storage under `/storage`, backed by any `object_store` URL. -storage = ["dep:et-storage-service", "std"] +storage = ["dep:et-storage-service"] ## Adds the HTTPS/WSS listener, serving the PEM pair named by `TLS_CERT_FILE` and `TLS_KEY_FILE`. ## Without it the hub serves plain HTTP only, which browsers refuse for camera, sensors and other secure-context APIs. -tls = ["actix-web/rustls-0_23", "dep:rustls", "dep:serde-inline-default", "et-actix-web-config/rustls-0_23", "std"] +tls = ["actix-web/rustls-0_23", "dep:rustls", "dep:serde-inline-default", "et-actix-web-config/rustls-0_23"] ## Generates a self-signed `localhost` pair into `TLS_CERT_FILE` and `TLS_KEY_FILE` when they do not exist yet. ## Without it a missing pair is a startup error, for deployments that provision their own certificates. tls-self-signed = ["dep:fs-err", "dep:rcgen", "tls"] @@ -100,7 +52,7 @@ tls-self-signed = ["dep:fs-err", "dep:rcgen", "tls"] tty-image = ["et-storage-service/tty-image", "storage"] ## Relays `/websockify` WebSocket traffic to the hub's own loopback HTTP port. ## It serves browser runtimes such as webR that cannot open a TCP socket. -websockify = ["dep:et-websockify-service", "std"] +websockify = ["dep:et-websockify-service"] ## Adds OpenAPI metadata to the REST handlers for `et-int-gen` to harvest into the published spec. ## Production builds do not need it. openapi-spec = [ @@ -108,7 +60,6 @@ openapi-spec = [ "et-modules-service?/openapi-spec", "et-storage-service?/openapi-spec", "et-ws-service/openapi-spec", - "std", ] ## Internal: derives JSON Schema for the hub's environment configuration. ## et-int-gen enables it to render the hub's environment variables into its HELP.md. @@ -119,46 +70,39 @@ env-schema = [ "et-modules-service?/env-schema", "et-storage-service?/env-schema", "et-ws-service/env-schema", - "std", ] ## Internal: renders this feature list into the crate documentation, as docs.rs builds it. docs = ["dep:document-features"] [dependencies] -actix-web = { workspace = true, optional = true } +actix-web.workspace = true document-features = { workspace = true, optional = true } -edge-toolkit.workspace = true -et-actix-web-config = { workspace = true, optional = true } -et-modules-service = { workspace = true, optional = true } +edge-toolkit = { workspace = true, features = ["std"] } +et-actix-web-config.workspace = true +et-modules-service = { workspace = true, features = ["std"], optional = true } et-otlp = { workspace = true, optional = true } -et-storage-service = { workspace = true, optional = true } +et-storage-service = { workspace = true, features = ["std"], optional = true } et-websockify-service = { workspace = true, optional = true } -et-ws-service.workspace = true +et-ws-service = { workspace = true, features = ["nats", "std"] } fs-err = { workspace = true, optional = true } -local-ip-address = { workspace = true, optional = true } -picoserve = { workspace = true, features = ["json"], optional = true } +local-ip-address.workspace = true qr2term = { workspace = true, optional = true } rcgen = { workspace = true, optional = true } rustls = { workspace = true, optional = true } schemars = { workspace = true, optional = true } -serde = { workspace = true, features = ["alloc"] } -serde-env = { workspace = true, optional = true } +serde = { workspace = true, features = ["std"] } +serde-env.workspace = true serde-inline-default = { workspace = true, optional = true } -serde_default = { workspace = true, optional = true } -thiserror = { workspace = true, features = ["std"], optional = true } -tracing.workspace = true -tracing-actix-web = { workspace = true, optional = true } -tracing-subscriber = { workspace = true, optional = true } +serde_default.workspace = true +thiserror = { workspace = true, features = ["std"] } +tracing = { workspace = true, features = ["std"] } +tracing-actix-web.workspace = true +tracing-subscriber.workspace = true utoipa = { workspace = true, optional = true } [dev-dependencies] actix-rt.workspace = true -futures-util.workspace = true -picoserve = { workspace = true, features = ["tokio"] } -serde_json = { workspace = true, features = ["std"] } tempfile.workspace = true -tokio = { workspace = true, features = ["io-util", "macros", "net", "rt", "time"] } -tokio-tungstenite = { workspace = true, features = ["connect"] } [lints] workspace = true diff --git a/services/ws-server/src/lib.rs b/services/ws-server/src/lib.rs index dbbdfe1e..07d6083d 100644 --- a/services/ws-server/src/lib.rs +++ b/services/ws-server/src/lib.rs @@ -2,42 +2,26 @@ #![cfg_attr(feature = "docs", doc = "## Feature flags")] #![cfg_attr(feature = "docs", doc = document_features::document_features!())] #![cfg_attr(docsrs, feature(doc_cfg))] -#![cfg_attr(not(feature = "std"), no_std)] - -#[cfg(not(feature = "std"))] -extern crate alloc; #[cfg(feature = "websockify")] use std::net::{Ipv4Addr, SocketAddr}; -#[cfg(feature = "std")] use actix_web::{HttpResponse, web}; -#[cfg(feature = "std")] pub use et_ws_service::{AgentSession, WsAgentRegistry}; -#[cfg(feature = "std")] pub mod config; -#[cfg(feature = "std")] pub mod net; -#[cfg(feature = "picoserve-server")] -pub mod picoserve_modules; -#[cfg(feature = "picoserve-server")] -pub mod picoserve_server; pub mod routes; #[cfg(feature = "tls")] pub mod tls; -#[cfg(feature = "std")] -use self::config::Config; -#[cfg(feature = "std")] pub use self::routes::health; +use crate::config::Config; -#[cfg(feature = "std")] pub async fn no_content() -> HttpResponse { HttpResponse::NoContent().finish() } -#[cfg(feature = "std")] pub fn configure_app(cfg: &mut web::ServiceConfig, agent_registry: web::Data, config: &Config) { let _configured = cfg .app_data(agent_registry) diff --git a/services/ws-server/src/picoserve_server.rs b/services/ws-server/src/picoserve_server.rs deleted file mode 100644 index 2dd3d471..00000000 --- a/services/ws-server/src/picoserve_server.rs +++ /dev/null @@ -1,54 +0,0 @@ -//! The `no_std` ws-server: the hub's `/ws`, with `/health`, `/favicon.ico` and the modules beside it, served through -//! picoserve. -//! -//! As with [`Hub::serve`], the embedder owns the network stack and the executor, accepts each connection, and runs -//! [`serve`] for it with the one [`Hub`] every connection shares. The modules are those built into the binary, as -//! [`Modules`] describes. The other services the `std` server mounts besides the hub -- storage, websockify -- need a -//! filesystem or an OS, and are not served here. - -use core::net::IpAddr; - -use et_ws_service::hub::Stamps; -use et_ws_service::picoserve_hub::Hub; -use picoserve::io::Socket; -use picoserve::response::{Json, StatusCode}; -use picoserve::routing::get; -use picoserve::{Config, Router, Server, Timer}; - -use crate::picoserve_modules::{ModuleRoutes, Modules}; -use crate::routes::HealthResponse; - -/// Serve one accepted connection from `client_ip` until it closes. -/// -/// `GET /health` answers what the `std` server's does, `GET /favicon.ico` answers 204, and `GET /ws` is `hub`'s, -/// as [`Hub::routes`] describes. Every other request is one of `modules`', or answered 404. `config` is usually -/// [`et_ws_service::picoserve_hub::config`], and `http_buffer` holds the request head. -/// -/// As for [`Hub::serve`], the crate awaiting this one needs `#![recursion_limit = "256"]`. -#[expect( - clippy::future_not_send, - reason = "picoserve's futures hold Cells; its executors run each connection on one thread" -)] -pub async fn serve( - hub: &Hub, - modules: &Modules, - socket: S, - client_ip: IpAddr, - timer: M, - config: &Config, - http_buffer: &mut [u8], -) -> Result<(), picoserve::Error> -where - T: Stamps, - M: Timer, - S: Socket, -{ - let app = hub.routes( - Router::from_service(ModuleRoutes::new(modules)) - .route("/favicon.ico", get(async || StatusCode::NO_CONTENT)) - .route("/health", get(async || Json(HealthResponse::healthy()))), - client_ip, - ); - let _disconnection = Server::custom(&app, timer, config, http_buffer).serve(socket).await?; - Ok(()) -} diff --git a/services/ws-server/src/routes.rs b/services/ws-server/src/routes.rs index 837455e3..6f172ee6 100644 --- a/services/ws-server/src/routes.rs +++ b/services/ws-server/src/routes.rs @@ -1,48 +1,20 @@ -//! `/health` route handler and its response schema, carrying the -//! `#[utoipa::path]` and `utoipa::ToSchema` annotations. +//! `/health` route handler, carrying the `#[utoipa::path]` annotation. Its response schema is et-ws-service's +//! `HealthResponse`, which a board's hub answers with too. //! -//! Lives in its own file so the file-level -//! `#![expect(clippy::exhaustive_structs)]` (active under the -//! `openapi-spec` feature) is scoped only to where utoipa's derives -//! emit `pub struct`s without `#[non_exhaustive]` -- neither location -//! can be silenced via a function- or item-level `#[expect]`. +//! Lives in its own file so the file-level `#![expect(clippy::exhaustive_structs)]` (active under the `openapi-spec` +//! feature) is scoped only to where utoipa's path macro emits a `pub struct` without `#[non_exhaustive]`, which no +//! item-level `#[expect]` can reach. #![cfg_attr( feature = "openapi-spec", expect( clippy::exhaustive_structs, - reason = "utoipa emits sibling pub structs without #[non_exhaustive]; only file scope can silence the lint" + reason = "utoipa emits a sibling pub struct without #[non_exhaustive]; only file scope can silence the lint" ) )] -#[cfg(not(feature = "std"))] -use alloc::string::{String, ToString as _}; - -#[cfg(feature = "std")] use actix_web::HttpResponse; -use serde::{Deserialize, Serialize}; - -/// Server liveness probe response. -/// -/// Returned by `GET /health`. -#[derive(Clone, Debug, Deserialize, Serialize)] -#[cfg_attr(feature = "openapi-spec", derive(utoipa::ToSchema))] -#[non_exhaustive] -pub struct HealthResponse { - pub status: String, - pub service: String, -} - -impl HealthResponse { - /// What a server that is up answers, whichever server it is. - #[must_use] - pub fn healthy() -> Self { - Self { - status: "healthy".to_string(), - service: "ws-server".to_string(), - } - } -} +use et_ws_service::health::HealthResponse; /// Liveness probe. /// @@ -57,7 +29,6 @@ impl HealthResponse { responses((status = 200, description = "Server is up", body = HealthResponse)) ) )] -#[cfg(feature = "std")] pub async fn health() -> HttpResponse { HttpResponse::Ok().json(HealthResponse::healthy()) } diff --git a/services/ws-server/tests/health.rs b/services/ws-server/tests/health.rs index 8f135839..647d55a8 100644 --- a/services/ws-server/tests/health.rs +++ b/services/ws-server/tests/health.rs @@ -4,7 +4,7 @@ use actix_web::http::header; use actix_web::{App, test, web}; use et_ws_server::health; -use et_ws_server::routes::HealthResponse; +use et_ws_service::health::HealthResponse; #[actix_rt::test] async fn health_returns_200_with_healthy_status() { diff --git a/services/ws-test-server/Cargo.toml b/services/ws-test-server/Cargo.toml index efbe3df2..4467b969 100644 --- a/services/ws-test-server/Cargo.toml +++ b/services/ws-test-server/Cargo.toml @@ -14,8 +14,8 @@ doctest = false actix-rt.workspace = true actix-web.workspace = true edge-toolkit = { workspace = true, features = ["std"] } -et-modules-service.workspace = true -et-storage-service.workspace = true +et-modules-service = { workspace = true, features = ["std"] } +et-storage-service = { workspace = true, features = ["std"] } et-test-helpers.workspace = true et-ws-service = { workspace = true, features = ["nats", "otel"] } fs-err.workspace = true diff --git a/services/ws-wasi-runner/Cargo.toml b/services/ws-wasi-runner/Cargo.toml index a9bebc81..8d83ece8 100644 --- a/services/ws-wasi-runner/Cargo.toml +++ b/services/ws-wasi-runner/Cargo.toml @@ -33,6 +33,10 @@ required-features = ["std"] name = "board" required-features = ["board", "precompile", "std"] +[[test]] +name = "board_net" +required-features = ["board", "precompile", "std"] + [[test]] name = "modules" required-features = ["std"] @@ -75,6 +79,7 @@ std = [ "dep:tracing", "dep:wasmtime-wasi", "edge-toolkit/std", + "embedded-io-async?/std", "serde_json/std", "thiserror/std", "wasmtime/default", @@ -85,7 +90,8 @@ precompile = ["wasmtime/cranelift", "wasmtime/pulley", "wasmtime/std"] ## The runner for a board with no operating system, `board::run`, which a firmware runs its guests on. ## It hosts a component compiled ahead of time on the clock, log and websocket the firmware provides, interpreting its ## Pulley bytecode, and keeps the board's storage buckets under a spin lock, since a board has no `std::sync::Mutex`. -board = ["dep:spin", "wasmtime/pulley"] +## Its `board::net` reaches a hub on another machine over the board's own sockets. +board = ["dep:edge-ws", "dep:embassy-sync", "dep:embedded-io-async", "dep:spin", "wasmtime/pulley"] ## Hosts `wasi:nn` (graph, tensor, inference) on ONNX Runtime through `wasmtime-wasi-nn`. ## Guests can then load an ONNX model and run inference on it. nn = ["dep:ort", "dep:wasmtime-wasi-nn", "std"] @@ -110,6 +116,10 @@ docs = ["dep:document-features"] chrono = { workspace = true, features = ["alloc"] } document-features = { workspace = true, optional = true } edge-toolkit.workspace = true +# A board's websocket to a hub elsewhere: the frames, the bytes they cross a socket as, and the queues between them. +edge-ws = { workspace = true, optional = true } +embassy-sync = { workspace = true, optional = true } +embedded-io-async = { workspace = true, optional = true } serde_json = { workspace = true, features = ["alloc"] } serde_path_to_error.workspace = true # The lock a board's guests share their buckets under, when there is no `std::sync::Mutex`. @@ -150,6 +160,7 @@ libc = { workspace = true, optional = true } [dev-dependencies] command-error.workspace = true +critical-section = { workspace = true, features = ["std"] } et-org.workspace = true et-test-helpers.workspace = true et-test-otlp.workspace = true diff --git a/services/ws-wasi-runner/src/board/buckets.rs b/services/ws-wasi-runner/src/board/buckets.rs new file mode 100644 index 00000000..bb8480e6 --- /dev/null +++ b/services/ws-wasi-runner/src/board/buckets.rs @@ -0,0 +1,46 @@ +//! Agent storage kept in a board's memory, for a transport whose hub runs on the same board. + +use alloc::collections::BTreeMap; +#[cfg(not(feature = "std"))] +use alloc::{string::String, vec::Vec}; + +/// Every storage bucket on one board, by agent id, each a map from key to value. +/// +/// A handle: its clones are the same buckets, so every guest a board runs, and the hub beside them, can read what +/// another wrote, as agents share the hub's storage elsewhere. +#[derive(Clone, Default)] +pub struct Buckets(alloc::sync::Arc>>); + +/// What one storage bucket holds, a map from key to value. +type Stored = BTreeMap>; + +/// That a bucket holds no value under the key asked for. +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +#[non_exhaustive] +pub struct Absent; + +impl Buckets { + /// What `bucket` holds under `key`. + /// + /// # Errors + /// + /// [`Absent`] when the bucket holds nothing under `key`. + pub fn get(&self, bucket: &str, key: &str) -> Result, Absent> { + self.0 + .lock() + .get(bucket) + .and_then(|stored| stored.get(key)) + .cloned() + .ok_or(Absent) + } + + /// Hold `value` under `key` in `bucket`, replacing what was there. + pub fn set(&self, bucket: &str, key: &str, value: Vec) { + let _previous = self + .0 + .lock() + .entry(String::from(bucket)) + .or_default() + .insert(String::from(key), value); + } +} diff --git a/services/ws-wasi-runner/src/board/host.rs b/services/ws-wasi-runner/src/board/host.rs index 0458430b..89b808cf 100644 --- a/services/ws-wasi-runner/src/board/host.rs +++ b/services/ws-wasi-runner/src/board/host.rs @@ -1,7 +1,7 @@ -//! A board's store data, and its hosting of the `runner` world: the guest's log through the board, its buckets in the -//! board's memory, and its websocket over the board's transport. +//! A board's store data, and its hosting of the `runner` world: the guest's log through the board, and its websocket +//! and storage buckets the hub's, over the board's transport. -use alloc::collections::{BTreeMap, VecDeque}; +use alloc::collections::VecDeque; #[cfg(not(feature = "std"))] use alloc::{ string::{String, ToString as _}, @@ -26,26 +26,14 @@ use crate::keyvalue::{Bucket, bucket_from_identifier, kv_not_implemented}; /// How long `connect` waits for each frame while it waits for the hub's `et-connect-ack`. const CONNECT_ACK_TIMEOUT_MS: u32 = 10_000; -/// Every storage bucket the guests on one board have written, by agent id, each a map from key to value. -/// -/// A handle: its clones are the same buckets, so every guest a board runs is handed one and can read what another -/// wrote, as agents share the hub's storage elsewhere. -#[derive(Clone, Default)] -pub struct Buckets(alloc::sync::Arc>>); - -/// What one storage bucket holds, a map from key to value. -type Stored = BTreeMap>; - /// The store data a board runs a guest with. pub struct BoardHost { /// The board's clock, log and console. platform: P, /// The guest's resources: its buckets, streams and pollables. table: ResourceTable, - /// The websocket to the hub. + /// The hub: its websocket, and the storage the guest's storage buckets are in. transport: T, - /// The board's storage, which the guest's storage buckets are in. - buckets: Buckets, /// The guest's draw on the board's memory, which wasmtime asks before each memory or table grows. allowance: Allowance, /// Messages that arrived before the guest asked for them: the connect ack, and any that came ahead of it. @@ -57,7 +45,9 @@ pub struct BoardHost { } impl BoardHost { - /// A guest's store data, keeping its storage in `buckets` and its memories in `memory`, with no websocket open. + /// A guest's store data, with no websocket open. + /// + /// The guest reaches the hub through `transport`, and keeps its memories in `memory`. #[cfg_attr( not(test), expect( @@ -65,12 +55,11 @@ impl BoardHost { reason = "the store data's constructor, called where run makes the store" ) )] - pub(super) fn new(platform: P, transport: T, buckets: Buckets, memory: MemoryBudget) -> Self { + pub(super) fn new(platform: P, transport: T, memory: MemoryBudget) -> Self { Self { platform, table: ResourceTable::new(), transport, - buckets, allowance: Allowance::new(memory), inbox: VecDeque::new(), agent_id: String::default(), @@ -145,47 +134,48 @@ impl wasi::keyvalue::store::Host for BoardHost } } -#[expect( - clippy::unused_async_trait_impl, - reason = "the generated HostBucket trait declares every method async, and the buckets are in memory" -)] impl wasi::keyvalue::store::HostBucket for BoardHost { async fn get(&mut self, self_: Resource, key: String) -> Result>, StoreError> { - Ok(match self.table.get(&self_)? { - Bucket::Storage { agent_id } => self - .buckets - .0 - .lock() - .get(agent_id) - .and_then(|stored| stored.get(&key)) - .cloned(), + let Bucket::Storage { agent_id } = self.table.get(&self_)? else { // A board serves no module's files, so none of them holds a key. - Bucket::Modules { .. } => None, - }) + return Ok(None); + }; + match self.transport.load(agent_id, &key).await { + Ok(value) => Ok(value), + Err(error) => Err(StoreError::Other(error.to_string())), + } } async fn set(&mut self, self_: Resource, key: String, value: Vec) -> Result<(), StoreError> { let Bucket::Storage { agent_id } = self.table.get(&self_)? else { return Err(StoreError::AccessDenied); }; - let _previous = self - .buckets - .0 - .lock() - .entry(agent_id.clone()) - .or_default() - .insert(key, value); - Ok(()) + match self.transport.store(agent_id, &key, value).await { + Ok(()) => Ok(()), + Err(error) => Err(StoreError::Other(error.to_string())), + } } + #[expect( + clippy::unused_async_trait_impl, + reason = "the generated HostBucket trait declares every method async, and a board refuses this one at once" + )] async fn delete(&mut self, _bucket: Resource, _key: String) -> Result<(), StoreError> { Err(kv_not_implemented("delete")) } + #[expect( + clippy::unused_async_trait_impl, + reason = "the generated HostBucket trait declares every method async, and a board refuses this one at once" + )] async fn exists(&mut self, _bucket: Resource, _key: String) -> Result { Err(kv_not_implemented("exists")) } + #[expect( + clippy::unused_async_trait_impl, + reason = "the generated HostBucket trait declares every method async, and a board refuses this one at once" + )] async fn list_keys( &mut self, _bucket: Resource, @@ -194,6 +184,10 @@ impl wasi::keyvalue::store::HostBucket for BoardHost< Err(kv_not_implemented("list-keys")) } + #[expect( + clippy::unused_async_trait_impl, + reason = "the generated HostBucket trait declares every method async, and a bucket handle is in memory" + )] async fn drop(&mut self, rep: Resource) -> wasmtime::Result<()> { let _dropped: Bucket = self.table.delete(rep)?; Ok(()) diff --git a/services/ws-wasi-runner/src/board/mod.rs b/services/ws-wasi-runner/src/board/mod.rs index d9476f63..94576a6f 100644 --- a/services/ws-wasi-runner/src/board/mod.rs +++ b/services/ws-wasi-runner/src/board/mod.rs @@ -3,17 +3,22 @@ //! It runs a component compiled ahead of time for the board's engine, with the `runner` world and the WASI Preview 2 //! its guest imports hosted on what the board provides. +mod buckets; #[cfg(feature = "coverage")] mod cov; mod host; mod memory; +pub mod net; +#[cfg(feature = "std")] +pub mod net_tokio; mod platform; mod run; #[cfg(test)] mod test_host; mod wasi; -pub use self::host::{BoardHost, Buckets}; +pub use self::buckets::{Absent, Buckets}; +pub use self::host::BoardHost; pub use self::memory::MemoryBudget; pub use self::platform::{Console, Platform, Transport}; pub use self::run::{BoardError, run}; diff --git a/services/ws-wasi-runner/src/board/net.rs b/services/ws-wasi-runner/src/board/net.rs new file mode 100644 index 00000000..32af1330 --- /dev/null +++ b/services/ws-wasi-runner/src/board/net.rs @@ -0,0 +1,582 @@ +//! A hub on another machine, reached over the board's own sockets: its websocket, and its storage over HTTP. +//! +//! A guest's [`Transport`] has to be `Send`, and a board's socket usually is not, so the two halves run apart. The +//! guest holds a [`NetTransport`], which only queues requests on a [`NetLink`] and waits on the answers, and a task of +//! the board's drives the sockets with [`drive`], taking the requests off the same link. Each guest has a link and a +//! driving task of its own. + +#![expect( + clippy::single_call_fn, + reason = "each step of driving the sockets, the handshake, the two directions and the response, is named apart" +)] + +#[cfg(not(feature = "std"))] +use alloc::{format, string::String, vec, vec::Vec}; +use core::fmt::{self, Write as _}; +use core::future::poll_fn; +use core::pin::pin; +use core::sync::atomic::{AtomicBool, Ordering}; +use core::task::Poll; + +use edge_ws::io::send; +use edge_ws::{FrameHeader, FrameType}; +use embassy_sync::blocking_mutex::raw::CriticalSectionRawMutex; +use embassy_sync::channel::Channel; +use embedded_io_async::{Read, Write}; + +use super::platform::{Platform, Transport}; +use crate::convert::Frame; + +/// The largest frame the hub may send a guest, which bounds what one takes of a board's heap. +pub const MAX_FRAME: usize = 64 * 1024; +/// The largest HTTP response a storage request reads: a stored file, at most the hub's limit, and its head. +const MAX_RESPONSE: usize = 68 * 1024; +/// How long a websocket goes without a frame from the board before the board pings it, inside the hub's 15 seconds. +const KEEP_ALIVE_NANOS: u64 = 5_000_000_000; +/// Nanoseconds in a millisecond, the unit a guest's `recv` timeout comes in. +const NANOS_PER_MILLI: u64 = 1_000_000; +/// The path the hub serves its websocket on. +const WS_PATH: &str = "/ws"; +/// The longest response head a hub's upgrade answers with. +const MAX_HEAD: usize = 1024; + +/// Why a request to the hub failed, which the guest is told. +#[derive(Clone, Debug, Eq, PartialEq)] +#[non_exhaustive] +pub enum NetError { + /// No socket to the hub could be opened, for the reason given. + Connect(String), + /// The hub did not upgrade the connection to a websocket. + Handshake, + /// The websocket is not open, or the hub closed it. + Closed, + /// The socket failed while it was in use. + Io, + /// A frame or response larger than a board reads. + TooLarge, + /// A response that is not HTTP. + Malformed, + /// The hub answered a storage request with this status. + Status(u16), +} + +impl fmt::Display for NetError { + /// What failed, as the guest is told it. + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + Self::Connect(reason) => write!(f, "could not reach the hub: {reason}"), + Self::Handshake => f.write_str("the hub did not open a websocket"), + Self::Closed => f.write_str("the websocket to the hub is closed"), + Self::Io => f.write_str("the connection to the hub failed"), + Self::TooLarge => f.write_str("the hub sent more than a board reads"), + Self::Malformed => f.write_str("the hub's answer was not HTTP"), + Self::Status(status) => write!(f, "the hub answered {status}"), + } + } +} + +/// What a guest asks of the task driving its sockets. +enum Request { + /// Open the websocket. + Connect, + /// Send a frame on the websocket. + Send(Frame), + /// Answer the hub's ping, with what it carried. + Pong(Vec), + /// Close the websocket. + Close, + /// Read `key` in `bucket` from the hub's storage. + Load { bucket: String, key: String }, + /// Write `value` under `key` in `bucket` to the hub's storage. + Store { + bucket: String, + key: String, + value: Vec, + }, +} + +/// The answer to a request that has one. +enum Reply { + /// The websocket opened, or why not. + Connected(Result<(), NetError>), + /// What a `Load` read. + Loaded(Result>, NetError>), + /// Whether a `Store` was taken. + Stored(Result<(), NetError>), +} + +/// The queues between one guest's [`NetTransport`] and the task driving its sockets. +/// +/// `static` on a board, since both sides hold it for as long as the board runs. +pub struct NetLink { + requests: Channel, + replies: Channel, + incoming: Channel, 4>, + open: AtomicBool, +} + +impl NetLink { + /// A link with nothing queued and no websocket open, made in a `const` so a board can keep it in a `static`. + #[must_use] + pub const fn idle() -> Self { + Self { + requests: Channel::new(), + replies: Channel::new(), + incoming: Channel::new(), + open: AtomicBool::new(false), + } + } + + /// Ask the driving task for `request`, and wait for its answer. + async fn ask(&self, request: Request) -> Reply { + self.requests.send(request).await; + self.replies.receive().await + } +} + +/// A guest's [`Transport`] to a hub on another machine, through the task that drives `link`. +pub struct NetTransport

{ + link: &'static NetLink, + platform: P, +} + +impl

NetTransport

{ + /// A transport through `link`, waiting on `platform`'s clock. + #[must_use] + pub const fn new(link: &'static NetLink, platform: P) -> Self { + Self { link, platform } + } +} + +impl Transport for NetTransport

{ + type Error = NetError; + + /// Open the websocket, dropping whatever an earlier one left unread. + async fn connect(&mut self) -> Result<(), NetError> { + self.link.incoming.clear(); + match self.link.ask(Request::Connect).await { + Reply::Connected(outcome) => outcome, + Reply::Loaded(_) | Reply::Stored(_) => Err(NetError::Malformed), + } + } + + /// Queue `frame` to send; a failure to send it closes the websocket, which the next `recv` reports. + async fn send(&mut self, frame: Frame) -> Result<(), NetError> { + if !self.link.open.load(Ordering::Acquire) { + return Err(NetError::Closed); + } + self.link.requests.send(Request::Send(frame)).await; + Ok(()) + } + + /// The next frame from the hub, if one comes within `timeout_ms`. + async fn recv(&mut self, timeout_ms: u32) -> Result, NetError> { + let deadline = self + .platform + .now() + .saturating_add(u64::from(timeout_ms).saturating_mul(NANOS_PER_MILLI)); + match race(self.link.incoming.receive(), self.platform.sleep_until(deadline)).await { + Won::First(Ok(frame)) => Ok(Some(frame)), + Won::First(Err(error)) => Err(error), + Won::Second(()) => Ok(None), + } + } + + /// Close the websocket. + async fn close(&mut self) { + if self.link.open.load(Ordering::Acquire) { + self.link.requests.send(Request::Close).await; + } + } + + /// What the hub stores under `key` in `bucket`, as `GET /storage/{bucket}/{key}` answers. + async fn load(&mut self, bucket: &str, key: &str) -> Result>, NetError> { + let request = Request::Load { + bucket: String::from(bucket), + key: String::from(key), + }; + match self.link.ask(request).await { + Reply::Loaded(outcome) => outcome, + Reply::Connected(_) | Reply::Stored(_) => Err(NetError::Malformed), + } + } + + /// Store `value` under `key` in `bucket` with `PUT /storage/{bucket}/{key}`. + async fn store(&mut self, bucket: &str, key: &str, value: Vec) -> Result<(), NetError> { + let request = Request::Store { + bucket: String::from(bucket), + key: String::from(key), + value, + }; + match self.link.ask(request).await { + Reply::Stored(outcome) => outcome, + Reply::Connected(_) | Reply::Loaded(_) => Err(NetError::Malformed), + } + } +} + +/// Opens a new socket to the hub each time it is asked, with buffers of its own. +pub trait Connector { + /// The socket, borrowing what the connector keeps for it. + type Socket<'socket>: Read + Write + Split + where + Self: 'socket; + + /// A socket to the hub, freshly connected. + /// + /// # Errors + /// + /// [`NetError::Connect`], saying why the hub could not be reached, as the guest is told it. + fn connect(&mut self) -> impl Future, NetError>>; +} + +/// A socket that can be read and written at once, from two halves. +pub trait Split { + /// The half that reads. + type Reader<'half>: Read + where + Self: 'half; + /// The half that writes. + type Writer<'half>: Write + where + Self: 'half; + + /// The two halves, borrowing the socket. + fn split(&mut self) -> (Self::Reader<'_>, Self::Writer<'_>); +} + +/// Answer what the guest on `link` asks, forever, over sockets to the hub at `host`. +/// +/// `host` is the hub's `address:port`. The websocket's socket comes from `websocket`, and each storage request's from +/// `http`, so a request can be made while the websocket is open. The keep-alive and the frames' masks are timed on +/// `platform`'s clock. +pub async fn drive(link: &NetLink, mut websocket: W, mut http: H, host: &str, platform: P) -> ! +where + W: Connector, + H: Connector, + P: Platform, +{ + loop { + match link.requests.receive().await { + Request::Connect => match websocket.connect().await { + Ok(mut socket) => match handshake(&mut socket, host).await { + Ok(()) => { + link.open.store(true, Ordering::Release); + link.replies.send(Reply::Connected(Ok(()))).await; + session(link, &mut socket, &mut http, host, &platform).await; + link.open.store(false, Ordering::Release); + } + Err(error) => link.replies.send(Reply::Connected(Err(error))).await, + }, + Err(refused) => link.replies.send(Reply::Connected(Err(refused))).await, + }, + Request::Load { bucket, key } => { + let loaded = load(&mut http, host, &bucket, &key).await; + link.replies.send(Reply::Loaded(loaded)).await; + } + Request::Store { bucket, key, value } => { + let stored = store(&mut http, host, &bucket, &key, &value).await; + link.replies.send(Reply::Stored(stored)).await; + } + // With no websocket open there is nothing to send on or close. + Request::Send(_) | Request::Pong(_) | Request::Close => {} + } + } +} + +/// Upgrade `socket` to a websocket on the hub at `host`. +async fn handshake(socket: &mut S, host: &str) -> Result<(), NetError> +where + S: Read + Write, +{ + // The key is only there to be echoed back: a hub checks it is 16 bytes of base64, not that it is random, and a + // board has no use for the answer. Sixteen zero bytes it is. + let request = format!( + concat!( + "GET {path} HTTP/1.1\r\nHost: {host}\r\nUpgrade: websocket\r\nConnection: Upgrade\r\n", + "Sec-WebSocket-Key: AAAAAAAAAAAAAAAAAAAAAA==\r\nSec-WebSocket-Version: 13\r\n\r\n" + ), + path = WS_PATH, + host = host, + ); + socket.write_all(request.as_bytes()).await.map_net_err()?; + let head = read_head(socket).await?; + if head.starts_with(b"HTTP/1.1 101") { + Ok(()) + } else { + Err(NetError::Handshake) + } +} + +/// The response head `socket` sends, up to and including the blank line that ends it. +/// +/// Read a byte at a time, so that nothing after it is taken. +async fn read_head(socket: &mut S) -> Result, NetError> +where + S: Read, +{ + let mut head = Vec::new(); + let mut byte = [0]; + while !head.ends_with(b"\r\n\r\n") { + if head.len() >= MAX_HEAD { + return Err(NetError::TooLarge); + } + socket.read_exact(&mut byte).await.map_net_err()?; + head.push(byte[0]); + } + Ok(head) +} + +/// Carry frames both ways on `socket` until the hub or the guest closes it, keeping the websocket alive. +/// +/// The hub's frames are read throughout. Only the wait for what the guest asks next races that reading: once a request +/// is taken off the queue it runs to its end, a storage request's HTTP and reply included, so a websocket that ends +/// meanwhile cannot drop a request the guest is still waiting on. +async fn session(link: &NetLink, socket: &mut S, http: &mut H, host: &str, platform: &P) +where + S: Split, + H: Connector, + P: Platform, +{ + let (reader, mut writer) = socket.split(); + let mut reading = pin!(read_frames(link, reader)); + loop { + let now = platform.now(); + let idle = now.saturating_add(KEEP_ALIVE_NANOS); + let next = race(link.requests.receive(), platform.sleep_until(idle)); + let Won::Second(next) = race(reading.as_mut(), next).await else { + return; + }; + // The mask is the clock's low bits: a board has no randomness to spare, and a hub needs none. + let mask = Some(u32::try_from(now & u64::from(u32::MAX)).unwrap_or_default()); + let sent = match next { + Won::Second(()) => send(&mut writer, FrameType::Ping, mask, &[]).await, + Won::First(request) => match answer(link, request, &mut writer, mask, http, host).await { + Answered::Continue(sent) => sent, + Answered::Closed => return, + }, + }; + if sent.is_err() { + link.incoming.send(Err(NetError::Io)).await; + return; + } + } +} + +/// What answering one of the guest's requests left the websocket as. +enum Answered { + /// Still open, with whether what was sent on it went. + Continue(Result<(), edge_ws::Error>), + /// Closed by the guest. + Closed, +} + +/// Answer `request` while the websocket is open: send on `writer` what it sends, and reply to what it asks. +async fn answer( + link: &NetLink, + request: Request, + writer: &mut W, + mask: Option, + http: &mut H, + host: &str, +) -> Answered +where + W: Write, + H: Connector, +{ + Answered::Continue(match request { + Request::Send(Frame::Text(text)) => send(writer, FrameType::Text(false), mask, text.as_bytes()).await, + Request::Send(Frame::Binary(bytes)) => send(writer, FrameType::Binary(false), mask, &bytes).await, + Request::Pong(payload) => send(writer, FrameType::Pong, mask, &payload).await, + Request::Close => { + let _closing = send(writer, FrameType::Close, mask, &[]).await; + return Answered::Closed; + } + Request::Connect => { + link.replies.send(Reply::Connected(Ok(()))).await; + Ok(()) + } + Request::Load { bucket, key } => { + let loaded = load(http, host, &bucket, &key).await; + link.replies.send(Reply::Loaded(loaded)).await; + Ok(()) + } + Request::Store { bucket, key, value } => { + let stored = store(http, host, &bucket, &key, &value).await; + link.replies.send(Reply::Stored(stored)).await; + Ok(()) + } + }) +} + +/// Hand every frame the hub sends to the guest, answering its pings, until the websocket ends. +async fn read_frames(link: &NetLink, mut reader: R) +where + R: Read, +{ + let mut message = Vec::new(); + // Whether the message being read is binary, which only its first frame says: the rest are continuations. + let mut binary = false; + let ended = loop { + let header = match FrameHeader::recv(&mut reader).await { + Ok(header) => header, + Err(_failed) => break NetError::Io, + }; + let Some(length) = usize::try_from(header.payload_len) + .ok() + .filter(|length| message.len().saturating_add(*length) <= MAX_FRAME) + else { + break NetError::TooLarge; + }; + let mut payload = vec![0; length]; + if header.recv_payload(&mut reader, &mut payload).await.is_err() { + break NetError::Io; + } + match header.frame_type { + FrameType::Ping => link.requests.send(Request::Pong(payload)).await, + FrameType::Pong => {} + FrameType::Close => break NetError::Closed, + FrameType::Text(_) | FrameType::Binary(_) | FrameType::Continue(_) => { + if !matches!(header.frame_type, FrameType::Continue(_)) { + binary = matches!(header.frame_type, FrameType::Binary(_)); + } + message.extend_from_slice(&payload); + if header.frame_type.is_final() { + let frame = if binary { + Frame::Binary(core::mem::take(&mut message)) + } else { + Frame::Text(String::from_utf8_lossy(&core::mem::take(&mut message)).into_owned()) + }; + link.incoming.send(Ok(frame)).await; + } + } + } + }; + link.incoming.send(Err(ended)).await; +} + +/// What the hub stores under `key` in `bucket`: its body on 200, nothing on 404. +async fn load(http: &mut H, host: &str, bucket: &str, key: &str) -> Result>, NetError> +where + H: Connector, +{ + match request(http, host, "GET", bucket, key, &[]).await? { + (200, body) => Ok(Some(body)), + (404, _) => Ok(None), + (status, _) => Err(NetError::Status(status)), + } +} + +/// Store `value` under `key` in `bucket` on the hub, which answers 200 when it took it. +async fn store(http: &mut H, host: &str, bucket: &str, key: &str, value: &[u8]) -> Result<(), NetError> +where + H: Connector, +{ + match request(http, host, "PUT", bucket, key, value).await? { + (200, _) => Ok(()), + (status, _) => Err(NetError::Status(status)), + } +} + +/// Make `method` on `/storage/{bucket}/{key}` with `body`, on a socket of its own, and return the status and body. +async fn request( + http: &mut H, + host: &str, + method: &str, + bucket: &str, + key: &str, + body: &[u8], +) -> Result<(u16, Vec), NetError> +where + H: Connector, +{ + let mut socket = http.connect().await?; + let head = format!( + "{method} /storage/{}/{} HTTP/1.1\r\nHost: {host}\r\nContent-Length: {}\r\nConnection: close\r\n\r\n", + encoded(bucket), + encoded(key), + body.len() + ); + socket.write_all(head.as_bytes()).await.map_net_err()?; + socket.write_all(body).await.map_net_err()?; + socket.flush().await.map_net_err()?; + let mut response = Vec::new(); + let mut chunk = [0; 512]; + loop { + let read = socket.read(&mut chunk).await.map_net_err()?; + if read == 0 { + break; + } + if response.len().saturating_add(read) > MAX_RESPONSE { + return Err(NetError::TooLarge); + } + response.extend_from_slice(chunk.get(..read).unwrap_or_default()); + } + parse_response(&response) +} + +/// The status and body of a whole HTTP response. +fn parse_response(response: &[u8]) -> Result<(u16, Vec), NetError> { + let split = response + .windows(4) + .position(|window| window == b"\r\n\r\n") + .ok_or(NetError::Malformed)?; + let (head, rest) = response.split_at(split); + let status = core::str::from_utf8(head) + .ok() + .and_then(|head| head.split(' ').nth(1)) + .and_then(|status| status.parse().ok()) + .ok_or(NetError::Malformed)?; + Ok((status, rest.get(4..).unwrap_or_default().to_vec())) +} + +/// `segment` as one path segment: every byte but the unreserved ones percent-encoded. +fn encoded(segment: &str) -> String { + let mut out = String::default(); + for byte in segment.bytes() { + if byte.is_ascii_alphanumeric() || matches!(byte, b'-' | b'.' | b'_' | b'~') { + out.push(char::from(byte)); + } else { + let _written = write!(out, "%{byte:02X}"); + } + } + out +} + +/// A socket call's failure, as the guest is told it: which error the socket gave says nothing a board can act on. +trait MapNetErr { + /// The value, or [`NetError::Io`]. + fn map_net_err(self) -> Result; +} + +impl MapNetErr for Result { + fn map_net_err(self) -> Result { + match self { + Ok(value) => Ok(value), + Err(_failed) => Err(NetError::Io), + } + } +} + +/// Which of two raced futures finished first, and with what. +enum Won { + First(A), + Second(B), +} + +/// Poll `first` and `second` together until one finishes, dropping the other. +async fn race(first: A, second: B) -> Won +where + A: Future, + B: Future, +{ + let mut first = pin!(first); + let mut second = pin!(second); + poll_fn(|context| { + if let Poll::Ready(output) = first.as_mut().poll(context) { + return Poll::Ready(Won::First(output)); + } + second.as_mut().poll(context).map(Won::Second) + }) + .await +} diff --git a/services/ws-wasi-runner/src/board/net_tokio.rs b/services/ws-wasi-runner/src/board/net_tokio.rs new file mode 100644 index 00000000..e2a0132f --- /dev/null +++ b/services/ws-wasi-runner/src/board/net_tokio.rs @@ -0,0 +1,82 @@ +//! [`super::net`]'s sockets on a host: tokio's, read and written as a board's are. +//! +//! What a firmware's host build drives its guests' links with, so it reaches a hub elsewhere as the board would. + +use std::net::SocketAddr; + +use embedded_io_async::{ErrorKind, ErrorType, Read, Write}; +use tokio::io::{AsyncRead, AsyncReadExt as _, AsyncWrite, AsyncWriteExt as _}; +use tokio::net::TcpStream; +use tokio::net::tcp::{ReadHalf, WriteHalf}; + +use super::net::{Connector, NetError, Split}; + +/// A tokio socket, or half of one, read and written as a board's socket is. +#[derive(Debug)] +pub struct TokioIo(T); + +impl ErrorType for TokioIo { + type Error = ErrorKind; +} + +impl Read for TokioIo { + /// Read what the socket has into `buf`, its error reduced to its kind, as a board's are. + async fn read(&mut self, buf: &mut [u8]) -> Result { + match self.0.read(buf).await { + Ok(read) => Ok(read), + Err(error) => Err(error.kind().into()), + } + } +} + +impl Write for TokioIo { + /// Write what the socket takes of `buf`, its error reduced to its kind, as a board's are. + async fn write(&mut self, buf: &[u8]) -> Result { + match self.0.write(buf).await { + Ok(written) => Ok(written), + Err(error) => Err(error.kind().into()), + } + } + + /// Send what was written, its error reduced to its kind, as a board's are. + async fn flush(&mut self) -> Result<(), ErrorKind> { + match self.0.flush().await { + Ok(()) => Ok(()), + Err(error) => Err(error.kind().into()), + } + } +} + +impl Split for TokioIo { + type Reader<'half> = TokioIo>; + type Writer<'half> = TokioIo>; + + fn split(&mut self) -> (Self::Reader<'_>, Self::Writer<'_>) { + let (reader, writer) = self.0.split(); + (TokioIo(reader), TokioIo(writer)) + } +} + +/// Opens a tokio socket to the hub at its address each time it is asked. +#[derive(Clone, Copy, Debug)] +pub struct TokioConnector(SocketAddr); + +impl TokioConnector { + /// A connector to the hub at `address`. + #[must_use] + pub const fn new(address: SocketAddr) -> Self { + Self(address) + } +} + +impl Connector for TokioConnector { + type Socket<'socket> = TokioIo; + + /// A socket to the hub, or why it could not be reached. + async fn connect(&mut self) -> Result, NetError> { + match TcpStream::connect(self.0).await { + Ok(stream) => Ok(TokioIo(stream)), + Err(error) => Err(NetError::Connect(error.to_string())), + } + } +} diff --git a/services/ws-wasi-runner/src/board/platform.rs b/services/ws-wasi-runner/src/board/platform.rs index f6a2b861..4e4f1d7e 100644 --- a/services/ws-wasi-runner/src/board/platform.rs +++ b/services/ws-wasi-runner/src/board/platform.rs @@ -1,4 +1,7 @@ -//! What a board gives its runner: a clock and a log, and a websocket to the hub. +//! What a board gives its runner: a clock and a log, and the hub. + +#[cfg(not(feature = "std"))] +use alloc::vec::Vec; use crate::bindings::wasi::logging::logging::Level; use crate::convert::Frame; @@ -28,10 +31,11 @@ pub trait Platform: Send + Sync { fn write(&self, console: Console, bytes: &[u8]); } -/// A websocket to the hub, opened on the guest's `connect`. +/// The hub as the guest reaches it: a websocket, opened on the guest's `connect`, and the hub's agent storage. /// /// Keeping the connection alive is the transport's: the hub closes one it has heard nothing on for 15 seconds, so a -/// transport pings it while the guest is busy elsewhere. +/// transport pings it while the guest is busy elsewhere. The storage is the hub's `/storage/{bucket}/{key}`, which a +/// transport with the hub beside it reads from memory, and one with the hub elsewhere over HTTP. pub trait Transport: Send { /// Why the websocket failed, which the guest is told as a `transport` error. type Error: core::fmt::Display; @@ -61,4 +65,26 @@ pub trait Transport: Send { /// Close the websocket. fn close(&mut self) -> impl Future + Send; + + /// What the hub stores under `key` in `bucket`. + /// + /// `None` is a key the bucket holds nothing under, which is not a failure: it is what `wasi:keyvalue`'s `get` + /// tells the guest then. + /// + /// # Errors + /// + /// When the hub's storage could not be read. + fn load(&mut self, bucket: &str, key: &str) -> impl Future>, Self::Error>> + Send; + + /// Store `value` under `key` in `bucket` on the hub, replacing what was there. + /// + /// # Errors + /// + /// When the hub refused it, or could not be reached. + fn store( + &mut self, + bucket: &str, + key: &str, + value: Vec, + ) -> impl Future> + Send; } diff --git a/services/ws-wasi-runner/src/board/run.rs b/services/ws-wasi-runner/src/board/run.rs index 3aca7d70..45966a9c 100644 --- a/services/ws-wasi-runner/src/board/run.rs +++ b/services/ws-wasi-runner/src/board/run.rs @@ -3,7 +3,7 @@ use wasmtime::component::{Component, HasSelf, Linker}; use wasmtime::{Engine, Store}; -use super::host::{BoardHost, Buckets}; +use super::host::BoardHost; use super::memory::MemoryBudget; use super::platform::{Platform, Transport}; use super::wasi; @@ -31,8 +31,8 @@ impl From for BoardError { /// Run `component` on `engine` until its `entry.run` returns, hosting it on what the board hands it. /// /// `engine` is [`crate::engine::board_engine`]'s, and `component` what `engine::precompile` compiled for it: a firmware -/// builds its guest in, compiled by its build script. The guest runs on `platform` and `transport`, and every guest a -/// board runs is handed the same `buckets` for its storage and the same `memory` for its memories and tables. +/// builds its guest in, compiled by its build script. The guest runs on `platform`, reaches the hub and its storage +/// through `transport`, and every guest a board runs is handed the same `memory` for its memories and tables. /// /// # Errors /// @@ -43,7 +43,6 @@ pub async fn run( component: &[u8], platform: P, transport: T, - buckets: Buckets, memory: MemoryBudget, ) -> Result<(), BoardError> where @@ -60,7 +59,7 @@ where let mut linker = Linker::new(engine); wasi::add_to_linker(&mut linker)?; bindings::Runner::add_to_linker::, HasSelf>>(&mut linker, |host| host)?; - let mut store = Store::new(engine, BoardHost::new(platform, transport, buckets, memory)); + let mut store = Store::new(engine, BoardHost::new(platform, transport, memory)); store.limiter(|host| host.allowance()); let runner = bindings::Runner::instantiate_async(&mut store, &component, &linker).await?; // `entry.run` is an async export, which only the concurrent API can call. diff --git a/services/ws-wasi-runner/src/board/test_host.rs b/services/ws-wasi-runner/src/board/test_host.rs index b3050b17..df4c24f3 100644 --- a/services/ws-wasi-runner/src/board/test_host.rs +++ b/services/ws-wasi-runner/src/board/test_host.rs @@ -11,7 +11,8 @@ use edge_toolkit::ws::{ConnectStatus, ServerMessage}; use wasmtime::ResourceLimiter as _; use wasmtime::component::Resource; -use super::host::{BoardHost, Buckets}; +use super::buckets::Buckets; +use super::host::BoardHost; use super::memory::MemoryBudget; use super::platform::{Console, Platform, Transport}; use crate::bindings::et::ws_messages::messages::ClientMessage; @@ -39,8 +40,21 @@ impl Platform for Quiet { fn write(&self, _console: Console, _bytes: &[u8]) {} } -/// A hub that acks every connect, and has nothing else to say. -struct Acking; +/// A hub that acks every connect, and has nothing else to say, its storage `buckets` unless it is `unreachable`. +struct Acking { + buckets: Buckets, + unreachable: bool, +} + +impl Acking { + /// Its storage, or why it cannot be reached. + const fn storage(&self) -> Result<&Buckets, &'static str> { + if self.unreachable { + return Err("storage unreachable"); + } + Ok(&self.buckets) + } +} #[expect( clippy::unused_async_trait_impl, @@ -66,6 +80,15 @@ impl Transport for Acking { } async fn close(&mut self) {} + + async fn load(&mut self, bucket: &str, key: &str) -> Result>, &'static str> { + Ok(self.storage()?.get(bucket, key).ok()) + } + + async fn store(&mut self, bucket: &str, key: &str, value: Vec) -> Result<(), &'static str> { + self.storage()?.set(bucket, key, value); + Ok(()) + } } /// `future`'s output, which every call below has at once, since nothing in them waits. @@ -86,7 +109,11 @@ fn host() -> BoardHost { /// A guest's store data on a quiet board with an acking hub, sharing `buckets` and `memory` with the board's others. fn on_board(buckets: Buckets, memory: MemoryBudget) -> BoardHost { - BoardHost::new(Quiet, Acking, buckets, memory) + let hub = Acking { + buckets, + unreachable: false, + }; + BoardHost::new(Quiet, hub, memory) } /// A budget no guest here spends. @@ -160,6 +187,22 @@ fn guests_handed_the_same_buckets_read_what_each_other_wrote() { ); } +#[test] +fn a_bucket_on_a_hub_whose_storage_is_unreachable_says_why() { + let hub = Acking { + buckets: Buckets::default(), + unreachable: true, + }; + let mut host = BoardHost::new(Quiet, hub, roomy()); + let rep = open(&mut host, "agent-1").rep(); + for failed in [ + now(host.get(Resource::new_own(rep), "key".to_owned())).err(), + now(host.set(Resource::new_own(rep), "key".to_owned(), vec![1])).err(), + ] { + assert!(matches!(failed, Some(Error::Other(detail)) if detail == "storage unreachable")); + } +} + #[test] fn a_bucket_refuses_the_calls_a_board_does_not_implement() { let mut host = host(); diff --git a/services/ws-wasi-runner/tests/board.rs b/services/ws-wasi-runner/tests/board.rs index 3e7714a6..92160ac4 100644 --- a/services/ws-wasi-runner/tests/board.rs +++ b/services/ws-wasi-runner/tests/board.rs @@ -82,6 +82,7 @@ struct Hub { closed: Arc>, replies: VecDeque, fault: Fault, + storage: Buckets, } impl Hub { @@ -147,6 +148,15 @@ impl Transport for Hub { async fn close(&mut self) { *self.closed.lock().unwrap() = true; } + + async fn load(&mut self, bucket: &str, key: &str) -> Result>, &'static str> { + Ok(self.storage.get(bucket, key).ok()) + } + + async fn store(&mut self, bucket: &str, key: &str, value: Vec) -> Result<(), &'static str> { + self.storage.set(bucket, key, value); + Ok(()) + } } /// `message` as the text frame the hub sends it in. @@ -191,15 +201,7 @@ async fn ran_within( lines: Arc::clone(&lines), }; let (sent, closed) = (Arc::clone(&hub.sent), Arc::clone(&hub.closed)); - let outcome = run( - &board_engine().unwrap(), - component, - host, - hub, - Buckets::default(), - memory, - ) - .await; + let outcome = run(&board_engine().unwrap(), component, host, hub, memory).await; let lines = lines.lock().unwrap().clone(); let sent = sent.lock().unwrap().clone(); let closed = *closed.lock().unwrap(); diff --git a/services/ws-wasi-runner/tests/board_net.rs b/services/ws-wasi-runner/tests/board_net.rs new file mode 100644 index 00000000..04f3873a --- /dev/null +++ b/services/ws-wasi-runner/tests/board_net.rs @@ -0,0 +1,239 @@ +//! A board's guest reaching a hub on another machine through `board::net`, over real sockets on this host. +//! +//! The sockets are tokio's, read and written as a board's are, and the hub is either the in-process ws-server, with +//! its storage, or one scripted here a frame at a time. +#![cfg(test)] + +use std::net::{Ipv4Addr, SocketAddr}; +use std::sync::{Arc, LazyLock, Mutex}; +use std::time::{Duration, Instant}; + +// The host's critical section, which the board's queues lock under, as a board's own does on the board. +use critical_section as _; +use et_ws_wasi_runner::bindings::wasi::logging::logging::Level; +use et_ws_wasi_runner::board::net::{NetError, NetLink, NetTransport, drive}; +use et_ws_wasi_runner::board::net_tokio::TokioConnector; +use et_ws_wasi_runner::board::{Console, MemoryBudget, Platform, Transport as _, run}; +use et_ws_wasi_runner::convert::Frame; +use et_ws_wasi_runner::engine::{NATIVE_PULLEY, board_engine, precompile}; +use tokio::io::{AsyncReadExt as _, AsyncWriteExt as _}; +use tokio::net::{TcpListener, TcpStream}; +use tokio::task::LocalSet; + +/// When the test process started, which the board's clock counts from. +static STARTED: LazyLock = LazyLock::new(Instant::now); + +/// How long a test waits on the hub before failing rather than hanging. +const PATIENCE: Duration = Duration::from_secs(20); + +/// The upgrade a hub answers a websocket's request with. +const UPGRADED: &str = "HTTP/1.1 101 Switching Protocols\r\nUpgrade: websocket\r\nConnection: Upgrade\r\n\r\n"; + +/// A board's clock, this host's, and its log, kept in a vector. +#[derive(Clone, Default)] +struct Board(Arc>>); + +impl Platform for Board { + fn now(&self) -> u64 { + u64::try_from(STARTED.elapsed().as_nanos()).unwrap() + } + + async fn sleep_until(&self, deadline: u64) { + tokio::time::sleep(Duration::from_nanos(deadline.saturating_sub(self.now()))).await; + } + + fn log(&self, _level: Level, _context: &str, message: &str) { + self.0.lock().unwrap().push(message.to_owned()); + } + + fn write(&self, _console: Console, bytes: &[u8]) { + self.0.lock().unwrap().push(String::from_utf8_lossy(bytes).into_owned()); + } +} + +/// A transport to the hub at `address`, its sockets driven on this thread until the test ends. +fn transport_to(address: SocketAddr, board: Board) -> NetTransport { + let link: &'static NetLink = Box::leak(Box::new(NetLink::idle())); + let host = address.to_string(); + let driving = board.clone(); + let dial = TokioConnector::new(address); + let _driving = tokio::task::spawn_local(async move { + drive(link, dial, dial, &host, driving).await; + }); + NetTransport::new(link, board) +} + +#[tokio::test(flavor = "current_thread")] +async fn a_guest_round_trips_a_value_through_a_hub_elsewhere() { + let server = et_ws_test_server::start(); + let address: SocketAddr = server.base_url.trim_start_matches("http://").parse().unwrap(); + let board = Board::default(); + LocalSet::new() + .run_until(async { + let transport = transport_to(address, board.clone()); + let built = edge_toolkit::config::get_project_root() + .join("services/ws-modules/wasi-data1/pkg/et_ws_wasi_data1.wasm"); + let component = precompile(NATIVE_PULLEY, &fs_err::read(built).unwrap()).unwrap(); + let engine = board_engine().unwrap(); + let ran = run( + &engine, + &component, + board.clone(), + transport, + MemoryBudget::new(usize::MAX), + ); + tokio::time::timeout(PATIENCE, ran).await.unwrap().unwrap(); + }) + .await; + let lines = board.0.lock().unwrap().clone(); + assert!(lines.iter().any(|line| line.contains("VERIFICATION SUCCESS"))); + // What the guest stored is in the hub's storage, not the board's. + let buckets = fs_err::read_dir(server.storage_dir.path()).unwrap().count(); + assert!(buckets > 0); +} + +#[tokio::test(flavor = "current_thread")] +async fn storage_answers_nothing_stored_and_refuses_a_stranger() { + let server = et_ws_test_server::start(); + let address: SocketAddr = server.base_url.trim_start_matches("http://").parse().unwrap(); + LocalSet::new() + .run_until(async { + let mut transport = transport_to(address, Board::default()); + assert_eq!(transport.load("nobody", "a b.json").await, Ok(None)); + assert_eq!( + transport.store("nobody", "x", vec![1]).await, + Err(NetError::Status(404)) + ); + // Asked of a socket with no websocket open, which the driving task has nothing to do with. + assert_eq!( + transport.send(Frame::Text(String::default())).await, + Err(NetError::Closed) + ); + transport.close().await; + }) + .await; +} + +#[tokio::test(flavor = "current_thread")] +async fn a_hub_that_cannot_be_reached_is_said_to_be() { + let unused = TcpListener::bind((Ipv4Addr::LOCALHOST, 0)).await.unwrap(); + let address = unused.local_addr().unwrap(); + drop(unused); + LocalSet::new() + .run_until(async { + let mut transport = transport_to(address, Board::default()); + let refused = transport.connect().await.unwrap_err(); + assert!(refused.to_string().starts_with("could not reach the hub: ")); + assert!(matches!( + transport.load("bucket", "key").await, + Err(NetError::Connect(_)) + )); + }) + .await; +} + +/// A hub answering one connection as `script` says, after the upgrade `upgrade` answers it with. +async fn scripted(upgrade: &'static str, script: S) -> SocketAddr +where + S: FnOnce(TcpStream) -> F + 'static, + F: Future + 'static, +{ + let listener = TcpListener::bind((Ipv4Addr::LOCALHOST, 0)).await.unwrap(); + let address = listener.local_addr().unwrap(); + let _serving = tokio::task::spawn_local(async move { + let (mut socket, _peer) = listener.accept().await.unwrap(); + let mut head = Vec::new(); + while !head.ends_with(b"\r\n\r\n") { + head.push(socket.read_u8().await.unwrap()); + } + socket.write_all(upgrade.as_bytes()).await.unwrap(); + script(socket).await; + }); + address +} + +/// An unmasked frame of `opcode`, final unless `more` follows, carrying `payload`. +fn frame(opcode: u8, more: bool, payload: &[u8]) -> Vec { + let fin = if more { 0 } else { 0x80 }; + let mut bytes = vec![fin | opcode, u8::try_from(payload.len()).unwrap()]; + bytes.extend_from_slice(payload); + bytes +} + +/// The next frame the board sent, its opcode and its payload unmasked. +async fn read_frame(socket: &mut TcpStream) -> (u8, Vec) { + let opcode = socket.read_u8().await.unwrap() & 0x0f; + let length = usize::from(socket.read_u8().await.unwrap() & 0x7f); + let mut mask = [0; 4]; + let _read = socket.read_exact(&mut mask).await.unwrap(); + let mut payload = vec![0; length]; + let _read = socket.read_exact(&mut payload).await.unwrap(); + for (byte, key) in payload.iter_mut().zip(mask.iter().cycle()) { + *byte ^= key; + } + (opcode, payload) +} + +#[tokio::test(flavor = "current_thread")] +async fn frames_cross_both_ways_and_a_ping_is_answered() { + let (seen_tx, seen_rx) = tokio::sync::oneshot::channel(); + LocalSet::new() + .run_until(async { + let address = scripted(UPGRADED, |mut socket| async move { + socket.write_all(&frame(0x9, false, b"hi")).await.unwrap(); + socket.write_all(&frame(0x1, true, b"split ")).await.unwrap(); + socket.write_all(&frame(0x0, false, b"in two")).await.unwrap(); + // Binary, though the frame that completes it is a continuation. + socket.write_all(&frame(0x2, true, &[1])).await.unwrap(); + socket.write_all(&frame(0x0, false, &[2])).await.unwrap(); + socket.write_all(&frame(0xA, false, b"")).await.unwrap(); + let pong = read_frame(&mut socket).await; + let text = read_frame(&mut socket).await; + let binary = read_frame(&mut socket).await; + let close = read_frame(&mut socket).await; + seen_tx.send([pong, text, binary, close]).unwrap(); + }) + .await; + let mut transport = transport_to(address, Board::default()); + transport.connect().await.unwrap(); + assert_eq!( + transport.recv(5000).await, + Ok(Some(Frame::Text("split in two".to_owned()))) + ); + assert_eq!(transport.recv(5000).await, Ok(Some(Frame::Binary(vec![1, 2])))); + assert_eq!(transport.recv(10).await, Ok(None)); + transport.send(Frame::Text("up".to_owned())).await.unwrap(); + transport.send(Frame::Binary(vec![3])).await.unwrap(); + transport.close().await; + let seen = tokio::time::timeout(PATIENCE, seen_rx).await.unwrap().unwrap(); + assert_eq!( + seen, + [ + (0xA, b"hi".to_vec()), + (0x1, b"up".to_vec()), + (0x2, vec![3]), + (0x8, Vec::new()) + ] + ); + }) + .await; +} + +#[tokio::test(flavor = "current_thread")] +async fn a_hub_that_closes_or_refuses_the_upgrade_is_reported() { + LocalSet::new() + .run_until(async { + let closing = scripted(UPGRADED, |mut socket| async move { + socket.write_all(&frame(0x8, false, b"")).await.unwrap(); + }) + .await; + let mut transport = transport_to(closing, Board::default()); + transport.connect().await.unwrap(); + assert_eq!(transport.recv(5000).await, Err(NetError::Closed)); + + let refusing = scripted("HTTP/1.1 403 Forbidden\r\n\r\n", |_socket| async {}).await; + let mut transport = transport_to(refusing, Board::default()); + assert_eq!(transport.connect().await, Err(NetError::Handshake)); + }) + .await; +} diff --git a/services/ws/Cargo.toml b/services/ws/Cargo.toml index 3cc822df..003bac11 100644 --- a/services/ws/Cargo.toml +++ b/services/ws/Cargo.toml @@ -32,10 +32,10 @@ required-features = ["std"] [[test]] name = "picoserve_hub" -required-features = ["picoserve-hub"] +required-features = ["board"] [features] -default = ["nats", "otel", "picoserve-hub"] +default = ["board", "nats", "otel"] ## The actix-web hub: the `/ws` endpoint, the admin API, idle timeouts and the registry file. ## Without it the crate is `no_std` + `alloc` and holds only `hub`, the routing an embedder drives over its own ## transport. @@ -47,7 +47,6 @@ std = [ "dep:chrono", "dep:futures-util", "dep:secrecy", - "dep:serde", "dep:serde-inline-default", "dep:serde_default", "dep:subtle", @@ -55,11 +54,12 @@ std = [ "dep:tokio", "dep:uuid", "edge-toolkit/std", + "serde/std", "serde_json/std", "tracing/std", ] ## A `no_std` hub serving `/ws` over HTTP and WebSocket through picoserve, on any socket the embedder accepts. -picoserve-hub = ["dep:picoserve", "dep:spin"] +board = ["dep:picoserve", "dep:spin"] ## Shares agents between hubs through the NATS JetStream backplane `WS_BACKPLANE` names. ## Without it every hub is local, keeping its agents in its own registry. nats = ["dep:et-hub-backplane", "std"] @@ -86,7 +86,7 @@ opentelemetry = { workspace = true, optional = true } picoserve = { workspace = true, optional = true } schemars = { workspace = true, optional = true } secrecy = { workspace = true, optional = true } -serde = { workspace = true, features = ["std"], optional = true } +serde.workspace = true serde-inline-default = { workspace = true, optional = true } serde_default = { workspace = true, optional = true } serde_json = { workspace = true, features = ["alloc"] } diff --git a/services/ws/src/health.rs b/services/ws/src/health.rs new file mode 100644 index 00000000..b1611577 --- /dev/null +++ b/services/ws/src/health.rs @@ -0,0 +1,28 @@ +//! The answer to `GET /health`, which every server of the hub gives, on an operating system or on a board. + +#[cfg(not(feature = "std"))] +use alloc::string::{String, ToString as _}; + +use serde::{Deserialize, Serialize}; + +/// Server liveness probe response. +/// +/// Returned by `GET /health`. +#[derive(Clone, Debug, Deserialize, Serialize)] +#[cfg_attr(feature = "openapi-spec", derive(utoipa::ToSchema))] +#[non_exhaustive] +pub struct HealthResponse { + pub status: String, + pub service: String, +} + +impl HealthResponse { + /// What a server that is up answers, whichever server it is. + #[must_use] + pub fn healthy() -> Self { + Self { + status: "healthy".to_string(), + service: "ws-server".to_string(), + } + } +} diff --git a/services/ws/src/lib.rs b/services/ws/src/lib.rs index 8974e8a2..e43dea25 100644 --- a/services/ws/src/lib.rs +++ b/services/ws/src/lib.rs @@ -3,7 +3,7 @@ #![cfg_attr(feature = "docs", doc = document_features::document_features!())] #![cfg_attr(not(feature = "std"), no_std)] -#[cfg(any(not(feature = "std"), feature = "picoserve-hub"))] +#[cfg(any(not(feature = "std"), feature = "board"))] extern crate alloc; #[cfg(feature = "std")] @@ -12,8 +12,9 @@ pub mod admin; mod backplane; #[cfg(feature = "std")] mod connection; +pub mod health; pub mod hub; -#[cfg(feature = "picoserve-hub")] +#[cfg(feature = "board")] pub mod picoserve_hub; #[cfg(feature = "std")] mod server; diff --git a/utilities/cli/src/deployment_types/dekit.rs b/utilities/cli/src/deployment_types/dekit.rs index 873ac735..8241f77e 100644 --- a/utilities/cli/src/deployment_types/dekit.rs +++ b/utilities/cli/src/deployment_types/dekit.rs @@ -9,9 +9,9 @@ use super::hub_sharing::server_containers; use super::mise::{COLLECTOR_TASK, OPENER_TASK, container}; use super::yaml::{insert, strings}; use crate::error::CliError; -use crate::input::ClusterInput; -use crate::placement::{Site, hub_site, hubs, shares_hubs}; -use crate::{OutputType, hub_http_base, scenario_runners}; +use crate::input::{ClusterInput, Delivery}; +use crate::placement::{Site, hub_site, hubs, server_delivery}; +use crate::{OutputType, hub_http_base, task_runners}; /// How long the collector, or any other container the deployment runs, gets to exit once stopped. /// @@ -31,15 +31,17 @@ const CONTAINER_STOP_TIMEOUT: &str = "30s"; /// runner on its hub's. The task names need no path of their own, because the file is its own project root: dekit /// takes the nearest `dekit.yaml` as the root, and starts every task there, which is where mise finds the `mise.toml`. pub fn generate_dekit_deployment(cluster: &ClusterInput, output_dir: &Path) -> Result<(), CliError> { - let runners = scenario_runners(cluster)?; + let runners = task_runners(cluster)?; // A multi-machine cluster tags each task with its machine and starts none on `dekit up`, since every machine runs // the same file and `up` would otherwise start the whole cluster on each of them; `dekit start +` starts // one machine's share instead. let autostart = cluster.machines.is_empty(); let server = hub_site(cluster); + // A server that is a board runs the hub as firmware, with no collector: every task here is then a runner elsewhere. + let server_tasks = server_delivery(cluster) == Delivery::Tasks; let mut tasks = Mapping::new(); - let containers = server_containers(shares_hubs(cluster)); + let containers = server_containers(cluster); let server_hub_deps: Vec<&str> = containers.iter().map(|(name, ..)| *name).collect(); for (name, port, path) in containers { let mut service = task(name, &[], autostart, &server); @@ -57,7 +59,9 @@ pub fn generate_dekit_deployment(cluster: &ClusterInput, output_dir: &Path) -> R insert(&mut service, "stop", Value::Mapping(stop)); insert(&mut tasks, name, Value::Mapping(service)); } - insert_hub_tasks(&mut tasks, cluster, &server_hub_deps, autostart); + if server_tasks { + insert_hub_tasks(&mut tasks, cluster, &server_hub_deps, autostart); + } for runner in runners { // A runner on a machine without a hub cannot wait on a hub task this machine does not hold, so it restarts @@ -74,10 +78,12 @@ pub fn generate_dekit_deployment(cluster: &ClusterInput, output_dir: &Path) -> R // A job, since it opens the browser and exits, and not started on `dekit up`: it is there to be run once the // collector is up, which its dependency then waits for. - let mut opener = Mapping::new(); - insert(&mut opener, "type", Value::from("job")); - opener.extend(task(OPENER_TASK, &[COLLECTOR_TASK], false, &server)); - insert(&mut tasks, OPENER_TASK, Value::Mapping(opener)); + if server_tasks { + let mut opener = Mapping::new(); + insert(&mut opener, "type", Value::from("job")); + opener.extend(task(OPENER_TASK, &[COLLECTOR_TASK], false, &server)); + insert(&mut tasks, OPENER_TASK, Value::Mapping(opener)); + } let config = DekitConfig { // SIGINT rather than dekit's SIGTERM, because it is the interrupt a mise task is written to stop on -- the same diff --git a/utilities/cli/src/deployment_types/firmware.rs b/utilities/cli/src/deployment_types/firmware.rs index d080cff0..2d79c90c 100644 --- a/utilities/cli/src/deployment_types/firmware.rs +++ b/utilities/cli/src/deployment_types/firmware.rs @@ -1,13 +1,17 @@ use std::path::Path; +use edge_toolkit::ports::Services; use et_path::{absolute_from, relative_path_from}; use fs_err as fs; use toml::{Table, Value}; use super::mise::{mise_task, registry_and_modules, wrap_module_paths}; use crate::error::CliError; -use crate::input::{ArtifactSource, Board, ClusterInput}; -use crate::{ModuleSource, ScenarioModules, cluster_module_names, resolve_cluster_modules, scenario_runners}; +use crate::input::{Agent, ArtifactSource, Board, ClusterInput, Delivery, OutputType}; +use crate::placement::agent_site; +use crate::{ + ModuleSource, RunnerInstance, ScenarioModules, cluster_module_names, resolve_cluster_modules, scenario_runners, +}; /// The task that builds the firmware with the scenario's modules in it. const BUILD_TASK: &str = "firmware-build"; @@ -17,37 +21,111 @@ pub(crate) const FLASH_TASK: &str = "firmware-flash"; pub(crate) const HOST_TEST_TASK: &str = "firmware-host-test"; /// Where npm packages too big for the board are fetched from instead: a CDN mirroring the npm registry by package path. pub(crate) const ELSEWHERE_URL: &str = "https://cdn.jsdelivr.net/npm"; +/// Why a published scenario is refused a board. +const PUBLISHED: &str = "uses published artifacts, since the firmware is built from this repository"; -/// Refuse a scenario `board` cannot run. +/// Refuse a scenario `board` cannot run as a `firmware-*` format, the whole scenario on the one board. /// -/// A board holds one hub, the modules built into it, and the runners its firmware hosts, and nothing else. A second -/// machine needs a backplane, and a published deployment modules staged from a release, neither of which the firmware -/// has. A second hub needs a second machine, so is refused with it. A runner is refused unless the board's firmware -/// runs its kind; whether the guest itself fits, the firmware's build decides. Checked before anything is written, as -/// every format's refusals are. +/// A second machine is refused: a scenario spanning several, the board among them, names it as a machine's `device:` +/// instead, which [`validate_devices`] checks. A published deployment is refused because the firmware is built from +/// this repository, and the agents are refused what [`validate_board_agents`] refuses them. Checked before anything is +/// written, as every format's refusals are. pub(crate) fn validate_firmware(cluster: &ClusterInput, board: Board) -> Result<(), CliError> { - let spec = board.spec(); - let refused = |reason| { - Err(CliError::BoardCannotRun { - board: spec.title, - reason, - }) - }; if !cluster.machines.is_empty() { - return refused("spans several machines"); + return refused( + board, + "spans several machines; name the board as a machine's device instead", + ); } if matches!(cluster.artifact_source, ArtifactSource::Published) { - return refused("uses published artifacts, since the firmware is built from this repository"); + return refused(board, PUBLISHED); + } + validate_board_agents(board, cluster.agents.iter()) +} + +/// Refuse a scenario whose machines name a device that `output_type` cannot deliver, or that the device cannot run. +/// +/// A board that is the server runs the hub and the runners its firmware hosts beside it; a board on any other machine +/// runs only the runners placed on it, reaching the server's hub across the network, as every machine running its share +/// as the format's tasks does. Only the formats built from `mise.toml` tasks can deliver that, since compose and k3s +/// ship container images a board cannot run. +pub(crate) fn validate_devices(cluster: &ClusterInput, output_type: OutputType) -> Result<(), CliError> { + let Some(named) = cluster.machines.iter().find(|machine| !machine.device.is_unspecified()) else { + return Ok(()); + }; + if !output_type.runs_mise_tasks() { + return Err(CliError::DeviceFormatUnsupported { + machine: named.name.clone(), + output_type: output_type.name().to_owned(), + }); + } + for (index, machine) in cluster.machines.iter().enumerate() { + let Delivery::Firmware(board) = machine.device.delivery() else { + continue; + }; + if matches!(cluster.artifact_source, ArtifactSource::Published) { + return refused(board, PUBLISHED); + } + if !cluster.services.hub.machines.is_empty() { + return refused( + board, + "runs a hub on another machine too, since the board has no backplane", + ); + } + let on_board = || { + cluster + .agents + .iter() + .filter(|agent| agent_site(cluster, &agent.name).machine_names().contains(&machine.name)) + }; + // A board that is not the hub's machine runs nothing but its runners, so one with none would run nothing. + if index > 0 && !on_board().any(|agent| agent.runner.is_some()) { + return Err(CliError::BoardWithoutRunners { + machine: machine.name.clone(), + board: board.spec().title, + }); + } + validate_board_agents(board, on_board())?; } - for agent in &cluster.agents { + Ok(()) +} + +/// Refuse what the board cannot run, as `board`'s refusal for `reason`. +const fn refused(board: Board, reason: &'static str) -> Result<(), CliError> { + Err(CliError::BoardCannotRun { + board: board.spec().title, + reason, + }) +} + +/// Refuse a runner among `agents` that `board` cannot run beside its hub. +/// +/// A runner is refused unless the board's firmware runs its kind; whether the guest itself fits, the firmware's build +/// decides. Limits and schedules are refused too, since the firmware has no way to hold a guest to either: it starts +/// what it runs once, at boot. +fn validate_board_agents<'cluster, A>(board: Board, agents: A) -> Result<(), CliError> +where + A: Iterator, +{ + let spec = board.spec(); + for agent in agents { let Some(runner) = agent.runner.as_deref().map(str::trim).filter(|kind| !kind.is_empty()) else { continue; }; if spec.runners.is_empty() { - return refused("starts a runner, since the board has no memory to run one in"); + return refused(board, "starts a runner, since the board has no memory to run one in"); } if !agent.limits.is_unset() { - return refused("sets limits on a runner, since a board has no way to hold one to them"); + return refused( + board, + "sets limits on a runner, since a board has no way to hold one to them", + ); + } + if !agent.schedule.is_unset() { + return refused( + board, + "schedules a runner, since a board starts what it runs once, at boot", + ); } if !spec.runners.contains(&runner) { return Err(CliError::BoardRunnerUnsupported { @@ -63,22 +141,125 @@ pub(crate) fn validate_firmware(cluster: &ClusterInput, board: Board) -> Result< /// Write the `mise.firmware-.toml` that builds the scenario's hub as `board` firmware and loads it onto one. /// -/// The board serves the modules the scenario names, and what they depend on, from flash: each module from this -/// repository or the scenario's own `module_paths:` is built into the firmware, and each npm package a mise tool -/// stages is redirected to a CDN instead, since onnxruntime-web alone is bigger than a board's flash. A board -/// that runs runners also starts one per runner the scenario declares, on its module's component. Selected with -/// `MISE_ENV=firmware-`, so it sits beside a `mise.toml` without either loading the other's tasks unasked. +/// Selected with `MISE_ENV=firmware-`, so it sits beside a `mise.toml` without either loading the other's tasks +/// unasked. What the tasks build is [`firmware_tasks`]'s to say. pub(crate) fn generate_firmware_deployment( cluster: &ClusterInput, output_dir: &Path, board: Board, ) -> Result<(), CliError> { validate_firmware(cluster, board)?; + let guests = scenario_runners(cluster)?; + let tasks = firmware_tasks(cluster, output_dir, board, &guests, &Build::Whole)?; + let mut root = Table::new(); + let _previous: Option = root.insert("tasks".to_owned(), Value::Table(tasks)); + fs::write( + output_dir.join(board.spec().output_type.output_file_name()), + toml::to_string(&Value::Table(root))?, + )?; + Ok(()) +} + +/// The tasks of every board among the machines, each suffixed with its machine's name; none when there is no board. +/// +/// The server's board runs the hub, and any other board the runners placed on it, reaching the server's hub. +pub(crate) fn board_firmware_tasks(cluster: &ClusterInput, output_dir: &Path) -> Result { + let mut tasks = Table::new(); + let Some(server) = cluster.machines.first() else { + return Ok(tasks); + }; + let runners = scenario_runners(cluster)?; + for machine in &cluster.machines { + let Delivery::Firmware(board) = machine.device.delivery() else { + continue; + }; + let guests = runners + .iter() + .filter(|runner| runner.site.machine_names().contains(&machine.name)) + .cloned() + .collect::>(); + let build = if machine.name == server.name { + Build::Hub { machine: &machine.name } + } else { + Build::Runners { + machine: &machine.name, + hub_address: format!("{}:{}", server.address, Services::InsecureWebSocketServer.port()), + } + }; + tasks.extend(firmware_tasks(cluster, output_dir, board, &guests, &build)?); + } + Ok(tasks) +} + +/// What one board's firmware is built as. +enum Build<'machine> { + /// The whole scenario on the one board, as a `firmware-*` format puts it. + Whole, + /// The hub, and the runners beside it, on `machine`, the server of a scenario spanning several. + Hub { machine: &'machine str }, + /// The runners on `machine`, reaching the hub at `hub_address`. + Runners { + machine: &'machine str, + hub_address: String, + }, +} + +impl Build<'_> { + /// What ends the name of each of the build's tasks: nothing for a whole scenario, else `-`. + fn suffix(&self) -> String { + match self { + Self::Whole => String::default(), + Self::Hub { machine } | Self::Runners { machine, .. } => format!("-{machine}"), + } + } + + /// Where the build's image lands, `image` as the board's spec gives it. + /// + /// A machine's board builds under a target directory of its own, so that two boards' builds do not overwrite each + /// other's image; a whole scenario's lands where the spec says. + fn image(&self, image: &str) -> String { + match self { + Self::Whole => image.to_owned(), + Self::Hub { machine } | Self::Runners { machine, .. } => { + image.replacen("target/", &format!("{}/", target_dir(machine)), 1) + } + } + } +} + +/// The target directory a machine's board is built in. +fn target_dir(machine: &str) -> String { + format!("target/firmware/{machine}") +} + +/// The task that loads `machine`'s board, which is what that machine's share of a scenario amounts to. +pub(crate) fn flash_task(machine: &str) -> String { + format!("{FLASH_TASK}-{machine}") +} + +/// The tasks that build `board`'s firmware as `build` says, load it onto the board, and serve it from this host. +/// +/// Only firmware that runs the hub is served from this host: one of runners alone serves nothing to fetch. +/// +/// The hub serves the modules the scenario names, and what they depend on, from flash: each module from this +/// repository or the scenario's own `module_paths:` is built into the firmware, and each npm package a mise tool +/// stages is redirected to a CDN instead, since onnxruntime-web alone is bigger than a board's flash. Every module is +/// built in, whichever machine runs it, since a runner elsewhere fetches its module from this hub. A board of runners +/// alone builds in only its runners' modules, and the address of the hub they reach. Each of `guests` is a runner the +/// firmware starts, on its module's component. +fn firmware_tasks( + cluster: &ClusterInput, + output_dir: &Path, + board: Board, + guests: &[RunnerInstance], + build: &Build<'_>, +) -> Result { let board = board.spec(); + let suffix = build.suffix(); let workspace_root = edge_toolkit::config::get_project_root(); let output_abs = absolute_from(&workspace_root, output_dir); let workspace_rel = relative_path_from(&output_abs, &workspace_root); - let (modules_paths, env) = module_env(cluster, &workspace_root)?; + let (modules_paths, env) = module_env(cluster, guests, build, &workspace_root)?; // Both run at the repository root, where `$PWD` names the module paths absolutely, as the build needs. let module_task = |description: &str, command: &str| { let run = format!("{modules_paths}export MODULES_PATHS\n{command}\n"); @@ -91,24 +272,23 @@ pub(crate) fn generate_firmware_deployment( Some(env.clone()), ) }; - let build = module_task( - &format!("Build the {} firmware serving {}", board.title, cluster.cluster_name), + let role = if matches!(build, Build::Runners { .. }) { + "running the runners of" + } else { + "serving" + }; + let build_task = module_task( + &format!("Build the {} firmware {role} {}", board.title, cluster.cluster_name), &format!("mise -E rust run {}", board.build_task), ); - let host_test = module_task( - &format!( - "Serve {}'s modules from this host as the board would, and fetch each", - cluster.cluster_name - ), - &format!("mise run cargo-test-pkg {}", board.package), - ); + let build_name = format!("{BUILD_TASK}{suffix}"); let mut depends = Table::new(); let _previous: Option = depends.insert( "depends".to_owned(), - Value::Array(vec![Value::String(BUILD_TASK.to_owned())]), + Value::Array(vec![Value::String(build_name.clone())]), ); - let flash_run = format!("{} {}", board.load_command, board.image); + let flash_run = format!("{} {}", board.load_command, build.image(board.image)); let flash_description = format!("Load the {} firmware onto the board, and start it", board.title); let flash = mise_task( None, @@ -120,34 +300,49 @@ pub(crate) fn generate_firmware_deployment( ); let mut tasks = Table::new(); - let _previous: Option = tasks.insert(BUILD_TASK.to_owned(), Value::Table(build)); - let _previous: Option = tasks.insert(FLASH_TASK.to_owned(), Value::Table(flash)); - let _previous: Option = tasks.insert(HOST_TEST_TASK.to_owned(), Value::Table(host_test)); - let mut root = Table::new(); - let _previous: Option = root.insert("tasks".to_owned(), Value::Table(tasks)); - fs::write( - output_dir.join(board.output_type.output_file_name()), - toml::to_string(&Value::Table(root))?, - )?; - Ok(()) + let _previous: Option = tasks.insert(build_name, Value::Table(build_task)); + let _previous: Option = tasks.insert(format!("{FLASH_TASK}{suffix}"), Value::Table(flash)); + // A board of runners alone serves nothing, so there is nothing of it to fetch from this host. + if !matches!(build, Build::Runners { .. }) { + let host_test = module_task( + &format!( + "Serve {}'s modules from this host as the board would, and fetch each", + cluster.cluster_name + ), + &format!("mise run cargo-test-pkg {}", board.package), + ); + let _previous: Option = tasks.insert(format!("{HOST_TEST_TASK}{suffix}"), Value::Table(host_test)); + } + Ok(tasks) } /// The `MODULES_PATHS` assignments naming the modules built in, and the rest of the build's environment. /// -/// That is the other `MODULES_*` variables, and `RUNNER_GUESTS`: a `name=package` pair for each runner the board -/// starts, naming the module built in whose component the firmware compiles for it. Set even when empty, so a build -/// never picks runners up from the shell it was started in. +/// That is the other `MODULES_*` variables, and `RUNNER_GUESTS`: a `name=package` pair for each of `guests`, naming +/// the module built in whose component the firmware compiles for it. Set even when empty, so a build never picks +/// runners up from the shell it was started in. A machine's board also builds into a target directory of its own, and a +/// board of runners alone is told the `HUB_ADDRESS` they reach. #[expect( clippy::single_call_fn, - reason = "the module-resolving step of generate_firmware_deployment, kept apart from writing the file" + reason = "the module-resolving step of firmware_tasks, kept apart from assembling the tasks" )] -fn module_env(cluster: &ClusterInput, workspace_root: &Path) -> Result<(String, Table), CliError> { +fn module_env( + cluster: &ClusterInput, + guests: &[RunnerInstance], + build: &Build<'_>, + workspace_root: &Path, +) -> Result<(String, Table), CliError> { let ws_server_dir = workspace_root.join("services/ws-server"); - let module_names = cluster_module_names(cluster); - // The page is served whether or not the board runs agents of its own, since a browser can still join it. - let scenario = ScenarioModules::new(&ws_server_dir, &module_names, true).with_module_paths(&cluster.module_paths); - let guests = scenario_runners(cluster)? - .into_iter() + // A hub serves the page whether or not the board runs agents of its own, since a browser can still join it. A + // board of runners alone serves nothing, so builds in only what its runners' guests are compiled from. + let (module_names, serves_a_page) = match build { + Build::Whole | Build::Hub { .. } => (cluster_module_names(cluster), true), + Build::Runners { machine, .. } => (machine_modules(cluster, machine), false), + }; + let scenario = + ScenarioModules::new(&ws_server_dir, &module_names, serves_a_page).with_module_paths(&cluster.module_paths); + let guests = guests + .iter() .map(|runner| format!("{}={}", runner.name, runner.module)) .collect::>(); @@ -168,9 +363,43 @@ fn module_env(cluster: &ClusterInput, workspace_root: &Path) -> Result<(String, let mut set = |name: &str, value: String| { let _previous: Option = env.insert(name.to_owned(), Value::String(value)); }; - set("MODULES_ROOT", super::hub_root_module(&ws_server_dir)); + let root = if serves_a_page { + super::hub_root_module(&ws_server_dir) + } else { + String::default() + }; + set("MODULES_ROOT", root); set("MODULES_ELSEWHERE", elsewhere.join(", ")); set("MODULES_ELSEWHERE_URL", ELSEWHERE_URL.to_owned()); set("RUNNER_GUESTS", guests.join(", ")); + match build { + Build::Whole => {} + Build::Hub { machine } => set("CARGO_TARGET_DIR", target_dir(machine)), + Build::Runners { machine, hub_address } => { + set("CARGO_TARGET_DIR", target_dir(machine)); + set("HUB_ADDRESS", hub_address.clone()); + } + } Ok((wrap_module_paths(&embedded), env)) } + +/// The modules of the runners placed on `machine`, by the names the scenario gives them. +fn machine_modules(cluster: &ClusterInput, machine: &str) -> Vec { + cluster + .agents + .iter() + .filter(|agent| agent.runner.is_some()) + .filter(|agent| { + agent_site(cluster, &agent.name) + .machine_names() + .iter() + .any(|placed| placed == machine) + }) + .flat_map(|agent| { + agent + .resources + .iter() + .map(|resource| resource.resource_type.trim().to_owned()) + }) + .collect() +} diff --git a/utilities/cli/src/deployment_types/hub_sharing.rs b/utilities/cli/src/deployment_types/hub_sharing.rs index a689c6e0..aa7dd917 100644 --- a/utilities/cli/src/deployment_types/hub_sharing.rs +++ b/utilities/cli/src/deployment_types/hub_sharing.rs @@ -7,6 +7,8 @@ use edge_toolkit::ports::Services; use super::mise::COLLECTOR_TASK; +use crate::input::{ClusterInput, Delivery}; +use crate::placement::{server_delivery, shares_hubs}; /// The task, service and `Deployment` that runs NATS. pub(crate) const BACKPLANE_TASK: &str = "nats"; @@ -49,10 +51,14 @@ pub(crate) const BACKPLANE_CONFIG_FILE: &str = "nats-server.conf"; /// Each container the server runs as a task, with the port and path its readiness check asks. /// -/// The collector always, and what the hubs share when `shared`: the tasks the server's hub waits on. -pub(crate) fn server_containers(shared: bool) -> Vec<(&'static str, u16, &'static str)> { +/// The collector, and what the hubs share when there are several: the tasks the server's hub waits on. None when the +/// server is a board, which runs the hub as firmware and no collector. +pub(crate) fn server_containers(cluster: &ClusterInput) -> Vec<(&'static str, u16, &'static str)> { + if server_delivery(cluster) != Delivery::Tasks { + return Vec::new(); + } let mut containers = vec![(COLLECTOR_TASK, Services::OtlpCollector.port(), "/healthz")]; - if shared { + if shares_hubs(cluster) { containers.extend([ (BACKPLANE_TASK, Services::BackplaneMonitor.port(), BACKPLANE_HEALTH_PATH), ( diff --git a/utilities/cli/src/deployment_types/mise.rs b/utilities/cli/src/deployment_types/mise.rs index 525d631d..bb82b7df 100644 --- a/utilities/cli/src/deployment_types/mise.rs +++ b/utilities/cli/src/deployment_types/mise.rs @@ -7,18 +7,18 @@ use et_path::{absolute_from, relative_path_from}; use fs_err as fs; use toml::{Table, Value}; +use super::firmware::{board_firmware_tasks, flash_task}; use super::hub_sharing::{ BACKPLANE_IMAGE, BACKPLANE_TASK, DATA_DIR, SHARED_STORAGE_IMAGE, SHARED_STORAGE_TASK, backplane_args, shared_hub_env, shared_storage_env, shared_storage_start, }; use super::process_compose::PROCESS_COMPOSE_TOOL; use crate::error::CliError; -use crate::input::{ArtifactSource, ClusterInput}; -use crate::placement::{hubs, shares_hubs}; +use crate::input::{ArtifactSource, ClusterInput, Delivery}; +use crate::placement::{hubs, server_delivery, shares_hubs}; use crate::{ COLLECTOR_SETTINGS, COLLECTOR_USERNAME, MODULE_SCOPE, ModuleRegistryEntry, ModuleSource, RunnerInstance, - cluster_module_names, module_registry, resolve_cluster_modules, resolve_module_paths, runner_crate, - scenario_runners, + cluster_module_names, module_registry, resolve_cluster_modules, resolve_module_paths, runner_crate, task_runners, }; /// Crate whose binary serves the hub, which a published deployment installs in place of building it. @@ -80,7 +80,12 @@ pub(crate) fn container(task: &str, cluster_name: &str) -> String { } /// The tasks the server runs besides its runners: the collector, its hub, and what the hubs share when there are more. +/// +/// A board is loaded with the hub instead, and runs no collector. fn server_tasks(cluster: &ClusterInput) -> Vec { + if let (Delivery::Firmware(_), Some(server)) = (server_delivery(cluster), cluster.machines.first()) { + return vec![flash_task(&server.name)]; + } let mut tasks = vec![COLLECTOR_TASK.to_string(), HUB_TASK.to_string()]; if shares_hubs(cluster) { tasks.extend([BACKPLANE_TASK.to_string(), SHARED_STORAGE_TASK.to_string()]); @@ -109,6 +114,15 @@ fn runner_task(runner: &RunnerInstance, workspace_rel: &str, artifacts: Artifact fn insert_entry_tasks(tasks: &mut Table, cluster: &ClusterInput, runners: &[RunnerInstance]) { let mut scenario_depends = server_tasks(cluster); scenario_depends.extend(runners.iter().map(|runner| runner.name.clone())); + // Every other board too, since its runners are its firmware's. + scenario_depends.extend( + cluster + .machines + .iter() + .skip(1) + .filter(|machine| machine.device.delivery() != Delivery::Tasks) + .map(|machine| flash_task(&machine.name)), + ); let description = format!("Run generated scenario for {}", cluster.cluster_name); let scenario = mise_task( None, @@ -120,6 +134,9 @@ fn insert_entry_tasks(tasks: &mut Table, cluster: &ClusterInput, runners: &[Runn ); let _previous: Option = tasks.insert("generated-scenario".to_string(), Value::Table(scenario)); insert_machine_tasks(tasks, cluster, runners); + if server_delivery(cluster) != Delivery::Tasks { + return; + } let opener = mise_task( None, Some("Open the OpenObserve UI"), @@ -142,6 +159,9 @@ fn insert_machine_tasks(tasks: &mut Table, cluster: &ClusterInput, runners: &[Ru for (index, machine) in cluster.machines.iter().enumerate() { let mut depends = if index == 0 { server_tasks(cluster) + } else if machine.device.delivery() != Delivery::Tasks { + // A board's runners are its firmware's, so loading it is its whole share. + vec![flash_task(&machine.name)] } else { hubs.iter() .filter(|hub| hub.site.machine_names().contains(&machine.name)) @@ -185,12 +205,17 @@ pub fn generate_mise_deployment( }; let mut root = Table::new(); - let mut tasks = service_tasks(cluster, &scenario, &output_abs, &ws_server_dir)?; + let mut tasks = if server_delivery(cluster) == Delivery::Tasks { + service_tasks(cluster, &scenario, &output_abs, &ws_server_dir)? + } else { + Table::new() + }; + tasks.extend(board_firmware_tasks(cluster, output_dir)?); // Each runner agent becomes its own task, and `generated-scenario` depends on all of them. mise runs `depends` // concurrently, which is what a cluster wants -- the hub and every runner are long-running peers, not a pipeline -- // but it also means a runner starts before the hub is listening. Each runner body therefore waits for the hub's own - // health endpoint first; see `runner_run_body`. - let runners = scenario_runners(cluster)?; + // health endpoint first; see `runner_run_body`. A runner on a board is the board's firmware's to start instead. + let runners = task_runners(cluster)?; for runner in &runners { let task = runner_task(runner, &workspace_rel, artifacts); let _previous: Option = tasks.insert(runner.name.clone(), Value::Table(task)); diff --git a/utilities/cli/src/deployment_types/mod.rs b/utilities/cli/src/deployment_types/mod.rs index 174bfb2d..49b68a86 100644 --- a/utilities/cli/src/deployment_types/mod.rs +++ b/utilities/cli/src/deployment_types/mod.rs @@ -15,7 +15,8 @@ pub(crate) use self::docker_compose::{HUB_IMAGE_BUILD, SCHEDULER_SERVICE, schedu pub use self::docker_compose::{docker_image_module_paths, generate_docker_compose_deployment}; pub(crate) use self::firmware::{ ELSEWHERE_URL as FIRMWARE_ELSEWHERE_URL, FLASH_TASK as FIRMWARE_FLASH_TASK, - HOST_TEST_TASK as FIRMWARE_HOST_TEST_TASK, generate_firmware_deployment, validate_firmware, + HOST_TEST_TASK as FIRMWARE_HOST_TEST_TASK, flash_task as firmware_flash_task, generate_firmware_deployment, + validate_devices, validate_firmware, }; pub(crate) use self::hub::{hub_root_module, serves_a_page}; pub(crate) use self::hub_sharing::{BACKPLANE_TASK, SHARED_STORAGE_TASK, STORAGE_SECRET_ENVS}; diff --git a/utilities/cli/src/deployment_types/process_compose.rs b/utilities/cli/src/deployment_types/process_compose.rs index a47ab36e..cce3d7a4 100644 --- a/utilities/cli/src/deployment_types/process_compose.rs +++ b/utilities/cli/src/deployment_types/process_compose.rs @@ -9,10 +9,10 @@ use super::mise::container; use super::yaml::{document, insert}; use super::{COLLECTOR_TASK, OPENER_TASK}; use crate::error::CliError; -use crate::input::ClusterInput; -use crate::placement::{Site, hub_site, hubs, shares_hubs}; +use crate::input::{ClusterInput, Delivery}; +use crate::placement::{Site, hub_site, hubs, server_delivery}; use crate::schedule::Schedule; -use crate::{OutputType, scenario_runners}; +use crate::{OutputType, task_runners}; /// The mise tool that installs process-compose, which the `mise.toml` written alongside declares. pub(crate) const PROCESS_COMPOSE_TOOL: &str = "aqua:F1bonacc1/process-compose"; @@ -46,11 +46,14 @@ const READINESS_FAILURE_THRESHOLD: u16 = 1800; /// A multi-machine cluster puts each process in the namespace of its machine, so `-n ` starts one machine's /// share. pub fn generate_process_compose_deployment(cluster: &ClusterInput, output_dir: &Path) -> Result<(), CliError> { - let runners = scenario_runners(cluster)?; + let runners = task_runners(cluster)?; let server = hub_site(cluster); + // A server that is a board runs the hub as firmware, with no collector: every process here is then a runner + // elsewhere. + let server_processes = server_delivery(cluster) == Delivery::Tasks; let mut processes = Mapping::new(); - let containers = server_containers(shares_hubs(cluster)); + let containers = server_containers(cluster); let server_hub_deps: Vec<&str> = containers.iter().map(|(name, ..)| *name).collect(); for (name, port, path) in containers { let mut service = process(name, &[], &server); @@ -64,7 +67,9 @@ pub fn generate_process_compose_deployment(cluster: &ClusterInput, output_dir: & } insert(&mut processes, name, Value::Mapping(service)); } - insert_hub_processes(&mut processes, cluster, &server_hub_deps); + if server_processes { + insert_hub_processes(&mut processes, cluster, &server_hub_deps); + } for runner in runners { // A runner on a machine without a hub cannot wait on a hub process this one does not hold, so it restarts until @@ -85,9 +90,11 @@ pub fn generate_process_compose_deployment(cluster: &ClusterInput, output_dir: & // Disabled, since it opens the browser and exits: it is there to be started once the collector is up, which its // dependency then waits for. - let mut opener = process(OPENER_TASK, &[COLLECTOR_TASK], &server); - insert(&mut opener, "disabled", Value::Bool(true)); - insert(&mut processes, OPENER_TASK, Value::Mapping(opener)); + if server_processes { + let mut opener = process(OPENER_TASK, &[COLLECTOR_TASK], &server); + insert(&mut opener, "disabled", Value::Bool(true)); + insert(&mut processes, OPENER_TASK, Value::Mapping(opener)); + } let mut config = Mapping::new(); insert(&mut config, "version", Value::from(SCHEMA_VERSION)); diff --git a/utilities/cli/src/error.rs b/utilities/cli/src/error.rs index 0fa1076a..c61c7bf7 100644 --- a/utilities/cli/src/error.rs +++ b/utilities/cli/src/error.rs @@ -183,6 +183,14 @@ pub enum CliError { supported: String, }, + #[error("Machine {machine:?} names a device, which {output_type} cannot deliver; only mise-task formats can")] + DeviceFormatUnsupported { machine: String, output_type: String }, + + #[error( + "Machine {machine:?} is a {board} with no runner placed on it, and off the hub's machine it runs only those" + )] + BoardWithoutRunners { machine: String, board: &'static str }, + #[error("Module path {0:?} has a name a shell would interpret or a filesystem would not hold; rename it")] UnsafeModulePath(PathBuf), diff --git a/utilities/cli/src/input.rs b/utilities/cli/src/input.rs index 13e83735..383e32e6 100644 --- a/utilities/cli/src/input.rs +++ b/utilities/cli/src/input.rs @@ -145,6 +145,55 @@ impl Board { } } +/// The hardware one machine of a scenario is, which decides what it can run and how its share reaches it. +/// +/// A board's firmware holds what its build put in it, so a machine naming one takes only the parts that firmware has: +/// the hub, and the runner kinds that board's firmware hosts. `Unspecified` names no hardware, and is any machine +/// with mise and this repository's toolchain on it, running its share as the format's own tasks. +#[derive(Clone, Copy, Debug, Default, Deserialize, Eq, PartialEq, Serialize)] +#[non_exhaustive] +#[serde(rename_all = "kebab-case")] +pub enum Device { + /// No hardware named: a machine that runs its share as the format's tasks, as every machine did before devices. + #[default] + Unspecified, + /// The Raspberry Pi Pico 2 W, running the hub as firmware. + Pico2, + /// An ESP32-C6 board, running the hub as firmware. + Esp32c6, + /// The Pimoroni Pico Plus 2 W, running the hub and the wasi runners beside it as firmware. + #[serde(alias = "pico_plus2")] + PicoPlus2, +} + +/// How a machine's share of the scenario is delivered to it. +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub(crate) enum Delivery { + /// As the tasks of the deployment format, which the machine runs itself. + Tasks, + /// As firmware built for the board, loaded onto it. + Firmware(Board), +} + +impl Device { + /// Whether no hardware is named, which is what a scenario leaving `device:` out means. + #[must_use] + pub const fn is_unspecified(&self) -> bool { + matches!(self, Self::Unspecified) + } + + /// How a machine of this device gets its share. + #[must_use] + pub(crate) const fn delivery(self) -> Delivery { + match self { + Self::Unspecified => Delivery::Tasks, + Self::Pico2 => Delivery::Firmware(Board::Pico2), + Self::Esp32c6 => Delivery::Firmware(Board::Esp32c6), + Self::PicoPlus2 => Delivery::Firmware(Board::PicoPlus2), + } + } +} + impl OutputType { /// Every format, in the order a generated README lists their files and run sections. /// @@ -407,6 +456,9 @@ pub struct Machine { pub name: String, /// The host name or IPv4 address the other machines reach this one at. pub address: String, + /// The hardware the machine is, from `device:`; unstated, a machine running its share as the format's tasks. + #[serde(default, skip_serializing_if = "Device::is_unspecified")] + pub device: Device, /// Names of the agents that run on this machine. #[serde(default, skip_serializing_if = "Vec::is_empty")] pub agents: Vec, diff --git a/utilities/cli/src/lib.rs b/utilities/cli/src/lib.rs index 3cc24585..fb9ddc5c 100644 --- a/utilities/cli/src/lib.rs +++ b/utilities/cli/src/lib.rs @@ -42,10 +42,10 @@ pub(crate) use self::deployment_types::{ pub use self::error::CliError; pub use self::hub_ws_url::{hub_service_ws_url, hub_service_ws_url_named, hub_ws_url, hub_ws_url_at}; pub use self::input::{ - ArtifactSource, ClusterInput, DEFAULT_CLUSTER_NAME, OutputType, Service, Services, infer_artifact_source, + ArtifactSource, ClusterInput, DEFAULT_CLUSTER_NAME, Device, OutputType, Service, Services, infer_artifact_source, manifest_declares_this_repository, }; -pub(crate) use self::input::{Board, BoardSpec, running_in_this_repository}; +pub(crate) use self::input::{Board, BoardSpec, Delivery, running_in_this_repository}; pub use self::limits::{Cpus, Limits, LimitsError, Ulimit}; pub use self::module_package_json::generate_module_package_json; pub use self::recovery::{Recovery, RecoverySettings}; @@ -531,6 +531,7 @@ pub fn regenerate_verification( || deployment_types::validate_firmware(&cluster, *board).is_ok() }) }) + .filter(|output_type| deployment_types::validate_devices(&cluster, *output_type).is_ok()) .collect(); if output_types.is_empty() { continue; @@ -572,6 +573,9 @@ fn validate_deployment_outputs(cluster: &ClusterInput, output_types: &[OutputTyp deployment_types::validate_firmware(cluster, *board)?; } } + for output_type in output_types { + deployment_types::validate_devices(cluster, *output_type)?; + } Ok(()) } @@ -902,7 +906,7 @@ fn generated_readme( let _write_result = write!(flags, " --build-context \"{name}={path}\""); flags }); - let run_instructions = output_types + let mut run_instructions = output_types .iter() .map(|output_type| { let mut section = generated_run_instructions( @@ -920,6 +924,7 @@ fn generated_readme( }) .collect::>() .join("\n"); + run_instructions.push_str(&board_instructions(cluster)); // Every way of running the tasks needs what they name installed first, so the step comes once, ahead of them all. let install_note = if output_types.iter().any(|output_type| output_type.runs_mise_tasks()) { mise_install_note(cluster.artifact_source) @@ -1096,6 +1101,9 @@ fn generated_run_instructions( artifacts: ArtifactSource, build_contexts: &str, ) -> String { + if let (Delivery::Firmware(board), Some(server)) = (placement::server_delivery(cluster), cluster.machines.first()) { + return board_hub_run_instructions(output_type, &server.name, board.spec()); + } match output_type { OutputType::Mise => concat!( "## Run With Mise\n\n", @@ -1133,20 +1141,115 @@ fn generated_run_instructions( OutputType::ProcessCompose => process_compose_run_instructions(artifacts).to_string(), OutputType::DockerCompose => docker_compose_run_instructions(cluster, artifacts), OutputType::K3s => k3s_run_instructions(cluster, image_build, runners, artifacts, build_contexts), - OutputType::FirmwarePico2 => firmware_run_instructions(Board::Pico2.spec()), - OutputType::FirmwareEsp32c6 => firmware_run_instructions(Board::Esp32c6.spec()), - OutputType::FirmwarePicoPlus2 => firmware_run_instructions(Board::PicoPlus2.spec()), + OutputType::FirmwarePico2 => firmware_output_instructions(Board::Pico2.spec()), + OutputType::FirmwareEsp32c6 => firmware_output_instructions(Board::Esp32c6.spec()), + OutputType::FirmwarePicoPlus2 => firmware_output_instructions(Board::PicoPlus2.spec()), } } -/// Render the README section that builds the hub as firmware for `board` and loads it onto one. -fn firmware_run_instructions(board: BoardSpec) -> String { +/// Render the README section of a format whose server is a board, which leaves the format only the runners elsewhere. +/// +/// Only a format that runs `mise.toml` tasks gets here, since the others are refused a board before anything is +/// written. +fn board_hub_run_instructions(output_type: OutputType, server: &str, board: BoardSpec) -> String { + let format_name = match output_type { + OutputType::Dekit => "dekit", + OutputType::ProcessCompose => "process-compose", + OutputType::Mise + | OutputType::DockerCompose + | OutputType::K3s + | OutputType::FirmwarePico2 + | OutputType::FirmwareEsp32c6 + | OutputType::FirmwarePicoPlus2 => "Mise", + }; + format!( + concat!( + "## Run With {format_name}\n\n", + "The hub runs on `{server}`, a {title},\n", + "as firmware the section loading it describes. The board runs no OpenObserve, so there is no UI to\n", + "open. Each runner on another machine reaches the hub across the network, and starts as that\n", + "machine's share below.\n" + ), + format_name = format_name, + server = server, + title = board.title, + ) +} + +/// Render the README sections that load each board among the machines. +/// +/// That is the hub onto the server, when it is a board, and the runners onto every other board; nothing when no +/// machine is a board. +fn board_instructions(cluster: &ClusterInput) -> String { + let Some(server) = cluster.machines.first() else { + return String::default(); + }; + let mut text = String::default(); + for machine in &cluster.machines { + let Delivery::Firmware(board) = machine.device.delivery() else { + continue; + }; + let flash = format!("mise run {}", deployment_types::firmware_flash_task(&machine.name)); + let section = if machine.name == server.name { + let heading = format!("## Load The Hub Onto `{}`", machine.name); + let host_test = format!( + "mise run {}-{}", + deployment_types::FIRMWARE_HOST_TEST_TASK, + machine.name + ); + firmware_run_instructions(board.spec(), &heading, &flash, &host_test) + } else { + runner_board_instructions(board.spec(), &machine.name, &hub_ws_url_at(&server.address), &flash) + }; + let _write_result = write!(text, "\n{section}"); + } + text +} + +/// Render the README section that loads the runners onto `machine`, a board reaching the hub at `hub_url`. +fn runner_board_instructions(board: BoardSpec, machine: &str, hub_url: &str, flash: &str) -> String { + format!( + concat!( + "## Load The Runners Onto `{machine}`\n\n", + "`{machine}` is a {title} running each {runners} runner the scenario places on it, compiled into the\n", + "firmware from its module's component, and no hub: the runners join the one at `{hub_url}`.\n", + "Build those modules first, since the firmware takes each one's built package.\n", + "{network}\n", + "Then {preparation}.\n", + "From this directory, build the firmware and load it with:\n\n", + "```bash\n", + "{flash}\n", + "```\n" + ), + flash = flash, + hub_url = hub_url, + machine = machine, + network = board.network_setup, + preparation = board.load_preparation, + runners = board.runners.join(", "), + title = board.title, + ) +} + +/// Render the README section of a `firmware-*` format, which puts the whole scenario on `board`. +fn firmware_output_instructions(board: BoardSpec) -> String { + let heading = format!("## Run As {} Firmware", board.title); + let env = board.output_type.name(); + let flash = format!("MISE_ENV={env} mise run {}", deployment_types::FIRMWARE_FLASH_TASK); + let host_test = format!("MISE_ENV={env} mise run {}", deployment_types::FIRMWARE_HOST_TEST_TASK); + firmware_run_instructions(board, &heading, &flash, &host_test) +} + +/// Render a README section, under `heading`, that builds the hub as firmware for `board` and loads it onto one. +/// +/// `flash` is the command that builds and loads it, and `host_test` the one that serves the same modules from here. +fn firmware_run_instructions(board: BoardSpec, heading: &str, flash: &str, host_test: &str) -> String { let runs = if board.runners.is_empty() { "The board runs the hub alone and".to_owned() } else { format!( concat!( - "The board runs the hub, and beside it each {} runner the scenario declares, compiled into the\n", + "The board runs the hub, and beside it each {} runner the scenario places on it, compiled into the\n", "firmware from its module's component. The build refuses a runner that imports what the board\n", "does not host, or runners needing more memory than it holds.\nThe board" ), @@ -1155,33 +1258,32 @@ fn firmware_run_instructions(board: BoardSpec) -> String { }; format!( concat!( - "## Run As {title} Firmware\n\n", - "{runs} serves this scenario's modules from its {flash} of flash.\n", + "{heading}\n\n", + "{runs} serves this scenario's modules from its {flash_size} of flash.\n", "Build those modules first, since the firmware takes each one's built package.\n", "{network}\n", "Then {preparation}.\n", "From this directory, build the firmware and load it with:\n\n", "```bash\n", - "MISE_ENV={env} mise run {task}\n", + "{flash}\n", "```\n\n", "To check what the board will serve without one, serve the same modules from this host and fetch\n", "each, as a page would:\n\n", "```bash\n", - "MISE_ENV={env} mise run {host_test}\n", + "{host_test}\n", "```\n\n", "The board takes an address over DHCP and serves the page at port 8080 of it. npm packages the\n", "modules use that are too big for its flash, such as onnxruntime-web, are not built in: the board\n", "redirects a request for one to the same path under `{elsewhere}`, which the browser has to be\n", "able to reach.\n" ), - title = board.title, + heading = heading, runs = runs, - flash = board.flash, + flash_size = board.flash, network = board.network_setup, preparation = board.load_preparation, - env = board.output_type.name(), - task = deployment_types::FIRMWARE_FLASH_TASK, - host_test = deployment_types::FIRMWARE_HOST_TEST_TASK, + flash = flash, + host_test = host_test, elsewhere = deployment_types::FIRMWARE_ELSEWHERE_URL, ) } @@ -1255,10 +1357,7 @@ fn machine_instructions(output_type: OutputType, cluster: &ClusterInput) -> Stri let (format_name, intro, command): (&str, &str, fn(&str) -> String) = match output_type { OutputType::Mise => ( "Mise", - concat!( - "Copy this directory to each machine -- for a scenario built from the repository, into a checkout\n", - "of it at the same path -- and start that machine's share of the scenario there.\n" - ), + "Copy this directory to each machine and start that machine's share of the scenario there.\n", |machine| format!("mise run {}", deployment_types::machine_task(machine)), ), OutputType::Dekit => ( @@ -1272,18 +1371,12 @@ fn machine_instructions(output_type: OutputType, cluster: &ClusterInput) -> Stri ), OutputType::ProcessCompose => ( "process-compose", - concat!( - "Copy this directory to each machine -- for a scenario built from the repository, into a checkout\n", - "of it at the same path -- and start only that machine's namespace there.\n" - ), + "Copy this directory to each machine and start only that machine's namespace there.\n", |machine| format!(concat!(process_compose_up!(), " -n {}"), machine), ), OutputType::DockerCompose => ( "Docker Compose", - concat!( - "Copy this directory to each machine -- for a scenario built from the repository, into a checkout\n", - "of it at the same path -- and start only that machine's profile there.\n" - ), + "Copy this directory to each machine and start only that machine's profile there.\n", |machine| format!("docker compose --profile {machine} up --build"), ), OutputType::K3s => return k3s_machine_instructions(cluster, server), @@ -1296,12 +1389,12 @@ fn machine_instructions(output_type: OutputType, cluster: &ClusterInput) -> Stri let loopback = loopback_note(cluster); let mut text = format!("\n### Run Across Machines With {format_name}\n\n{intro}{roles}{loopback}"); for machine in &cluster.machines { - let _write_result = writeln!( - text, - "On `{}`:\n\n```bash\n{}\n```\n", - machine.name, - command(&machine.name) - ); + // A board's share is its firmware, which every format loads the same way. + let start = match machine.device.delivery() { + Delivery::Tasks => command(&machine.name), + Delivery::Firmware(_) => format!("mise run {}", deployment_types::firmware_flash_task(&machine.name)), + }; + let _write_result = writeln!(text, "On `{}`:\n\n```bash\n{start}\n```\n", machine.name); } text.truncate(text.len().saturating_sub(1)); text @@ -1351,6 +1444,18 @@ fn compose_single_host_profiles(cluster: &ClusterInput) -> String { /// The paragraph every format's machine section opens with. #[expect(clippy::single_call_fn, reason = "the roles step of machine_instructions")] fn machine_roles(cluster: &ClusterInput, server: &input::Machine) -> String { + if let Delivery::Firmware(board) = server.device.delivery() { + return format!( + concat!( + "`{server}` is a {title} running `ws-server` as firmware,\n", + "and the runners on every other machine reach the hub at `{address}`, which the network the board\n", + "joins has to resolve to it. Its share is loading the firmware, which it then runs from power-on.\n\n" + ), + address = server.address, + server = server.name, + title = board.spec().title, + ); + } if placement::shares_hubs(cluster) { return format!( concat!( @@ -2332,6 +2437,14 @@ pub(crate) fn scenario_runners(cluster: &ClusterInput) -> Result Result, CliError> { + Ok(scenario_runners(cluster)? + .into_iter() + .filter(|runner| placement::delivery_at(cluster, &runner.site) == Delivery::Tasks) + .collect()) +} + /// Base HTTP URL a generated deployment reaches the hub on. /// /// Every generator addresses the hub by its standard insecure port, so spelling the URL out in each of them meant diff --git a/utilities/cli/src/placement.rs b/utilities/cli/src/placement.rs index 678d8482..1a670c33 100644 --- a/utilities/cli/src/placement.rs +++ b/utilities/cli/src/placement.rs @@ -5,7 +5,7 @@ use std::collections::BTreeMap; use crate::deployment_types::HUB_TASK; use crate::error::CliError; use crate::hub_ws_url::{hub_ws_url, hub_ws_url_at}; -use crate::input::{ClusterInput, Machine}; +use crate::input::{ClusterInput, Delivery, Machine}; use crate::rfc1123_label; /// Longest machine name, which is a Kubernetes label value as it is: an RFC 1123 label's 63 characters. @@ -90,6 +90,26 @@ impl Hub { } } +/// How the share at `site` reaches its machine: as the device there takes it, or as tasks on a single host. +#[must_use] +pub(crate) fn delivery_at(cluster: &ClusterInput, site: &Site) -> Delivery { + let names = site.machine_names(); + cluster + .machines + .iter() + .find(|machine| names.contains(&machine.name)) + .map_or(Delivery::Tasks, |machine| machine.device.delivery()) +} + +/// How the server's share reaches it, the hub's among it: as firmware when the server is a board. +#[must_use] +pub(crate) fn server_delivery(cluster: &ClusterInput) -> Delivery { + cluster + .machines + .first() + .map_or(Delivery::Tasks, |server| server.device.delivery()) +} + /// The site of the collector, and of NATS and the object store when the hubs share them: the server, or the one host. #[must_use] pub(crate) fn hub_site(cluster: &ClusterInput) -> Site { diff --git a/utilities/cli/tests/firmware.rs b/utilities/cli/tests/firmware.rs index 50a576db..e57cdec0 100644 --- a/utilities/cli/tests/firmware.rs +++ b/utilities/cli/tests/firmware.rs @@ -1,6 +1,7 @@ //! Covers the firmware formats, mostly through `firmware-pico2`: which modules a firmware build embeds and which it //! redirects, the scenarios a board refuses, what `firmware-esp32c6` builds and loads differently, and the wasi runners -//! `firmware-pico-plus2` starts beside its hub. +//! `firmware-pico-plus2` starts beside its hub. Also a board named as the server machine's `device:`, its hub reached +//! by the runners on the other machines. #![cfg(test)] use std::path::PathBuf; @@ -30,6 +31,27 @@ agents: - type: "wasi-math1" "#; +/// The pyo3 exchange with its wasi trigger on a Pico Plus 2 W serving the hub, and the twin on a machine with mise. +const MIXED: &str = r#"cluster_name: "mixed" +agents: + - name: "math1-trigger" + runner: "wasi" + resources: + - type: "wasi-math1-sender" + - name: "pyo3-math1-twin" + runner: "pyo3" + resources: + - type: "pyo3-math1" +machines: + - name: "hub-board" + address: "et-hub-board.local" + device: "pico-plus2" + agents: ["math1-trigger"] + - name: "edge" + address: "10.0.0.2" + agents: ["pyo3-math1-twin"] +"#; + /// The file `firmware-pico2` writes. const CONFIG_FILE: &str = "mise.firmware-pico2.toml"; @@ -191,7 +213,7 @@ fn a_pico_plus_2_w_starts_each_wasi_runner_beside_its_hub() { assert_eq!(config.tasks.host_test.env, build.env); assert!(config.tasks.flash.run.starts_with("picotool load")); assert!(readme.contains("## Run As Pimoroni Pico Plus 2 W Firmware")); - assert!(readme.contains("The board runs the hub, and beside it each wasi runner the scenario declares")); + assert!(readme.contains("The board runs the hub, and beside it each wasi runner the scenario places on it")); assert!(readme.contains("MISE_ENV=firmware-pico-plus2 mise run firmware-flash")); } @@ -239,6 +261,7 @@ fn verification_renders_it_only_for_a_scenario_the_board_can_run() { fs::write(input_dir.join("headless.yaml"), headless).unwrap(); fs::write(input_dir.join("wasi.yaml"), WASI).unwrap(); + fs::write(input_dir.join("mixed.yaml"), MIXED).unwrap(); let _regenerated = regenerate_verification(&verification_root, None).unwrap(); let output = verification_root.join("local/output"); @@ -255,4 +278,186 @@ fn verification_renders_it_only_for_a_scenario_the_board_can_run() { assert!(!output.join("wasi").join(CONFIG_FILE).exists()); assert!(!output.join("wasi").join(esp_file).exists()); assert!(output.join("wasi").join(plus_file).is_file()); + // A scenario naming a board as a machine's device, only in the formats that deliver one. + for format in OutputType::ALL { + let delivered = matches!( + format, + OutputType::Mise | OutputType::Dekit | OutputType::ProcessCompose + ); + assert_eq!( + output.join("mixed").join(format.output_file_name()).is_file(), + delivered + ); + } +} + +#[test] +fn a_board_as_the_server_runs_the_hub_and_its_guests_as_firmware() { + let (_root, output_dir, outcome) = generate_in(tempdir().unwrap(), MIXED, OutputType::Mise); + assert_eq!(outcome, ""); + let config: toml::Table = toml::from_str(&fs::read_to_string(output_dir.join("mise.toml")).unwrap()).unwrap(); + let tasks = config["tasks"].as_table().unwrap(); + // No collector, no hub of its own and no guest as a task: the board runs those. + assert_eq!( + tasks.keys().map(String::as_str).collect::>(), + [ + "firmware-build-hub-board", + "firmware-flash-hub-board", + "firmware-host-test-hub-board", + "generated-scenario", + "machine-edge", + "machine-hub-board", + "pyo3-math1-twin", + ] + ); + let build = &tasks["firmware-build-hub-board"]; + assert_eq!( + build["env"]["RUNNER_GUESTS"].as_str(), + Some("math1-trigger=@edge-toolkit/et-ws-wasi-math1-sender") + ); + let run = build["run"].as_str().unwrap(); + assert!(run.contains("mise -E rust run build-firmware-pico-plus2w")); + // The twin's module too, since it is fetched from the board's hub. + assert!(run.contains("$PWD/services/ws-modules/pyo3-math1")); + let flash = toml::Value::from("firmware-flash-hub-board"); + assert_eq!( + tasks["machine-hub-board"]["depends"].as_array().unwrap(), + std::slice::from_ref(&flash) + ); + assert_eq!( + tasks["pyo3-math1-twin"]["env"]["WS_SERVER_URL"].as_str(), + Some("ws://et-hub-board.local:8080/ws") + ); + let readme = fs::read_to_string(output_dir.join("README.md")).unwrap(); + assert!(readme.contains("## Load The Hub Onto `hub-board`")); + assert!(readme.contains("The board runs no OpenObserve")); +} + +#[test] +fn dekit_and_process_compose_supervise_only_the_runners_off_the_board() { + for (format, file, key) in [ + (OutputType::Dekit, "dekit.yaml", "tasks"), + (OutputType::ProcessCompose, "process-compose.yaml", "processes"), + ] { + let (_root, output_dir, outcome) = generate_in(tempdir().unwrap(), MIXED, format); + assert_eq!(outcome, ""); + let written: serde_yaml::Value = + serde_yaml::from_str(&fs::read_to_string(output_dir.join(file)).unwrap()).unwrap(); + let names = written[key].as_mapping().unwrap().keys().collect::>(); + assert_eq!(names, [&serde_yaml::Value::from("pyo3-math1-twin")]); + let readme = fs::read_to_string(output_dir.join("README.md")).unwrap(); + assert!(readme.contains("On `hub-board`:\n\n```bash\nmise run firmware-flash-hub-board\n```")); + } +} + +#[test] +fn a_format_that_ships_images_is_refused_a_device() { + for format in [OutputType::DockerCompose, OutputType::K3s] { + let (_root, _output_dir, outcome) = generate_in(tempdir().unwrap(), MIXED, format); + assert_eq!( + outcome, + format!( + r#"Machine "hub-board" names a device, which {} cannot deliver; only mise-task formats can"#, + format.name() + ) + ); + } +} + +#[test] +fn a_board_off_the_hub_s_machine_runs_its_runners_against_that_hub() { + // The hub on a machine with mise, and the wasi trigger on a board of its own. + let runner_board = concat!( + " - name: \"runner-board\"\n address: \"10.0.0.3\"\n device: \"pico-plus2\"\n", + " agents: [\"math1-trigger\"]\n - name: \"edge\"", + ); + let swapped = MIXED + .replace(r#"device: "pico-plus2""#, "") + .replace(" - name: \"edge\"", runner_board); + let swapped = swapped.replacen(r#"agents: ["math1-trigger"]"#, "agents: []", 1); + let (_root, output_dir, outcome) = generate_in(tempdir().unwrap(), &swapped, OutputType::Mise); + assert_eq!(outcome, ""); + let config: toml::Table = toml::from_str(&fs::read_to_string(output_dir.join("mise.toml")).unwrap()).unwrap(); + let tasks = config["tasks"].as_table().unwrap(); + let build = &tasks["firmware-build-runner-board"]; + assert_eq!(build["env"]["HUB_ADDRESS"].as_str(), Some("et-hub-board.local:8080")); + assert_eq!( + build["env"]["CARGO_TARGET_DIR"].as_str(), + Some("target/firmware/runner-board") + ); + assert_eq!(build["env"]["MODULES_ROOT"].as_str(), Some("")); + assert_eq!( + build["env"]["RUNNER_GUESTS"].as_str(), + Some("math1-trigger=@edge-toolkit/et-ws-wasi-math1-sender") + ); + // Only the runner's own module is built in, since the board serves nothing. + let run = build["run"].as_str().unwrap(); + assert!(run.contains("$PWD/services/ws-modules/wasi-math1-sender")); + assert!(!run.contains("pyo3-math1")); + let flash = tasks["firmware-flash-runner-board"]["run"].as_str().unwrap(); + assert!(flash.ends_with("target/firmware/runner-board/thumbv8m.main-none-eabihf/release/et-ws-server-firmware")); + assert!(!tasks.contains_key("firmware-host-test-runner-board")); + let readme = fs::read_to_string(output_dir.join("README.md")).unwrap(); + assert!(readme.contains("## Load The Runners Onto `runner-board`")); + assert!(readme.contains("the runners join the one at `ws://et-hub-board.local:8080/ws`")); +} + +#[test] +fn a_board_off_the_hub_s_machine_with_no_runner_is_refused() { + let idle = MIXED + .replace(r#"device: "pico-plus2""#, "") + .replace(r#"address: "10.0.0.2""#, "address: \"10.0.0.2\"\n device: \"pico2\"") + .replace(r#"agents: ["pyo3-math1-twin"]"#, "agents: []"); + let (_root, output_dir, outcome) = generate_in(tempdir().unwrap(), &idle, OutputType::Mise); + assert_eq!( + outcome, + concat!( + r#"Machine "edge" is a Raspberry Pi Pico 2 W with no runner placed on it, "#, + "and off the hub's machine it runs only those" + ) + ); + assert!(!output_dir.join("mise.toml").exists()); +} + +#[test] +fn a_board_as_a_machine_refuses_what_its_firmware_cannot_run() { + let pico2 = MIXED.replace(r#"device: "pico-plus2""#, r#"device: "pico2""#); + let pyo3_on_board = MIXED + .replace(r#"["math1-trigger"]"#, "[swapped]") + .replace(r#"["pyo3-math1-twin"]"#, r#"["math1-trigger"]"#) + .replace("[swapped]", r#"["pyo3-math1-twin"]"#); + let limited = MIXED.replacen( + " resources:", + " limits:\n memory: \"64m\"\n resources:", + 1, + ); + let scheduled = MIXED.replacen( + " resources:", + " schedule:\n cron: \"0 * * * *\"\n resources:", + 1, + ); + let published = format!("{MIXED}artifact_source: \"published\"\n"); + let shared = format!("{MIXED}services:\n hub:\n machines: [\"hub-board\", \"edge\"]\n"); + let plus = "The Pimoroni Pico Plus 2 W firmware cannot deploy a scenario that"; + for (input, format, refusal) in [ + ( + pico2, + OutputType::Mise, + "The Raspberry Pi Pico 2 W firmware cannot deploy a scenario that starts a runner".to_owned(), + ), + ( + pyo3_on_board, + OutputType::Mise, + r#"Agent "pyo3-math1-twin" names the "pyo3" runner, but the Pimoroni Pico Plus 2 W firmware runs only wasi"# + .to_owned(), + ), + (limited, OutputType::Mise, format!("{plus} sets limits on a runner")), + (scheduled, OutputType::ProcessCompose, format!("{plus} schedules a runner")), + (published, OutputType::Mise, format!("{plus} uses published artifacts")), + (shared, OutputType::Mise, format!("{plus} runs a hub on another machine too")), + ] { + let (_root, output_dir, outcome) = generate_in(tempdir().unwrap(), &input, format); + assert!(outcome.starts_with(&refusal)); + assert!(!output_dir.join("mise.toml").exists()); + } } diff --git a/utilities/cli/tests/input.rs b/utilities/cli/tests/input.rs index c90603a8..83309e42 100644 --- a/utilities/cli/tests/input.rs +++ b/utilities/cli/tests/input.rs @@ -7,7 +7,7 @@ #![cfg(test)] use et_cli::{ - ArtifactSource, ClusterInput, DEFAULT_CLUSTER_NAME, OutputType, infer_artifact_source, + ArtifactSource, ClusterInput, DEFAULT_CLUSTER_NAME, Device, OutputType, infer_artifact_source, manifest_declares_this_repository, }; @@ -152,6 +152,30 @@ fn every_deployment_type_spelling_deserializes_to_its_own_output_file() { assert_eq!(OutputType::ALL.len(), 8, "and the list of them is complete"); } +#[test] +fn a_machine_s_device_is_named_by_its_hardware_and_left_out_when_unspecified() { + let machines = concat!( + "machines:\n", + " - name: \"board\"\n address: \"board.local\"\n device: \"pico_plus2\"\n", + " - name: \"edge\"\n address: \"10.0.0.2\"\n", + ); + let cluster = parse(&format!("{ONLY_A_NAME}{machines}")); + let devices = cluster + .machines + .iter() + .map(|machine| machine.device) + .collect::>(); + assert_eq!(devices, [Device::PicoPlus2, Device::Unspecified]); + + let rendered = serde_yaml::to_string(&cluster).unwrap(); + assert_eq!(rendered.matches("device:").count(), 1); + assert!(rendered.contains("device: pico-plus2")); + + let unknown = + serde_yaml::from_str::(&format!("{ONLY_A_NAME}{}", machines.replace("pico_plus2", "pi"))); + assert!(unknown.unwrap_err().to_string().contains("pi")); +} + #[test] fn an_unknown_deployment_type_is_rejected() { // The reason it is an enum: as a string, a misspelling reached the generator instead of the reader. diff --git a/utilities/int-gen/src/openapi.rs b/utilities/int-gen/src/openapi.rs index 202ee2d9..fc240d26 100644 --- a/utilities/int-gen/src/openapi.rs +++ b/utilities/int-gen/src/openapi.rs @@ -37,7 +37,7 @@ the hub has blocked is refused by them with 403. The admin routes take the hub's et_ws_service::admin::block_ip, et_ws_service::admin::evict_agent, ), - components(schemas(et_ws_server::routes::HealthResponse, et_ws_service::admin::EvictedAgent)), + components(schemas(et_ws_service::health::HealthResponse, et_ws_service::admin::EvictedAgent)), modifiers(&AdminTokenScheme) )] struct ApiDoc; diff --git a/verification/local/input/mixed-hardware.yaml b/verification/local/input/mixed-hardware.yaml new file mode 100644 index 00000000..c676bf2c --- /dev/null +++ b/verification/local/input/mixed-hardware.yaml @@ -0,0 +1,34 @@ +cluster_name: "mixed-hardware" +deployment_type: "mise" + +# The pyo3-math1 exchange split across three kinds of hardware. The server is a Raspberry Pi Pico 2 W, whose firmware +# runs the hub alone: it has no memory for a runner. The WASI trigger runs on a Pimoroni Pico Plus 2 W, compiled into +# that board's firmware from its module's component, and reaches the hub across the network. The pyo3 twin needs real +# CPython, which no board has, so it runs on `edge`, a machine with mise on it. Every exchange goes through the hub's +# storage, which the Pico 2 W keeps in memory. The boards are named as the hosts they answer to on the network they +# join, since no board can be loopback. +agents: + - name: "math1-trigger" + runner: "wasi" + resources: + - type: "wasi-math1-sender" + + - name: "pyo3-math1-twin" + runner: "pyo3" + resources: + - type: "pyo3-math1" + +machines: + - name: "hub-board" + address: "et-hub-board.local" + device: "pico2" + + # Before the trigger's board, so the README starts the twin first: the trigger broadcasts for only a minute. + - name: "edge" + address: "10.0.0.2" + agents: ["pyo3-math1-twin"] + + - name: "runner-board" + address: "et-runner-board.local" + device: "pico-plus2" + agents: ["math1-trigger"] diff --git a/verification/local/output/default/README.md b/verification/local/output/default/README.md index 4b930de3..ebdf78fa 100644 --- a/verification/local/output/default/README.md +++ b/verification/local/output/default/README.md @@ -204,7 +204,7 @@ able to reach. ## Run As Pimoroni Pico Plus 2 W Firmware -The board runs the hub, and beside it each wasi runner the scenario declares, compiled into the +The board runs the hub, and beside it each wasi runner the scenario places on it, compiled into the firmware from its module's component. The build refuses a runner that imports what the board does not host, or runners needing more memory than it holds. The board serves this scenario's modules from its 16 MiB of flash. diff --git a/verification/local/output/facility-security-scenario/README.md b/verification/local/output/facility-security-scenario/README.md index 7c068a65..aa59088c 100644 --- a/verification/local/output/facility-security-scenario/README.md +++ b/verification/local/output/facility-security-scenario/README.md @@ -204,7 +204,7 @@ able to reach. ## Run As Pimoroni Pico Plus 2 W Firmware -The board runs the hub, and beside it each wasi runner the scenario declares, compiled into the +The board runs the hub, and beside it each wasi runner the scenario places on it, compiled into the firmware from its module's component. The build refuses a runner that imports what the board does not host, or runners needing more memory than it holds. The board serves this scenario's modules from its 16 MiB of flash. diff --git a/verification/local/output/math1-multi-hub/README.md b/verification/local/output/math1-multi-hub/README.md index 2d75ebea..d0041783 100644 --- a/verification/local/output/math1-multi-hub/README.md +++ b/verification/local/output/math1-multi-hub/README.md @@ -45,8 +45,7 @@ mise run open-o2 ### Run Across Machines With Mise -Copy this directory to each machine -- for a scenario built from the repository, into a checkout -of it at the same path -- and start that machine's share of the scenario there. +Copy this directory to each machine and start that machine's share of the scenario there. `server` runs OpenObserve, `ws-server`, and the NATS server and object store the hubs share, which the other machines' hubs reach at `127.0.0.1`. Each runner joins the hub on its own machine, or the one on `server` when its machine runs none. @@ -134,8 +133,7 @@ Quitting the TUI stops every process. Open the OpenObserve UI, once it is up, by ### Run Across Machines With process-compose -Copy this directory to each machine -- for a scenario built from the repository, into a checkout -of it at the same path -- and start only that machine's namespace there. +Copy this directory to each machine and start only that machine's namespace there. `server` runs OpenObserve, `ws-server`, and the NATS server and object store the hubs share, which the other machines' hubs reach at `127.0.0.1`. Each runner joins the hub on its own machine, or the one on `server` when its machine runs none. @@ -184,8 +182,7 @@ docker compose --profile server down ### Run Across Machines With Docker Compose -Copy this directory to each machine -- for a scenario built from the repository, into a checkout -of it at the same path -- and start only that machine's profile there. +Copy this directory to each machine and start only that machine's profile there. `server` runs OpenObserve, `ws-server`, and the NATS server and object store the hubs share, which the other machines' hubs reach at `127.0.0.1`. Each runner joins the hub on its own machine, or the one on `server` when its machine runs none. diff --git a/verification/local/output/math1-multi/README.md b/verification/local/output/math1-multi/README.md index 619fb4f9..8b1a30dc 100644 --- a/verification/local/output/math1-multi/README.md +++ b/verification/local/output/math1-multi/README.md @@ -45,8 +45,7 @@ mise run open-o2 ### Run Across Machines With Mise -Copy this directory to each machine -- for a scenario built from the repository, into a checkout -of it at the same path -- and start that machine's share of the scenario there. +Copy this directory to each machine and start that machine's share of the scenario there. `server` runs OpenObserve and `ws-server`, and the runners on every other machine reach the hub at `127.0.0.1`. @@ -132,8 +131,7 @@ Quitting the TUI stops every process. Open the OpenObserve UI, once it is up, by ### Run Across Machines With process-compose -Copy this directory to each machine -- for a scenario built from the repository, into a checkout -of it at the same path -- and start only that machine's namespace there. +Copy this directory to each machine and start only that machine's namespace there. `server` runs OpenObserve and `ws-server`, and the runners on every other machine reach the hub at `127.0.0.1`. @@ -178,8 +176,7 @@ docker compose --profile server --profile edge down ### Run Across Machines With Docker Compose -Copy this directory to each machine -- for a scenario built from the repository, into a checkout -of it at the same path -- and start only that machine's profile there. +Copy this directory to each machine and start only that machine's profile there. `server` runs OpenObserve and `ws-server`, and the runners on every other machine reach the hub at `127.0.0.1`. diff --git a/verification/local/output/mixed-hardware/.gitignore b/verification/local/output/mixed-hardware/.gitignore new file mode 100644 index 00000000..4b22f893 --- /dev/null +++ b/verification/local/output/mixed-hardware/.gitignore @@ -0,0 +1,7 @@ +# Written by `et-cli`. +# What the hub writes into the directory it runs from: its self-signed TLS pair (the key is a +# credential too), the agent registry it saves on shutdown, and agent storage. +cert.pem +key.pem +registry.yaml +storage/ diff --git a/verification/local/output/mixed-hardware/README.md b/verification/local/output/mixed-hardware/README.md new file mode 100644 index 00000000..4351609d --- /dev/null +++ b/verification/local/output/mixed-hardware/README.md @@ -0,0 +1,162 @@ +# mixed-hardware + +This directory contains generated deployment configs for the `mixed-hardware` scenario. +Files: `mise.toml`, `dekit.yaml`, `process-compose.yaml`. + +The scenario exposes these workflow modules: pyo3-math1, wasi-math1-sender. + +`secrets.env` holds the scenario's derived OpenObserve and OTLP credentials. It is derived from the +scenario input, so regenerating this deployment rewrites it; if it is missing, regenerate before +starting the stack. A deployment generated outside the repository is written with a `.gitignore` +covering it, so its credential is not committed by whatever repository it lands in. + +## Lock The Tools + +Before the first run, lock the tools `mise.toml` names: + +```bash +mise lock +``` + +Commit `mise.lock` alongside the deployment, together with the `.mise/locks/` directory when +mise writes one -- it holds the dependency graph of each npm tool, which `mise.lock` refers to. +The two record the version and checksum every `latest` in `mise.toml` resolved to, for each of +mise's common platforms, so every later install -- on any machine -- fetches the same binaries +rather than whatever is newest that day. Move to newer releases deliberately, with +`mise lock --bump`, and commit the result. + +## Run With Mise + +The hub runs on `hub-board`, a Raspberry Pi Pico 2 W, +as firmware the section loading it describes. The board runs no OpenObserve, so there is no UI to +open. Each runner on another machine reaches the hub across the network, and starts as that +machine's share below. + +### Run Across Machines With Mise + +Copy this directory to each machine and start that machine's share of the scenario there. +`hub-board` is a Raspberry Pi Pico 2 W running `ws-server` as firmware, +and the runners on every other machine reach the hub at `et-hub-board.local`, which the network the board +joins has to resolve to it. Its share is loading the firmware, which it then runs from power-on. + +On `hub-board`: + +```bash +mise run firmware-flash-hub-board +``` + +On `edge`: + +```bash +mise run machine-edge +``` + +On `runner-board`: + +```bash +mise run firmware-flash-runner-board +``` + +## Run With dekit + +The hub runs on `hub-board`, a Raspberry Pi Pico 2 W, +as firmware the section loading it describes. The board runs no OpenObserve, so there is no UI to +open. Each runner on another machine reaches the hub across the network, and starts as that +machine's share below. + +### Run Across Machines With dekit + +Copy this directory to each machine and start that machine's share there. Every task is tagged +with its machine and none starts on `dekit up` alone, so `dekit up` on any machine starts nothing +until its tag is started. +`hub-board` is a Raspberry Pi Pico 2 W running `ws-server` as firmware, +and the runners on every other machine reach the hub at `et-hub-board.local`, which the network the board +joins has to resolve to it. Its share is loading the firmware, which it then runs from power-on. + +On `hub-board`: + +```bash +mise run firmware-flash-hub-board +``` + +On `edge`: + +```bash +dekit up +dekit start +edge +``` + +On `runner-board`: + +```bash +mise run firmware-flash-runner-board +``` + +## Run With process-compose + +The hub runs on `hub-board`, a Raspberry Pi Pico 2 W, +as firmware the section loading it describes. The board runs no OpenObserve, so there is no UI to +open. Each runner on another machine reaches the hub across the network, and starts as that +machine's share below. + +### Run Across Machines With process-compose + +Copy this directory to each machine and start only that machine's namespace there. +`hub-board` is a Raspberry Pi Pico 2 W running `ws-server` as firmware, +and the runners on every other machine reach the hub at `et-hub-board.local`, which the network the board +joins has to resolve to it. Its share is loading the firmware, which it then runs from power-on. + +On `hub-board`: + +```bash +mise run firmware-flash-hub-board +``` + +On `edge`: + +```bash +process-compose up --no-server -f process-compose.yaml -n edge +``` + +On `runner-board`: + +```bash +mise run firmware-flash-runner-board +``` + +## Load The Hub Onto `hub-board` + +The board runs the hub alone and serves this scenario's modules from its 4 MiB of flash. +Build those modules first, since the firmware takes each one's built package. +Set `WIFI_NETWORK` and `WIFI_PASSWORD` to the Wi-Fi network the board joins. +Then hold the board's BOOTSEL button while plugging it in. +From this directory, build the firmware and load it with: + +```bash +mise run firmware-flash-hub-board +``` + +To check what the board will serve without one, serve the same modules from this host and fetch +each, as a page would: + +```bash +mise run firmware-host-test-hub-board +``` + +The board takes an address over DHCP and serves the page at port 8080 of it. npm packages the +modules use that are too big for its flash, such as onnxruntime-web, are not built in: the board +redirects a request for one to the same path under `https://cdn.jsdelivr.net/npm`, which the browser has to be +able to reach. + +## Load The Runners Onto `runner-board` + +`runner-board` is a Pimoroni Pico Plus 2 W running each wasi runner the scenario places on it, compiled into the +firmware from its module's component, and no hub: the runners join the one at `ws://et-hub-board.local:8080/ws`. +Build those modules first, since the firmware takes each one's built package. +Set `WIFI_NETWORK` and `WIFI_PASSWORD` to the Wi-Fi network the board joins. +Then hold the board's BOOTSEL button while plugging it in. +From this directory, build the firmware and load it with: + +```bash +mise run firmware-flash-runner-board +``` diff --git a/verification/local/output/mixed-hardware/dekit.yaml b/verification/local/output/mixed-hardware/dekit.yaml new file mode 100644 index 00000000..9f0a59cf --- /dev/null +++ b/verification/local/output/mixed-hardware/dekit.yaml @@ -0,0 +1,11 @@ +defaults: + stop: SIGINT +tasks: + pyo3-math1-twin: + cmd: + - mise + - run + - pyo3-math1-twin + tags: + - edge + autorestart: on-failure diff --git a/verification/local/output/mixed-hardware/mise.toml b/verification/local/output/mixed-hardware/mise.toml new file mode 100644 index 00000000..0ec78bc9 --- /dev/null +++ b/verification/local/output/mixed-hardware/mise.toml @@ -0,0 +1,99 @@ +[env] +OTLP_AUTH_USERNAME = "root@example.com" + +[env._] +file = "secrets.env" + +[tasks.firmware-build-hub-board] +description = "Build the Raspberry Pi Pico 2 W firmware serving mixed-hardware" +dir = "../../../.." +run = """ +MODULES_PATHS="$PWD/services/ws-modules/pyo3-math1, $PWD/services/ws-modules/wasi-math1-sender" +MODULES_PATHS="$MODULES_PATHS, $PWD/services/ws-server/static, $PWD/services/ws-wasm-agent" +export MODULES_PATHS +mise -E rust run build-firmware-pico2w +""" + +[tasks.firmware-build-hub-board.env] +CARGO_TARGET_DIR = "target/firmware/hub-board" +MODULES_ELSEWHERE = "onnxruntime-web, stats-gl" +MODULES_ELSEWHERE_URL = "https://cdn.jsdelivr.net/npm" +MODULES_ROOT = "@edge-toolkit/et-ws-server-static" +RUNNER_GUESTS = "" + +[tasks.firmware-build-runner-board] +description = "Build the Pimoroni Pico Plus 2 W firmware running the runners of mixed-hardware" +dir = "../../../.." +run = """ +MODULES_PATHS="$PWD/services/ws-modules/wasi-math1-sender" +export MODULES_PATHS +mise -E rust run build-firmware-pico-plus2w +""" + +[tasks.firmware-build-runner-board.env] +CARGO_TARGET_DIR = "target/firmware/runner-board" +HUB_ADDRESS = "et-hub-board.local:8080" +MODULES_ELSEWHERE = "" +MODULES_ELSEWHERE_URL = "https://cdn.jsdelivr.net/npm" +MODULES_ROOT = "" +RUNNER_GUESTS = "math1-trigger=@edge-toolkit/et-ws-wasi-math1-sender" + +[tasks.firmware-flash-hub-board] +depends = ["firmware-build-hub-board"] +description = "Load the Raspberry Pi Pico 2 W firmware onto the board, and start it" +dir = "../../../.." +run = "picotool load -x -t elf target/firmware/hub-board/thumbv8m.main-none-eabihf/release/et-ws-server-firmware" + +[tasks.firmware-flash-runner-board] +depends = ["firmware-build-runner-board"] +description = "Load the Pimoroni Pico Plus 2 W firmware onto the board, and start it" +dir = "../../../.." +run = "picotool load -x -t elf target/firmware/runner-board/thumbv8m.main-none-eabihf/release/et-ws-server-firmware" + +[tasks.firmware-host-test-hub-board] +description = "Serve mixed-hardware's modules from this host as the board would, and fetch each" +dir = "../../../.." +run = """ +MODULES_PATHS="$PWD/services/ws-modules/pyo3-math1, $PWD/services/ws-modules/wasi-math1-sender" +MODULES_PATHS="$MODULES_PATHS, $PWD/services/ws-server/static, $PWD/services/ws-wasm-agent" +export MODULES_PATHS +mise run cargo-test-pkg et-int-ws-server-firmware +""" + +[tasks.firmware-host-test-hub-board.env] +CARGO_TARGET_DIR = "target/firmware/hub-board" +MODULES_ELSEWHERE = "onnxruntime-web, stats-gl" +MODULES_ELSEWHERE_URL = "https://cdn.jsdelivr.net/npm" +MODULES_ROOT = "@edge-toolkit/et-ws-server-static" +RUNNER_GUESTS = "" + +[tasks.generated-scenario] +depends = ["firmware-flash-hub-board", "firmware-flash-runner-board", "pyo3-math1-twin"] +description = "Run generated scenario for mixed-hardware" + +[tasks.machine-edge] +depends = ["pyo3-math1-twin"] +description = "Run what machine edge runs" + +[tasks.machine-hub-board] +depends = ["firmware-flash-hub-board"] +description = "Run what machine hub-board runs" + +[tasks.machine-runner-board] +depends = ["firmware-flash-runner-board"] +description = "Run what machine runner-board runs" + +[tasks.pyo3-math1-twin] +description = "Run @edge-toolkit/et-ws-pyo3-math1 in the pyo3 runner" +dir = "../../../.." +run = """ +cargo run --quiet -p et-ws-pyo3-runner +""" + +[tasks.pyo3-math1-twin.env] +RUNNER_MODULE = "@edge-toolkit/et-ws-pyo3-math1" +WS_SERVER_URL = "ws://et-hub-board.local:8080/ws" + +[tools] +"aqua:F1bonacc1/process-compose" = "latest" +"cargo:open" = "latest" diff --git a/verification/local/output/mixed-hardware/process-compose.yaml b/verification/local/output/mixed-hardware/process-compose.yaml new file mode 100644 index 00000000..2155804b --- /dev/null +++ b/verification/local/output/mixed-hardware/process-compose.yaml @@ -0,0 +1,9 @@ +version: "0.5" +processes: + pyo3-math1-twin: + command: mise run pyo3-math1-twin + namespace: edge + shutdown: + signal: 2 + availability: + restart: on_failure diff --git a/verification/local/output/mixed-hardware/secrets.env b/verification/local/output/mixed-hardware/secrets.env new file mode 100644 index 00000000..c9d53b4c --- /dev/null +++ b/verification/local/output/mixed-hardware/secrets.env @@ -0,0 +1,5 @@ +# Generated from the scenario input -- regenerating this deployment rewrites it. +# skipcq: SCT-A000 -- derived from the scenario input, not an independently generated secret +ZO_ROOT_USER_PASSWORD=Aut-ut2349 +# skipcq: SCT-A000 -- derived from the scenario input, not an independently generated secret +OTLP_AUTH_PASSWORD=Aut-ut2349 diff --git a/verification/local/output/wasi-math1/README.md b/verification/local/output/wasi-math1/README.md index fe8552b2..435b8308 100644 --- a/verification/local/output/wasi-math1/README.md +++ b/verification/local/output/wasi-math1/README.md @@ -157,7 +157,7 @@ kubectl get pods -n "$ns" --watch ## Run As Pimoroni Pico Plus 2 W Firmware -The board runs the hub, and beside it each wasi runner the scenario declares, compiled into the +The board runs the hub, and beside it each wasi runner the scenario places on it, compiled into the firmware from its module's component. The build refuses a runner that imports what the board does not host, or runners needing more memory than it holds. The board serves this scenario's modules from its 16 MiB of flash. diff --git a/verification/published/output/math1-multi/README.md b/verification/published/output/math1-multi/README.md index d175aae9..0272e7a9 100644 --- a/verification/published/output/math1-multi/README.md +++ b/verification/published/output/math1-multi/README.md @@ -58,8 +58,7 @@ mise run open-o2 ### Run Across Machines With Mise -Copy this directory to each machine -- for a scenario built from the repository, into a checkout -of it at the same path -- and start that machine's share of the scenario there. +Copy this directory to each machine and start that machine's share of the scenario there. `server` runs OpenObserve and `ws-server`, and the runners on every other machine reach the hub at `127.0.0.1`. @@ -143,8 +142,7 @@ Quitting the TUI stops every process. Open the OpenObserve UI, once it is up, by ### Run Across Machines With process-compose -Copy this directory to each machine -- for a scenario built from the repository, into a checkout -of it at the same path -- and start only that machine's namespace there. +Copy this directory to each machine and start only that machine's namespace there. `server` runs OpenObserve and `ws-server`, and the runners on every other machine reach the hub at `127.0.0.1`. @@ -188,8 +186,7 @@ docker compose --profile server --profile edge down ### Run Across Machines With Docker Compose -Copy this directory to each machine -- for a scenario built from the repository, into a checkout -of it at the same path -- and start only that machine's profile there. +Copy this directory to each machine and start only that machine's profile there. `server` runs OpenObserve and `ws-server`, and the runners on every other machine reach the hub at `127.0.0.1`.