diff --git a/.github/workflows/no-std.yaml b/.github/workflows/no-std.yaml index a9225e8a..29beeba8 100644 --- a/.github/workflows/no-std.yaml +++ b/.github/workflows/no-std.yaml @@ -2,8 +2,10 @@ name: no-std # The crates that build without std, built where there is none, and et-ws-server as Raspberry Pi Pico 2 W firmware. -# One Linux job: a bare-metal target compiles the same on every host, so another platform would only repeat it. The -# board is not here, so the firmware is built and linted, not run; nothing is uploaded or published. +# The firmware is linted bare, and built as a generated scenario's, whose modules its host build then serves to a +# test that fetches each. One Linux job: a bare-metal target compiles the same on every host, so another platform +# would only repeat it. The board is not here, so the firmware itself is built and linted, not run; nothing is +# uploaded or published. # # Not path-gated: what breaks a no_std build is a dependency or a feature gaining std somewhere in the tree, which a # path list would have to name every crate to catch. @@ -58,17 +60,22 @@ jobs: timeout-minutes: 20 run: mise run test-no-std - - name: Fetch the Pico 2 W radio's firmware blobs - timeout-minutes: 5 - run: mise run fetch-cyw43-firmware-rclone - # The firmware bakes in the network it joins, so CI builds it for one that does not exist. - - name: Build and lint the Pico 2 W firmware - timeout-minutes: 30 + # The lint covers the firmware with no modules in it, and the build the default scenario's, as et-cli generates + # a deployment of it: the page and the wasm agent built in, which the agent's package has to be built for first. + # The same modules are then served from this host by the firmware's host build, and every one it lists fetched. + - name: Lint the Pico 2 W firmware, then build and serve a generated scenario's + timeout-minutes: 45 env: - CYW43_FIRMWARE_DIR: ${{ github.workspace }}/target/cyw43-firmware WIFI_NETWORK: edge-toolkit-ci WIFI_PASSWORD: not-a-real-passphrase run: | - mise run build-firmware-pico2w mise run cargo-clippy-check-pico2w + mise run build-ws-wasm-agent + src=verification/local/input/default.yaml + dst=target/firmware-default + fmt=firmware-pico2 + cargo run -q -p et-cli -- generate-deployment --input-file "$src" --output-dir "$dst" --output-type "$fmt" + cd "$dst" + MISE_ENV="$fmt" mise run firmware-build + MISE_ENV="$fmt" mise run firmware-host-test diff --git a/.mise/config.coverage.toml b/.mise/config.coverage.toml index f4650541..428293d7 100644 --- a/.mise/config.coverage.toml +++ b/.mise/config.coverage.toml @@ -43,7 +43,7 @@ version = "0.9.1" # only be asserted where that path runs. A crate belongs to exactly one list: naming it in both would let a # gap in one pipeline be answered by the other. native_cov_libs = """ -{{ "actix-web-config edge-toolkit et-otlp path test-helpers " ~ +{{ "actix-web-config edge-toolkit et-otlp path sanitise test-helpers " ~ "test-nats test-otlp test-recovery ws-runner-common" -}} """ wasm_cov_libs = "wasi-guest web" diff --git a/.mise/config.rust.toml b/.mise/config.rust.toml index 8c07ca2a..7f5e2f3a 100644 --- a/.mise/config.rust.toml +++ b/.mise/config.rust.toml @@ -9,16 +9,21 @@ # ws-server, the e2e tests) still live in the default config; moving those out is a later pass. # et-ws-server as Raspberry Pi Pico 2 W firmware: an ELF that picotool or probe-rs loads onto the board. -# Its build reads the radio's blobs from CYW43_FIRMWARE_DIR, and the network to join from WIFI_NETWORK and -# WIFI_PASSWORD; its name stays outside `build-modules`'s `build-ws-*` glob, since that has none of them. +# Its build reads the radio's blobs from CYW43_FIRMWARE_DIR, which it fetches first, and the network to join from +# WIFI_NETWORK and WIFI_PASSWORD; its name stays outside `build-modules`'s `build-ws-*` glob, since that has none of +# them. [tasks.build-firmware-pico2w] +depends = ["fetch-cyw43-firmware-rclone"] description = "Build et-ws-server as Raspberry Pi Pico 2 W firmware" +env = { CYW43_FIRMWARE_DIR = "{{ config_root }}/{{ vars.cyw43_dir }}" } run = "cargo build --release -p et-int-ws-server-pico2w --target thumbv8m.main-none-eabihf" # The firmware's body compiles only for the board, so the workspace's host clippy never reaches it; this does. # It needs what build-firmware-pico2w needs. [tasks.cargo-clippy-check-pico2w] +depends = ["fetch-cyw43-firmware-rclone"] description = "Lint the Raspberry Pi Pico 2 W firmware for its own target" +env = { CYW43_FIRMWARE_DIR = "{{ config_root }}/{{ vars.cyw43_dir }}" } run = "cargo clippy --release -p et-int-ws-server-pico2w --target thumbv8m.main-none-eabihf" # The Pico 2 W radio's firmware, CLM and NVRAM, from embassy's cyw43-firmware directory at the pinned commit. diff --git a/.mise/config.toml b/.mise/config.toml index 2201c0ee..d1c59bea 100644 --- a/.mise/config.toml +++ b/.mise/config.toml @@ -3220,6 +3220,12 @@ alias = "cargo-nextest" description = "Run every Rust test via nextest (web-runner excluded on gnullvm)" run = "cargo nextest run --workspace {{ vars.cargo_ws_excludes }} --config-file config/nextest.toml" +# One package's tests, under the same runner config as the whole suite, for a caller that needs only that package's. +[tasks.cargo-test-pkg] +description = "Run one workspace package's Rust tests via nextest" +run = 'cargo nextest run --config-file config/nextest.toml -p "$usage_package"' +usage = 'arg "" help="Workspace package name (e.g. et-int-ws-server-pico2w)"' + [tasks.test-recovery-supervisor] description = "Test recovery policy, Kubernetes observations, persistence, and hub lifecycle transport" run = "cargo test -p et-recovery-supervisor" diff --git a/CHANGELOG.md b/CHANGELOG.md index 4c59e61f..22357962 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -19,10 +19,21 @@ Changelog 1.1.0. `TcpSocket`. It is a local hub, with no backplane, admin API, idle timeout or registry file. A crate awaiting `Hub::serve` needs `#![recursion_limit = "256"]`. - `et-ws-server` builds as `no_std` + `alloc` without its new `std` feature. Its `picoserve-server` feature, on by - default, adds `picoserve_server::serve`: the hub's `/ws` with `/health` and `/favicon.ico` beside it, over picoserve. + default, adds `picoserve_server::serve`: the hub's `/ws` with `/health` and `/favicon.ico` beside it, over picoserve, + and the modules a `picoserve_modules::Modules` builds into the binary under `/modules` and `/`, as the `std` server + serves them from disk. A module listed as served elsewhere is redirected to a base URL instead. - The `et-ws-server-pico2w` firmware runs et-ws-server on a Raspberry Pi Pico 2 W, over the board's Wi-Fi. - `MISE_ENV=rust mise run build-firmware-pico2w` builds it from `WIFI_NETWORK`, `WIFI_PASSWORD` and a - `CYW43_FIRMWARE_DIR` holding the radio's firmware blobs. + `MISE_ENV=rust mise run build-firmware-pico2w` fetches the radio's firmware blobs and builds it for the network + `WIFI_NETWORK` and `WIFI_PASSWORD` name, embedding the modules `MODULES_PATHS` names and serving `MODULES_ROOT` at + `/`. +- `et-cli generate-deployment --output-type firmware-pico2` writes a `mise.firmware-pico2.toml` whose + `firmware-flash` task builds that firmware with the scenario's modules in it and loads it onto a board. The npm + packages the modules use, such as onnxruntime-web, are redirected to jsDelivr rather than built in. A scenario with + a runner, `machines:` or published artifacts is refused, since the board runs one hub and nothing else. Its + `firmware-host-test` task serves the same modules from the host, as the firmware's host build does, and fetches + every module it lists, with no board needed. +- `et-firmware-modules` generates the source a firmware's build script writes to build ws-modules into its image, + from the same `MODULES_*` variables, so every board's firmware embeds modules the same way. - An opt-in `et-recovery-supervisor` service observes application progress and coordinates bounded Kubernetes recovery. - The Python SDK provides recovery lifecycle messages and integrity-checked checkpoint storage helpers. - A scenario can list `machines:` to span several hosts. The first is the server, running the hub and the collector, @@ -82,6 +93,11 @@ Changelog 1.1.0. feature. A build with `default-features = false` and none of the features that imply `std` -- `modules`, `otlp`, `qr`, `storage`, `tls`, `websockify` -- now gets the `no_std` library and no binary; add `std` to keep the server. +- **Breaking:** `et-cli` refuses a scenario whose `module_paths:` entry, or a module directory found under one, has a + name a shell would need quoted or that some filesystem could not hold, such as `my mod`, `mod$(id)` or `con`; rename + the directory. Generated deployments no longer escape these paths. The check is the new `et-sanitise` crate, which + accepts a name only when both sanitize-filename, with its Windows rules, and shlex leave it unchanged. + - The workspace requires actix-web 4.17. - **Breaking:** the minimum supported Rust version is 1.99.0, up from 1.91.1, for every published crate. - **Breaking:** an agent's IP address is always known. `AgentSummary::last_known_ip` and diff --git a/CHECKS.md b/CHECKS.md index 5e08efbc..f334eb93 100644 --- a/CHECKS.md +++ b/CHECKS.md @@ -1,6 +1,6 @@ # Checks -- **Generated:** 2026-10-30 +- **Generated:** 2026-10-31 - **Checks:** 79 - **Tools:** 47 - **Shell-script checks:** 10 diff --git a/Cargo.lock b/Cargo.lock index 91697824..98cf87ab 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1925,11 +1925,11 @@ dependencies = [ [[package]] name = "cranelift-assembler-x64" -version = "0.136.4" +version = "0.136.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c9c587da2696fbaa58e7a2fa2e5bfdd365d0928219f36992304ac1f15e5bc254" +checksum = "249a61efca6a54425e027f5704a6f68a0232b4df527e05a3ef947d89dbdcb078" dependencies = [ - "cranelift-assembler-x64-meta 0.136.4", + "cranelift-assembler-x64-meta 0.136.5", ] [[package]] @@ -1940,9 +1940,9 @@ checksum = "c7496a6e92b5cee48c5d772b0443df58816dee30fed6ba19b2a28e78037ecedf" [[package]] name = "cranelift-assembler-x64-meta" -version = "0.136.4" +version = "0.136.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f28a60027b7556b680b051651291053ec381edf4ffbc01784997ae51a64269bb" +checksum = "d5336a46e05db2d515440b1ce8a9363e7bc04b608c0f66cc98dddc5492a66fbf" dependencies = [ "cranelift-srcgen", ] @@ -1958,11 +1958,11 @@ dependencies = [ [[package]] name = "cranelift-bforest" -version = "0.136.4" +version = "0.136.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "62bfdb846acebfd591d601994ea283c98d525ac0bc9a84ca3f2c36bcf5b69ff1" +checksum = "01b3675b13214270fddbd257d5fbf4109be8b02d92fc636fe96e2829514a7235" dependencies = [ - "cranelift-entity 0.136.4", + "cranelift-entity 0.136.5", "wasmtime-internal-core", ] @@ -1974,9 +1974,9 @@ checksum = "573c641174c40ef31021ae4a5a3ad78974e280633502d0dfc6e362385e0c100f" [[package]] name = "cranelift-bitset" -version = "0.136.4" +version = "0.136.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "aa9f8cc1080739d5f72b4ee7e00aff7d589f1284beaf0a19472c07878be7f299" +checksum = "d7c1d67aa02b6e174bd2b165c113e94af9498fcc20e2892d0f4d5ab8c9618d09" dependencies = [ "serde", "serde_derive", @@ -2010,19 +2010,19 @@ dependencies = [ [[package]] name = "cranelift-codegen" -version = "0.136.4" +version = "0.136.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d85417d493ecec833b5134546b4a252c78cc0bd5be11729a35d25dcff9e4e70b" +checksum = "26fe6f3ec5344c32602c414e986dcbe699fa501a3e3e7b02f887d9d12b14ae42" dependencies = [ "bumpalo", - "cranelift-assembler-x64 0.136.4", - "cranelift-bforest 0.136.4", - "cranelift-bitset 0.136.4", - "cranelift-codegen-meta 0.136.4", - "cranelift-codegen-shared 0.136.4", - "cranelift-control 0.136.4", - "cranelift-entity 0.136.4", - "cranelift-isle 0.136.4", + "cranelift-assembler-x64 0.136.5", + "cranelift-bforest 0.136.5", + "cranelift-bitset 0.136.5", + "cranelift-codegen-meta 0.136.5", + "cranelift-codegen-shared 0.136.5", + "cranelift-control 0.136.5", + "cranelift-entity 0.136.5", + "cranelift-isle 0.136.5", "gimli 0.33.0", "hashbrown 0.17.1", "libm", @@ -2051,12 +2051,12 @@ dependencies = [ [[package]] name = "cranelift-codegen-meta" -version = "0.136.4" +version = "0.136.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d01c8ed0098360bae69ad6b82952d4247516799ce8050d45ad168d6689f4eae6" +checksum = "96c5e6323d458486d95781defdb5016a1176fc911d8bc8e64b696adbda6e9044" dependencies = [ - "cranelift-assembler-x64-meta 0.136.4", - "cranelift-codegen-shared 0.136.4", + "cranelift-assembler-x64-meta 0.136.5", + "cranelift-codegen-shared 0.136.5", "cranelift-srcgen", "heck", "pulley-interpreter", @@ -2070,9 +2070,9 @@ checksum = "0f4ff8d2e1235f2d6e7fc3c6738be6954ba972cd295f09079ebffeca2f864e22" [[package]] name = "cranelift-codegen-shared" -version = "0.136.4" +version = "0.136.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "04f9b86d809cec9c4e88b201f1b3ed3b0347674c9ccfbb8c355228669e34be81" +checksum = "f1260aa93b0bd48e8e55c1c7fd9ff6a260f1a1bc46e1f0151a5055b124e5b15d" [[package]] name = "cranelift-control" @@ -2085,9 +2085,9 @@ dependencies = [ [[package]] name = "cranelift-control" -version = "0.136.4" +version = "0.136.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ce182038fb10b719841e65fdf5deeee258c684f63439f5be680c2be059274b58" +checksum = "50c179632a09850f81f1a1e9116cb09330c734a86689f3ee4c3f4ec81543acbd" dependencies = [ "arbitrary", ] @@ -2103,11 +2103,11 @@ dependencies = [ [[package]] name = "cranelift-entity" -version = "0.136.4" +version = "0.136.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7bca812b03a7b942796391f25bc02aa70a1a007fee3ce2756490c788429e9c39" +checksum = "d8c071188cb1c3b1803e23ae415a8d5f58d741d3a4b26a79b83dd61921112eac" dependencies = [ - "cranelift-bitset 0.136.4", + "cranelift-bitset 0.136.5", "serde", "serde_derive", "wasmtime-internal-core", @@ -2127,11 +2127,11 @@ dependencies = [ [[package]] name = "cranelift-frontend" -version = "0.136.4" +version = "0.136.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a0c30563d2aef95e8d1a00a920199c99117b350339599f0e9487a7a1cf5d59de" +checksum = "e2b85ba0a55d7685117b275fe6f36c819d438862e4223d28f0b97902460ff0ac" dependencies = [ - "cranelift-codegen 0.136.4", + "cranelift-codegen 0.136.5", "hashbrown 0.17.1", "log", "smallvec 1.16.3", @@ -2146,9 +2146,9 @@ checksum = "f900e0a3847d51eed0321f0777947fb852ccfce0da7fb070100357f69a2f37fc" [[package]] name = "cranelift-isle" -version = "0.136.4" +version = "0.136.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9c54184333a275698edb1d3bdff226131a25d47133d792aa716e47e32b8b7f05" +checksum = "20ed364edcaed769da7b2709d540f8f7d0ff257749ba193f475e22dba6a0178b" [[package]] name = "cranelift-module" @@ -2174,20 +2174,20 @@ dependencies = [ [[package]] name = "cranelift-native" -version = "0.136.4" +version = "0.136.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "eb67a4c8a3180b5f1d77dffcd6317c05386ced42e5a31c7004523a9d99dc85c1" +checksum = "418d98c3dce08f0d13526692bbd1888bff4c5f45207f5093d168b5a224fbc88c" dependencies = [ - "cranelift-codegen 0.136.4", + "cranelift-codegen 0.136.5", "libc", "target-lexicon", ] [[package]] name = "cranelift-srcgen" -version = "0.136.4" +version = "0.136.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b533db1679b4b728568bd490da6533cf50974c48a4d7534e78f1e0e02bc39769" +checksum = "4f016b9994e39cad68107f33fa5b8dcee8fb7e38332bde3f97690e5e8abe1b10" [[package]] name = "crc-any" @@ -4986,6 +4986,7 @@ dependencies = [ "et-org", "et-path", "et-recovery-supervisor", + "et-sanitise", "et-test-helpers", "et-ws-test-server", "fake", @@ -5008,6 +5009,16 @@ dependencies = [ "toml", ] +[[package]] +name = "et-firmware-modules" +version = "0.1.0" +dependencies = [ + "fs-err", + "serde_json", + "tempfile", + "thiserror 2.0.21", +] + [[package]] name = "et-hub-backplane" version = "0.2.0" @@ -5101,12 +5112,16 @@ dependencies = [ "embassy-rp", "embassy-time", "embedded-alloc", + "et-firmware-modules", "et-ws-server", "et-ws-service", "fs-err", "panic-probe", "picoserve", + "serde_json", "static_cell", + "tokio", + "tungstenite", ] [[package]] @@ -5213,6 +5228,14 @@ dependencies = [ "web-sys", ] +[[package]] +name = "et-sanitise" +version = "0.1.0" +dependencies = [ + "sanitize-filename", + "shlex 2.0.1", +] + [[package]] name = "et-storage-service" version = "0.2.0" @@ -9901,11 +9924,11 @@ dependencies = [ [[package]] name = "pulley-interpreter" -version = "49.0.4" +version = "49.0.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5cd3de43f4f4cff536de8902097edbdf749321b97dda548f36a37918eb85bba3" +checksum = "1d74df9e20cba0f6dd07f01c3140535a6b23e40aecccfcacfb921621b41d0acb" dependencies = [ - "cranelift-bitset 0.136.4", + "cranelift-bitset 0.136.5", "log", "pulley-macros", "wasmtime-internal-core", @@ -9913,9 +9936,9 @@ dependencies = [ [[package]] name = "pulley-macros" -version = "49.0.4" +version = "49.0.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8319a8fa5269b4895957f8795861c42822bd26218e495f70060a963323d94847" +checksum = "8415513bfa407c06de6f11dbe9437f2352e075a5520ff491a131166d3d90aed1" dependencies = [ "proc-macro2", "quote", @@ -10945,6 +10968,12 @@ dependencies = [ "winapi-util", ] +[[package]] +name = "sanitize-filename" +version = "0.8.0-beta" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "937371a64f83e8adaf269ae1f9b5e6a48891fd1e1ba09f3a5bf0533e4937be42" + [[package]] name = "schannel" version = "0.1.30" @@ -13800,9 +13829,9 @@ dependencies = [ [[package]] name = "wasmtime" -version = "49.0.4" +version = "49.0.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "36abb25de5518707e5ba96b586366b09db168d2d4976cb9dac57265a317497ea" +checksum = "d45bf3b13781b9febb0eef42e1e145146c92ef891f345fd918e0e0fa1568c992" dependencies = [ "addr2line", "async-trait", @@ -13853,15 +13882,15 @@ dependencies = [ [[package]] name = "wasmtime-environ" -version = "49.0.4" +version = "49.0.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "92f16a4d09fb1fbf2dcfeddb9818e1517a258e13b505b3bfdadcf47a19bf31ca" +checksum = "833f00f3dc6128159284c5b71e211f24c97d45e052230a14f767fd830a6fb8b0" dependencies = [ "anyhow", "cpp_demangle", - "cranelift-bforest 0.136.4", - "cranelift-bitset 0.136.4", - "cranelift-entity 0.136.4", + "cranelift-bforest 0.136.5", + "cranelift-bitset 0.136.5", + "cranelift-entity 0.136.5", "gimli 0.33.0", "hashbrown 0.17.1", "indexmap", @@ -13884,9 +13913,9 @@ dependencies = [ [[package]] name = "wasmtime-internal-cache" -version = "49.0.4" +version = "49.0.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "96e4469e885ac50f8dc06337245f9905b22c68d55416267f9afd07f7f76687be" +checksum = "85f1bafc5f2237c5fb6f8babc77b5db2eee8128766b8f1cfe52e771ebbcbed89" dependencies = [ "base64 0.22.1", "directories-next", @@ -13904,9 +13933,9 @@ dependencies = [ [[package]] name = "wasmtime-internal-component-macro" -version = "49.0.4" +version = "49.0.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d4fca03bd4425856574c0807d1dee16fad43f947feaf4e9ae1d2129596ee4de4" +checksum = "0d69c34412409072d2e8ec399101524c330e334ff313f77415e6984cc9c6a172" dependencies = [ "anyhow", "proc-macro2", @@ -13919,15 +13948,15 @@ dependencies = [ [[package]] name = "wasmtime-internal-component-util" -version = "49.0.4" +version = "49.0.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4006009fc3598af4e895ab18c6c33d2c6b7c657b1bd0a755ca7ee1abef6f0c16" +checksum = "b168d39f2d6b896d89fe096afa4ccdd7a8e927b01469b590af84f02fca3126a1" [[package]] name = "wasmtime-internal-core" -version = "49.0.4" +version = "49.0.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "947a732ad659f42c10ec639706fc8cb1f068dd9a55eebde8a1f24dfd06dfa525" +checksum = "7ad86a65d12c6a9202584ceb658c9a20cf5b25c32eed61c9be5e5f863f88f0d9" dependencies = [ "anyhow", "hashbrown 0.17.1", @@ -13937,15 +13966,15 @@ dependencies = [ [[package]] name = "wasmtime-internal-cranelift" -version = "49.0.4" +version = "49.0.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fbfb8b455d524939e03df42e3e749f8e26e26ba2acd8b08fa00b3bda55686750" +checksum = "a71f0fd1dd40e4e18d4a20cdba29ff8e0eb2c0f21b12c2eb5869e0fea789dd05" dependencies = [ - "cranelift-codegen 0.136.4", - "cranelift-control 0.136.4", - "cranelift-entity 0.136.4", - "cranelift-frontend 0.136.4", - "cranelift-native 0.136.4", + "cranelift-codegen 0.136.5", + "cranelift-control 0.136.5", + "cranelift-entity 0.136.5", + "cranelift-frontend 0.136.5", + "cranelift-native 0.136.5", "gimli 0.33.0", "itertools 0.14.0", "log", @@ -13963,9 +13992,9 @@ dependencies = [ [[package]] name = "wasmtime-internal-fiber" -version = "49.0.4" +version = "49.0.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bd8aa8711f68b2cdd94c2b85ba5c8317edaa1ac4ee0a0534b957123680239307" +checksum = "fa4dfa39e58735b24ce3d28bfc8c2b3041603cc8a941bb268395351beff90172" dependencies = [ "cc", "libc", @@ -13977,9 +14006,9 @@ dependencies = [ [[package]] name = "wasmtime-internal-jit-debug" -version = "49.0.4" +version = "49.0.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e873e4504faf50aa8ce255d6ea7210d87b39d5f379c2486782e2c5f179c70ca8" +checksum = "f2eb98d4486c7b0163447d77dea06f3cc87e5e5ba5fc93613f8ec42e27ea181d" dependencies = [ "cc", "object 0.40.0", @@ -13989,9 +14018,9 @@ dependencies = [ [[package]] name = "wasmtime-internal-jit-icache-coherence" -version = "49.0.4" +version = "49.0.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "197ab1fda5da958a5c0b8a97eb241beafc48ac568b82ef4f12f38d6866c74173" +checksum = "852cae234735d79db6c58fe0d04b32f4078ad01fdb3fd6d488ac366132ba9785" dependencies = [ "libc", "wasmtime-internal-core", @@ -14000,11 +14029,11 @@ dependencies = [ [[package]] name = "wasmtime-internal-unwinder" -version = "49.0.4" +version = "49.0.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5577f3a5a979a7c3b1b23c1e9f13e70a8583acc8828471ec56f282117fc4952b" +checksum = "b9289378c82b9db7e7cf933d6bb441ad968536ac48f3f38c55b7cf9d3e8d87e1" dependencies = [ - "cranelift-codegen 0.136.4", + "cranelift-codegen 0.136.5", "log", "object 0.40.0", "wasmtime-environ", @@ -14012,9 +14041,9 @@ dependencies = [ [[package]] name = "wasmtime-internal-versioned-export-macros" -version = "49.0.4" +version = "49.0.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f6399c9c4050ed431cf66e21c05fadd6f9731db44476f452f8c7142b4597b883" +checksum = "cc5c60c732d99dbcc5d670b71ba6ccf2b015cd16fc828990eaad2c4b8805ebdb" dependencies = [ "proc-macro2", "quote", @@ -14023,9 +14052,9 @@ dependencies = [ [[package]] name = "wasmtime-internal-wit-bindgen" -version = "49.0.4" +version = "49.0.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b4eaf45d4da7681d234bb6efd846b4a528b08a75384a7fc23f125d0145364b06" +checksum = "c77e4fcc7cd22772578458c5e233df47190ebc9b8b65788797d99a02787c9000" dependencies = [ "anyhow", "bitflags 2.13.2", @@ -14037,9 +14066,9 @@ dependencies = [ [[package]] name = "wasmtime-wasi" -version = "49.0.4" +version = "49.0.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "821e45765c35555d3b0bdf9277cb99478f7d1f29a17b3315ba76e01220f02a38" +checksum = "705b90d1fa1084cc73df656868948da77711b97f6b895d145a4949f86bcea674" dependencies = [ "async-trait", "bitflags 2.13.2", @@ -14062,9 +14091,9 @@ dependencies = [ [[package]] name = "wasmtime-wasi-io" -version = "49.0.4" +version = "49.0.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8cdff38d1c730bdaf4fcfbe0eedf3265fb9c7a4deb119deecbfc1c1778950716" +checksum = "f3d2b6a255be488639f450b380e7e0923789f946de3463696cd30f2fe8a65cc3" dependencies = [ "async-trait", "bytes", @@ -14075,9 +14104,9 @@ dependencies = [ [[package]] name = "wasmtime-wasi-nn" -version = "49.0.4" +version = "49.0.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "01dad9a43e8dc1e9a4c7b073faf5d7814d926d1939ee7d27c976288eedefd21e" +checksum = "f317f2edc9e3079b160f8a72a8b3b77a2c5fc443d64c588a82f5b845b977240a" dependencies = [ "ort", "thiserror 2.0.21", @@ -14389,9 +14418,9 @@ checksum = "72069c3113ab32ab29e5584db3c6ec55d416895e60715417b5b883a357c3e471" [[package]] name = "wiggle" -version = "49.0.4" +version = "49.0.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "af3872869836d404f722e2361f869c09630e43a73fcc7060f7892a929b74b6a3" +checksum = "f1ef4f7720340c50b9d28718516f7ab881a169b9021268e7dc67729435d9ccfd" dependencies = [ "bitflags 2.13.2", "thiserror 2.0.21", @@ -14403,9 +14432,9 @@ dependencies = [ [[package]] name = "wiggle-generate" -version = "49.0.4" +version = "49.0.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cf7e31dec6de37e7e848aecbad54777dd246cd440a1a547ab1a1e31c27f55e49" +checksum = "1692e27a88a6fa5bacd69a645e982a439c5ae835d9180394992627b0124cb705" dependencies = [ "heck", "proc-macro2", @@ -14417,9 +14446,9 @@ dependencies = [ [[package]] name = "wiggle-macro" -version = "49.0.4" +version = "49.0.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a2af323f33b28ea84b24e2bccef19e51c7a75aa1d2929280ad1a4865f2877dcc" +checksum = "562354c56fa0b83911512f0b60baf33c824e431de67a6012939ce52d224713b9" dependencies = [ "proc-macro2", "quote", diff --git a/Cargo.toml b/Cargo.toml index b5caedd2..18b54b28 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -11,6 +11,7 @@ members = [ "libs/et-otlp", "libs/org", "libs/path", + "libs/sanitise", "libs/test-helpers", "libs/test-nats", "libs/test-otlp", @@ -39,6 +40,7 @@ members = [ "services/ws-modules/wasi-data1", "services/ws-modules/wasi-math1", "services/ws-modules/wasi-math1-sender", + "services/firmware-modules", "services/hub-backplane", "services/modules", "services/storage", @@ -110,6 +112,7 @@ embassy-time = "0.5" embedded-alloc = { version = "0.7", default-features = false, features = ["llff"] } et-actix-web-config = { path = "libs/actix-web-config", version = "0.1.0" } et-cli = { path = "utilities/cli", version = "0.2.0" } +et-firmware-modules = { path = "services/firmware-modules", version = "0.1.0" } et-hub-backplane = { path = "services/hub-backplane", version = "0.2.0" } et-modules-service = { path = "services/modules", version = "0.3.0" } et-onnx = { path = "utilities/onnx", version = "0.2.0" } @@ -118,6 +121,7 @@ et-otlp = { path = "libs/et-otlp", version = "0.3.0" } et-path = { path = "libs/path", version = "0.3.0" } et-recovery-supervisor = { path = "services/recovery-supervisor", version = "0.1.0" } et-rest-client = { path = "generated/rust-rest", version = "0.2.0", default-features = false } +et-sanitise = { path = "libs/sanitise", version = "0.1.0" } et-storage-service = { path = "services/storage", version = "0.2.0", default-features = false } et-test-helpers = { path = "libs/test-helpers", version = "0.2.0" } et-test-nats = { path = "libs/test-nats", version = "0.2.0" } @@ -206,6 +210,7 @@ opentelemetry-proto = { version = "0.32", default-features = false, features = [ "with-serde", ] } opentelemetry_sdk = "0.32" +# Pinned exactly, because rc.11 and later moved API that wasmtime-wasi-nn calls. ort = { version = "=2.0.0-rc.10", default-features = false, features = ["copy-dylibs"] } panic-probe = "1" picoserve = { version = "0.20", default-features = false, features = ["ws"] } @@ -238,6 +243,7 @@ retry = { version = "2", default-features = false } retry-policies = "0.5" rstest = "0.28" rustls = "0.23" +sanitize-filename = "0.8.0-beta" schemars = { version = "1.2", features = ["derive"] } secrecy = { version = "0.10", features = ["serde"] } semver = "1" @@ -250,6 +256,7 @@ serde_path_to_error = "0.1" serde_with = { version = "3.24", default-features = false, features = ["macros"] } serde_yaml = "0.9" sha2 = { version = "0.11", default-features = false } +shlex = "2" spin = { version = "0.10", default-features = false, features = ["spin_mutex"] } static_cell = "2" strum = { version = "0.28", features = ["derive"] } @@ -276,6 +283,7 @@ tracing-subscriber = { version = "0.3", features = ["env-filter"] } tracing-wasm = { package = "wasm-tracing", version = "2.1" } tree-sitter = "0.27" tree-sitter-zig = "1" +tungstenite = { version = "0.30", default-features = false, features = ["handshake"] } # Already resolved transitively (object_store, reqwest, ...), so this adds no new crate or license to review. # object_store's parse_url_opts takes a `&Url` and does not re-export the type, so a backend-URL config needs it. url = "2" @@ -286,17 +294,18 @@ wasm-bindgen = "0.2" wasm-bindgen-futures = "0.4" wasm-bindgen-test = "0.3" # Keep all three on the same major for ABI compat -- wasmtime-wasi-nn tracks wasmtime's version exactly. -# 49.0.4 is the floor, not a preference: RUSTSEC-2026-0352 through RUSTSEC-2026-0364 are fixed only from 49.0.3, -# and RUSTSEC-2026-0378 / RUSTSEC-2026-0379 only from 49.0.4, on the 49 line. The major is held at 49 by -# wasi-webgpu-wasmtime, whose 0.4 line requires wasmtime ^49. -wasmtime = { version = "49.0.4", features = ["async", "component-model"] } +# 49.0.5 is the floor, not a preference: RUSTSEC-2026-0352 through RUSTSEC-2026-0364 are fixed only from 49.0.3, +# RUSTSEC-2026-0378 / RUSTSEC-2026-0379 only from 49.0.4, and RUSTSEC-2026-0420 and RUSTSEC-2026-0422 through +# RUSTSEC-2026-0424 only from 49.0.5, on the 49 line. The major is held at 49 by wasi-webgpu-wasmtime, whose 0.4 +# line requires wasmtime ^49. +wasmtime = { version = "49.0.5", features = ["async", "component-model"] } # The generator behind wasmtime's `bindgen!` macro, driven directly by et-int-gen. # Emitting the runner's host bindings as a checked-in source file keeps ws-wasi-runner self-contained: the # macro read WIT from `../../generated/specs/wit`, which `cargo package` cannot include, so the crate failed # to build from its own tarball. Named `internal` upstream, so treat its API as unstable and pinned to the # wasmtime version above; the drift check catches any change in what it emits. -wasmtime-internal-wit-bindgen = { version = "49.0.4", features = ["async", "component-model-async"] } -wasmtime-wasi = "49.0.4" +wasmtime-internal-wit-bindgen = { version = "49.0.5", features = ["async", "component-model-async"] } +wasmtime-wasi = "49.0.5" wasmtime-wasi-nn = { version = "49", default-features = false, features = ["onnx"] } web-sys = "0.3" # wgpu-core/wgpu-types come in via wasi-webgpu-wasmtime's re-exports; only the instance handle is ours. diff --git a/README.md b/README.md index 110bb735..30175b5a 100644 --- a/README.md +++ b/README.md @@ -425,6 +425,26 @@ non-root user, with `emptyDir` mounts for the paths that still have to be writab plaintext -- `ws://` to the hub, `http://` to the collector -- because the hub's self-signed certificate is issued for `localhost` only and the runners have no way to be told to trust it. +### Firmware + +The `firmware-` output types deploy the hub alone, as firmware for a networked microcontroller board that +serves the scenario's modules itself. Each board has one: `firmware-pico2` builds for the Raspberry Pi Pico 2 W, over +its own Wi-Fi. It writes a `mise.firmware-pico2.toml` beside the other formats, and its `firmware-flash` task builds +the firmware and loads it onto a board in BOOTSEL mode: + +```bash +input=verification/local/input/facility-security-scenario.yaml +output=verification/local/output/facility-security-scenario +et-cli generate-deployment --input-file "$input" --output-dir "$output" --output-type firmware-pico2 +cd "$output" +MISE_ENV=firmware-pico2 mise run firmware-flash +``` + +A board has no filesystem, so each module from this repository, or from the scenario's `module_paths:`, is built +into the firmware from its built package, and the modules have to be built first. The npm packages they use, such as +onnxruntime-web, are bigger than a board's flash: those are redirected to jsDelivr instead. A scenario with a runner, +more than one machine or published artifacts is refused, since the board runs nothing but the hub. + ### Multiple machines A scenario runs on one host unless it lists `machines:`. Listing them is opt-in: the first machine is the server, diff --git a/config/conftest/policy/cargo/cargo.rego b/config/conftest/policy/cargo/cargo.rego index 1940132c..6bb2b68c 100644 --- a/config/conftest/policy/cargo/cargo.rego +++ b/config/conftest/policy/cargo/cargo.rego @@ -236,20 +236,24 @@ deny contains msg if { # # Path deps are exempt by construction rather than by entry: cargo wants their `version` to equal the member # crate's own `package.version` -- the rule above enforces exactly that -- which is always a full triple. +# A prerelease is exempt the same way: cargo matches a prerelease only through a requirement that names its whole +# version, `0.8.0-beta` included, so there is no major.minor form for it to be trimmed back to. patch_pin_exception := { "deno_error": "exact pin: JsErrorBox must be the type deno_core holds, so a 0.7.2 would be a second copy", "minicov": "exact pin: wasm-bindgen-test 0.3.79 requires =0.3.8, and cargo holds one 0.3.x copy for both", "mock-collector": "exact pin: 0.2.15 moved to opentelemetry 0.33, which tracing-actix-web cannot follow yet", - "ort": "exact pin on a prerelease, which has no two-part form; rc.11+ moved API wasmtime-wasi-nn calls", - "wasmtime": "49.0.3 is the security floor; RUSTSEC-2026-0352..0364 have no fix below it in the 49 line", - "wasmtime-internal-wit-bindgen": "49.0.3 tracks the wasmtime release; the crate is internal and its API can move", - "wasmtime-wasi": "49.0.3 carries the same security floor as the wasmtime entry it ships beside", + "wasmtime": "49.0.5 is the security floor; RUSTSEC-2026-0424 and others have no fix below it in the 49 line", + "wasmtime-internal-wit-bindgen": "49.0.5 tracks the wasmtime release; the crate is internal and its API can move", + "wasmtime-wasi": "49.0.5 carries the same security floor as the wasmtime entry it ships beside", } # Matches the requirement's leading version token rather than splitting the whole string on dots. -# That keeps a comparator (`=`, `>=`, `~`) and a prerelease suffix (`-rc.10`, whose dot is its own) out of the -# component count, so only the version core decides. -patch_pinned(req) if regex.match(`^[^0-9]*[0-9]+\.[0-9]+\.[0-9]+`, req) +# That keeps a comparator (`=`, `>=`, `~`) out of the component count, so only the version core decides, and a +# prerelease suffix (`-rc.10`) marks the exempt case rather than adding dots of its own. +patch_pinned(req) if { + regex.match(`^[^0-9]*[0-9]+\.[0-9]+\.[0-9]+`, req) + not regex.match(`^[^0-9]*[0-9]+\.[0-9]+\.[0-9]+-`, req) +} requirement(spec) := spec if is_string(spec) diff --git a/config/conftest/policy/cargo/cargo_test.rego b/config/conftest/policy/cargo/cargo_test.rego index 77b86d30..1d789c3a 100644 --- a/config/conftest/policy/cargo/cargo_test.rego +++ b/config/conftest/policy/cargo/cargo_test.rego @@ -56,10 +56,11 @@ test_comparator_on_major_minor_is_accepted if { not flagged(msgs, "crate-a") } -# A prerelease tail contributes dots of its own, which belong to the tail rather than to the version core. -test_prerelease_counts_only_the_version_core if { - msgs := cargo.deny with input as manifest({"crate-a": "=2.0.0-rc.10"}) - flagged(msgs, "crate-a") +# A prerelease has no two-part form, so it keeps its full version without needing an exception entry. +test_prerelease_keeps_its_full_version if { + msgs := cargo.deny with input as manifest({"crate-a": "=2.0.0-rc.10", "crate-b": "0.8.0-beta"}) + not flagged(msgs, "crate-a") + not flagged(msgs, "crate-b") } # A path dep has no two-part form available to it, so it is exempt without needing an exception entry. @@ -70,31 +71,37 @@ test_path_dep_keeps_its_full_triple if { # An excepted dep keeps its pin, and the entry that permits it is what stops the denial. test_excepted_dep_keeps_its_patch_pin if { - msgs := cargo.deny with input as manifest({"deno_error": "=0.7.1", "ort": "=2.0.0-rc.10"}) + msgs := cargo.deny with input as manifest({"deno_error": "=0.7.1", "minicov": "=0.3.8"}) not flagged(msgs, "deno_error") } test_exception_only_covers_the_dep_it_names if { - deps := {"deno_error": "=0.7.1", "ort": "=2.0.0-rc.10", "other": "1.2.3"} + deps := {"deno_error": "=0.7.1", "minicov": "=0.3.8", "other": "1.2.3"} msgs := cargo.deny with input as manifest(deps) flagged(msgs, "other") } -# The map is only honest while every entry still describes the manifest, so both ways of going stale report. +# The map is only honest while every entry still describes the manifest, so every way of going stale reports. test_trimmed_dep_reports_its_stale_exception if { - msgs := cargo.deny with input as manifest({"deno_error": "=0.7.1", "ort": "2.0"}) - stale(msgs, "ort") + msgs := cargo.deny with input as manifest({"deno_error": "=0.7.1", "minicov": "0.3"}) + stale(msgs, "minicov") } test_dropped_dep_reports_its_stale_exception if { msgs := cargo.deny with input as manifest({"deno_error": "=0.7.1"}) - stale(msgs, "ort") + stale(msgs, "minicov") +} + +# A prerelease needs no entry, so one written for it describes nothing the rule would deny. +test_prerelease_dep_reports_its_stale_exception if { + msgs := cargo.deny with input as manifest({"deno_error": "=0.7.1", "minicov": "=0.3.8-rc.1"}) + stale(msgs, "minicov") } test_live_exceptions_report_nothing if { - msgs := cargo.deny with input as manifest({"deno_error": "=0.7.1", "ort": "=2.0.0-rc.10"}) + msgs := cargo.deny with input as manifest({"deno_error": "=0.7.1", "minicov": "=0.3.8"}) not stale(msgs, "deno_error") - not stale(msgs, "ort") + not stale(msgs, "minicov") } tracing_log_drift(msgs) if { diff --git a/config/deny.toml b/config/deny.toml index 56a95f08..9fb743e7 100644 --- a/config/deny.toml +++ b/config/deny.toml @@ -30,17 +30,17 @@ ignore = [ # newest release. Unmaintained != vulnerable. The advisory covers every version of the crate, 2.x and 3.0.0 # included, so only dropping bincode clears it -- which deno_core did at 0.411.0. deno_runtime 0.267.0 is the # first release built on it (deno_core ^0.412.0); drop this entry with the deno_runtime bump that reaches it. - { id = "RUSTSEC-2025-0141", reason = "expires 2026-10-31" }, + { id = "RUSTSEC-2025-0141", reason = "expires 2026-11-13" }, # paste 1.0.15 (unmaintained). # Proc-macro pulled by v8 (rusty_v8) via deno_core / serde_v8; expands at compile time only, so there is no # runtime exposure. Every published v8 release through 152.2.0 declares it as a non-optional dependency, so no # upgrade clears this -- only a rusty_v8 move to the maintained pastey fork. Awaiting that migration. - { id = "RUSTSEC-2024-0436", reason = "expires 2026-10-31" }, + { id = "RUSTSEC-2024-0436", reason = "expires 2026-11-13" }, # rustls-pemfile 2.2.0 (unmaintained; final release -- parsing moved into rustls-pki-types). # PEM cert parser for loading the system CA bundle. Unmaintained, not vulnerable. Pulled directly by both # deno_native_certs and deno_tls, each at its newest release, so no bump drops it. rustls-native-certs # migrated at 0.8.1 and our 0.8.4 copy is clean; the affected 0.7.3 copy is pinned by deno_native_certs. - { id = "RUSTSEC-2025-0134", reason = "expires 2026-10-31" }, + { id = "RUSTSEC-2025-0134", reason = "expires 2026-11-13" }, # rustls 0.23.40 -- the four rustls entries below share one blocker, so it is described once, here. # deno_tls pins `rustls = "=0.23.40"` by equality through 0.246.0, its newest release, and because a lockfile # holds one rustls 0.23.x that pin also holds ws-server's own TLS listener and every reqwest client at @@ -53,57 +53,57 @@ ignore = [ # for example a plaintext `EncryptedExtensions` packed into the same record as the `ServerHello`. The transcript # is still authenticated, so an attacker cannot alter or complete a handshake with it. CVSS 5.3, client and # server roles alike, and live code here. - { id = "RUSTSEC-2026-0285", reason = "expires 2026-10-31" }, + { id = "RUSTSEC-2026-0285", reason = "expires 2026-11-13" }, # `Debug` on client session values prints the TLS 1.2 master secret or TLS 1.3 resumption PSK, fixed 0.23.46. # Reached only through a custom `ClientSessionStore` that formats those values; nothing here implements one, and # the built-in session cache's `Debug` output is redacted. - { id = "RUSTSEC-2026-0346", reason = "expires 2026-10-31" }, + { id = "RUSTSEC-2026-0346", reason = "expires 2026-11-13" }, # Brotli certificate decompression can allocate 1 GiB before authentication, fixed in 0.23.46. # Needs rustls's non-default `brotli` feature, which our resolution does not enable -- `cargo tree -i rustls -e # features` reports aws-lc-rs,default,log,logging,prefer-post-quantum,std,tls12 -- so the decompressor is absent. - { id = "RUSTSEC-2026-0348", reason = "expires 2026-10-31" }, + { id = "RUSTSEC-2026-0348", reason = "expires 2026-11-13" }, # An unauthenticated ServerHello can steer later client connections to classical-only key exchange, fixed 0.23.46. # Client role, and live here: `prefer-post-quantum` is on, so the downgrade is possible. The cost is losing # post-quantum confidentiality on the affected connections, not a key compromise or an authentication bypass, # and it needs a network attacker who can answer one connection attempt to the same server name. - { id = "RUSTSEC-2026-0349", reason = "expires 2026-10-31" }, + { id = "RUSTSEC-2026-0349", reason = "expires 2026-11-13" }, # webpki-roots / webpki-root-certs 0.26.11 bundle the distrusted emSign ECC Root CA - G3, fixed only in 1.0.10. # Both 0.26.11 releases are thin re-export shims over `^1` of the same crate, and the lockfile resolves that to # 1.0.10, so the roots they hand out are the patched set; the advisories match on the shim's own version # number. deno_tls (`webpki-roots = "^0.26"`) and deno_node (`webpki-root-certs = "^0.26.5"`) require the shims # through their newest releases, 0.246.0 and 0.197.0, so no bump drops them. Drop both when Deno moves to ^1. - { id = "RUSTSEC-2026-0381", reason = "expires 2026-10-31" }, - { id = "RUSTSEC-2026-0383", reason = "expires 2026-10-31" }, + { id = "RUSTSEC-2026-0381", reason = "expires 2026-11-13" }, + { id = "RUSTSEC-2026-0383", reason = "expires 2026-11-13" }, # smartstring 1.0.1 (unmaintained; repo archived 2026-05-03). # Pulled only by the swc stack (swc_ecma_lexer/parser 26/27) under deno_ast -> deno_runtime. Maintenance # notice, not a vulnerability. Upstream swc migrated to compact_str at swc_ecma_lexer 41.0.0 and # swc_ecma_parser 42.0.0, but deno_ast pins both by equality (=26.0.0 / =27.0.7) through its newest release, # so no bump on our side reaches them. Drop when a deno_runtime/deno_ast bump pulls the migrated swc. - { id = "RUSTSEC-2026-0249", reason = "expires 2026-10-31" }, + { id = "RUSTSEC-2026-0249", reason = "expires 2026-11-13" }, # hickory-proto 0.25.2 NSEC3 closest-encloser proof unbounded loop (DoS on DNSSEC validation). # Pulled by deno_net for resolver. The advisory scopes reachability to builds carrying dnssec-ring or # dnssec-aws-lc-rs, and our resolution enables neither -- cargo tree -i hickory-proto -f "{p} [{f}]" reports # default,futures-io,serde,std,tokio -- so DnssecDnsHandle is never compiled in. (Cargo.lock listing ring # under hickory-proto is not evidence otherwise; it records optional deps regardless of activation.) The fix # landed in 0.26.0-beta.1, but deno_net requires ^0.25.2 through its newest release, so it is out of reach. - { id = "RUSTSEC-2026-0118", reason = "expires 2026-10-31" }, + { id = "RUSTSEC-2026-0118", reason = "expires 2026-11-13" }, # hickory-proto 0.25.2 O(n^2) name compression during message encode. # Encoder side, hit only when we *send* a DNS message. The runner uses the OS resolver via hickory. It # never crafts outbound DNS frames with attacker-controlled name lists. - { id = "RUSTSEC-2026-0119", reason = "expires 2026-10-31" }, + { id = "RUSTSEC-2026-0119", reason = "expires 2026-11-13" }, # hickory-resolver 0.25.2 lookup() / lookup_ip() return Ok for a response DNSSEC validation found bogus. # Pulled by deno_net and deno_fetch for the web runner's resolver. The advisory needs a resolver with DNSSEC # validation enabled, which needs a dnssec-* feature, and our resolution enables none -- cargo tree -i # hickory-resolver -e features reports default,serde,system-config,tokio -- so no response is ever validated and # none can be passed off as valid. The fix is 0.26.2, but deno_net and deno_fetch require ^0.25.2 through their # newest releases (0.251.0 / 0.283.0), so it is out of reach. - { id = "GHSA-5j98-2g5x-46v6", reason = "expires 2026-10-31" }, + { id = "GHSA-5j98-2g5x-46v6", reason = "expires 2026-11-13" }, # hickory-resolver 0.25.2 follows CNAME records outside the answer's chain, sending queries it does not need. # Same deno_net / deno_fetch path and the same unreachable 0.26.2 fix as the entry above. Unlike that one, no # feature gates it, so the code is live: a response carrying stray CNAMEs costs extra queries. The cost is query # amplification, not a wrong answer, and the runner resolves only the ws-server host it is configured with, so # the responses come from the operator's own DNS rather than one an attacker chooses. - { id = "GHSA-6f2x-v7q7-m7m5", reason = "expires 2026-10-31" }, + { id = "GHSA-6f2x-v7q7-m7m5", reason = "expires 2026-11-13" }, # rand 0.8.5 unsound ThreadRng reseed when a custom `log` logger calls rand::rng() (UB via aliased &mut). # Transitive via deno_crypto / deno_fs / deno_node (web-runner), used for non-cryptographic randomness # (jitter, salt scratchpads). The advisory's first precondition is the `log` feature, which our resolution @@ -111,7 +111,7 @@ ignore = [ # Fixed in the 0.8 line at 0.8.6, which deleted the log optional dep -- but deno_crypto pins rand "=0.8.5" # by equality through its newest release, so neither a lockfile bump nor a deno bump moves it. cargo-deny # does not match this id and warns advisory-not-detected; the entry stays because osv-scanner does flag it. - { id = "RUSTSEC-2026-0097", reason = "expires 2026-10-31" }, + { id = "RUSTSEC-2026-0097", reason = "expires 2026-11-13" }, # rsa 0.9.10 Marvin Attack timing sidechannel on RSA decryption. # No patched version exists -- the advisory lists none, upstream is still migrating to a constant-time # implementation, and 0.9.10 is the newest stable (the 0.10 line is release candidates, equally affected), @@ -119,7 +119,7 @@ ignore = [ # RSA keys or decrypt RSA payloads. Only TLS server cert verification, which uses *signature* checks, no # decryption. Not reachable unless guest JS explicitly imports crypto.subtle and performs RSA-OAEP decrypt # with a private key. Our modules don't. - { id = "RUSTSEC-2023-0071", reason = "expires 2026-10-31" }, + { id = "RUSTSEC-2023-0071", reason = "expires 2026-11-13" }, # crypto-bigint 0.5.5 secret-dependent branches in operations meant to be constant-time (timing side channel). # The arithmetic under elliptic-curve 0.13, which deno_crypto and deno_node_crypto use for P-256/384/521 and # secp256k1 in WebCrypto and node:crypto. The fix is only in 0.7.6 and later, and deno_crypto requires @@ -128,7 +128,7 @@ ignore = [ # those operations on the same host. Our modules do neither; the hub's TLS uses rustls and aws-lc, not this. Drop # when deno_crypto moves to elliptic-curve 0.14. cargo-deny ignores by id only, so this also mutes the 0.7 copy # elliptic-curve 0.14 pulls -- Cargo.lock pins the patched 0.7.10 there, so re-check that copy when touching this. - { id = "RUSTSEC-2026-0387", reason = "expires 2026-10-31" }, + { id = "RUSTSEC-2026-0387", reason = "expires 2026-11-13" }, # h2 0.3.27 unbounded empty DATA frames (low-severity HTTP/2 DoS: undrained streams grow memory). # Accepted for the 0.3 copy only. hyperium patched the 0.4 line (>= 0.4.16, which Cargo.lock now pins for the # hyper/reqwest path) and shipped no 0.3 backport, so 0.3.27 is the last release of a dead line. It arrives @@ -138,14 +138,14 @@ ignore = [ # this when it ships one built on h2 0.4. cargo-deny 0.20 ignores by id only (no version scoping), so this # also mutes the 0.4 line -- Cargo.lock pins 0.4.16 there, and a bump below that would have to come back # through this entry, so re-check both copies when touching it. - { id = "RUSTSEC-2026-0258", reason = "expires 2026-10-31" }, + { id = "RUSTSEC-2026-0258", reason = "expires 2026-11-13" }, # unsafe-libyaml 0.2.11 (unmaintained; repository archived 2024-03-25, so no version is patched). # A c2rust translation of libyaml, reached through serde_yaml, itself deprecated. Two things pull serde_yaml: # our own crates, for registry, scenario and generated-deployment YAML, and progenitor-macro, which parses the # OpenAPI document at compile time. Unmaintained, not vulnerable. Moving our crates to a safe-Rust parser # (serde-saphyr is the advisory's suggestion) is in our hands, but progenitor-macro would still carry it, so # drop this only once both have moved. - { id = "RUSTSEC-2024-0449", reason = "expires 2026-10-31" }, + { id = "RUSTSEC-2024-0449", reason = "expires 2026-11-13" }, # zstd 0.13.2 / zstd-safe 7.3.0 -- the zstd entries below share one blocker, so it is described once, here. # The fixes are only in zstd 0.14.3 and zstd-safe 8.2.0, and Cargo.lock pins those for every consumer that # accepts them (actix-http, compression-codecs, tonic). Two cannot: deno_node pins `zstd = "=0.13.2"` by equality @@ -157,42 +157,42 @@ ignore = [ # Leftover input from an abandoned stream leaks into the next one on a reused context, fixed in zstd 0.14.3. # Reached only through the `with_context` stream constructors. deno_node's node:zlib ops build their raw # encoders and decoders with `new`, and wasmtime-internal-cache calls only `encode_all` / `decode_all`. - { id = "RUSTSEC-2026-0391", reason = "expires 2026-10-31" }, + { id = "RUSTSEC-2026-0391", reason = "expires 2026-11-13" }, # Three memory-safety flaws in `Seekable`, all fixed in 8.2.0 and all in the `seekable` feature, which is off. # `decompress` sets a `Vec` length past its capacity for an offset past the end, an uninitialized or failed # `Seekable` reads a NULL seek table, and `seekable_compressStream` reads past an `InBuffer` whose `pos` is out # of range. - { id = "RUSTSEC-2026-0396", reason = "expires 2026-10-31" }, - { id = "RUSTSEC-2026-0398", reason = "expires 2026-10-31" }, - { id = "RUSTSEC-2026-0405", reason = "expires 2026-10-31" }, + { id = "RUSTSEC-2026-0396", reason = "expires 2026-11-13" }, + { id = "RUSTSEC-2026-0398", reason = "expires 2026-11-13" }, + { id = "RUSTSEC-2026-0405", reason = "expires 2026-11-13" }, # Three memory-safety flaws in `experimental` APIs, all fixed in 8.2.0; that feature is off, so none is compiled in. # `CParameter::StableInBuffer` compresses freed input, `init_using_cdict` / `init_using_ddict` outlive their # dictionary, and `DCtx::try_clone` double-frees one. - { id = "RUSTSEC-2026-0395", reason = "expires 2026-10-31" }, - { id = "RUSTSEC-2026-0399", reason = "expires 2026-10-31" }, - { id = "RUSTSEC-2026-0404", reason = "expires 2026-10-31" }, + { id = "RUSTSEC-2026-0395", reason = "expires 2026-11-13" }, + { id = "RUSTSEC-2026-0399", reason = "expires 2026-11-13" }, + { id = "RUSTSEC-2026-0404", reason = "expires 2026-11-13" }, # Two memory-safety flaws in multithreaded compression, both fixed in 8.2.0 and both needing `zstdmt` (off). # `CCtx::try_clone` overflows the heap after a multithreaded frame, and an unfinished one lets a reset free a # dictionary or job table a worker still uses. deno_node does pass guest JS's `NbWorkers` through, but without # `zstdmt` the bundled library has no worker threads, so no multithreaded frame is ever started. - { id = "RUSTSEC-2026-0402", reason = "expires 2026-10-31" }, - { id = "RUSTSEC-2026-0406", reason = "expires 2026-10-31" }, + { id = "RUSTSEC-2026-0402", reason = "expires 2026-11-13" }, + { id = "RUSTSEC-2026-0406", reason = "expires 2026-11-13" }, # `OutBuffer::around_pos` marks a `Vec` initialized up to a position past its length, fixed in 8.2.0. # deno_node wraps plain slices with `OutBuffer::around`, and wasmtime-internal-cache calls only # `encode_all` / `decode_all`, so neither consumer ever builds an `OutBuffer` over a `Vec` with a position. - { id = "RUSTSEC-2026-0397", reason = "expires 2026-10-31" }, + { id = "RUSTSEC-2026-0397", reason = "expires 2026-11-13" }, # A one-shot call on a context with an unfinished stream leaves it pointing at a freed dictionary, fixed in 8.2.0. # Needs one context used for both streaming and one-shot calls. deno_node's node:zlib ops only stream, on a # context per JS stream, and wasmtime-internal-cache only makes one-shot calls, each on a context of its own. - { id = "RUSTSEC-2026-0400", reason = "expires 2026-10-31" }, + { id = "RUSTSEC-2026-0400", reason = "expires 2026-11-13" }, # `train_from_buffer` reads past its samples buffer when the sample sizes overflow, fixed in 8.2.0. # `zdict_builder` is on, but neither consumer trains a dictionary, so the trainer is compiled in and never called. - { id = "RUSTSEC-2026-0401", reason = "expires 2026-10-31" }, + { id = "RUSTSEC-2026-0401", reason = "expires 2026-11-13" }, # A failed streaming decompression of a legacy (v0.4 to v0.7) frame exposes uninitialized memory, fixed in 8.2.0. # Live: `legacy` is on, and deno_node's raw decoder runs `decompress_stream`. It is reached only when guest JS in # the web runner decompresses zstd through node:zlib, which our modules never do. wasmtime-internal-cache # decompresses only its own compilation cache, and ws-wasi-runner configures none, so that path never runs. - { id = "RUSTSEC-2026-0403", reason = "expires 2026-10-31" }, + { id = "RUSTSEC-2026-0403", reason = "expires 2026-11-13" }, # block-buffer 0.10.4: a panic caught inside the compress callback leaves the buffer's cursor corrupted. # Fixed only in 0.12.1, and the 0.10 line is pulled by digest 0.10, which the RustCrypto hashes under deno_crypto, # deno_node_crypto and curve25519-dalek require through their newest releases, so no bump reaches it; Cargo.lock @@ -200,18 +200,18 @@ ignore = [ # that copy. Exploiting it needs the callback to panic, the panic to be caught, and the same buffer to be used # again. The callbacks are the hashes' fixed-size block functions, which do not panic, and nothing here catches # a panic mid-hash and keeps the hasher. Drop this when digest 0.10 leaves the tree. - { id = "GHSA-qwgh-2vcv-g2f7", reason = "expires 2026-10-31" }, + { id = "GHSA-qwgh-2vcv-g2f7", reason = "expires 2026-11-13" }, # bare-metal 0.2.5 (deprecated and archived, so no version is patched). # Pulled only by the Pico 2 W firmware, through cortex-m 0.7, which cyw43 0.7 and embassy-rp 0.10 require through # their newest releases. Deprecated != vulnerable: cortex-m uses it for its single-core `Mutex` and # `CriticalSection` types, and the firmware takes its critical sections from embassy-rp's `critical-section` # implementation. Drop this when cortex-m 0.8 reaches them. - { id = "RUSTSEC-2026-0110", reason = "expires 2026-10-31" }, + { id = "RUSTSEC-2026-0110", reason = "expires 2026-11-13" }, # proc-macro-error2 2.0.1 (unmaintained, so no version is patched). # Pulled only by the Pico 2 W firmware, through the pio-proc assembler macro embassy-rp 0.10 requires through its # newest release. It runs at compile time inside that macro and reports its errors; none of it is in the firmware. # Drop this when pio-proc leaves it. - { id = "RUSTSEC-2026-0173", reason = "expires 2026-10-31" }, + { id = "RUSTSEC-2026-0173", reason = "expires 2026-11-13" }, ] [licenses] diff --git a/config/jscpd-baseline.json b/config/jscpd-baseline.json index fe828c3d..9407a0b6 100644 --- a/config/jscpd-baseline.json +++ b/config/jscpd-baseline.json @@ -37,7 +37,6 @@ "162b7eccbfa52a74": 1, "16acb47f836cc1f8": 1, "1808b7fc36bbb822": 2, - "1887cc62f89c1e8c": 1, "1a07ddf368451c0a": 1, "1a1867cca4e1d4e8": 1, "1a1e1611e4db4656": 1, @@ -59,13 +58,13 @@ "264e155c8fdaf095": 1, "265b3a5011a42fc4": 1, "266cd9babed7be0c": 1, + "268a187843c419fa": 1, "26bec4de311d99cf": 1, "2c47cf795be8c0d9": 1, "2c61d8307488c819": 1, "2d5e921d26ac1b89": 1, "2e16d6fbda66df34": 1, "2e738d4de357d244": 1, - "2ed92f1bb371f90f": 1, "2f97adcddfac1644": 1, "3192274800f8c0d1": 1, "31a1985df946271b": 1, @@ -76,6 +75,7 @@ "34b2a7c1a1a932c1": 1, "34cd0de3c97e4264": 1, "35d7f304546de8a8": 1, + "373a54c3fd093400": 1, "378e3311bed8161a": 1, "389aea3e8662fb9b": 1, "39df8b0c7c0dccc1": 1, @@ -90,6 +90,7 @@ "3e5aadc3e77078b6": 1, "3e6e51f257b8453d": 1, "4268c1cf96611f46": 1, + "439a73019944af1b": 1, "43f0e3fca21daafa": 1, "448a09ade148d8c5": 1, "4534795197b3c34c": 1, @@ -103,6 +104,7 @@ "506af004dd69d586": 1, "50b373907a5a8f78": 1, "50f5f12046dd3030": 1, + "51c917fb5e31c2b5": 1, "51d9869c79c352cd": 1, "521c8a862b4eacb3": 1, "533bdecb47a87756": 1, @@ -119,10 +121,12 @@ "583f38ee93d4fa73": 1, "58b70a86860c6b0d": 1, "58d184730f880246": 1, + "58e9f34c08256872": 1, "5990eb8206926248": 1, "5a38489b585cef78": 1, "5a5b6a5ae3b87eea": 1, "5b29341cf992fa9f": 1, + "5badf35d2055b4a7": 1, "5cfb0f0873663a20": 1, "5da193ae29a89044": 1, "5e1507230a488a93": 1, @@ -164,6 +168,7 @@ "7558cd4dfb81e574": 1, "76b4bda9089c9985": 1, "773fa53d9003b640": 1, + "77faaeba6d9492d5": 1, "781cc393068d5440": 1, "7ac63870d0087e3b": 1, "7aee849465ea91ab": 1, @@ -185,7 +190,6 @@ "879bfe322b3d676b": 1, "87f1eade46af4d27": 1, "88bca8b73d89f892": 1, - "893d7c317a045d82": 1, "8996a093b7bb3b54": 1, "89bb6ca7b46aa5cd": 1, "8acea4476f04e4a0": 2, @@ -247,7 +251,6 @@ "acdebc833a1835f4": 1, "ad6c8d0f5db047f5": 1, "ae590b605758bec9": 1, - "b01781537f9ea400": 1, "b04b47360d15964e": 1, "b07cc7ec51bc1602": 1, "b29077918336343b": 1, @@ -312,7 +315,6 @@ "da7b27ae8c89cb73": 1, "da812f4e54fefbf1": 1, "db4fb9ed218e4d66": 1, - "dbf01492cda8f9be": 1, "dd101dfd2d94984f": 1, "dd162080de996936": 1, "dd41701c3663b9f3": 1, diff --git a/config/lychee.toml b/config/lychee.toml index 962a9e71..7c28e905 100644 --- a/config/lychee.toml +++ b/config/lychee.toml @@ -52,6 +52,8 @@ exclude = [ '\{|%7B', '^file:///app/storage$', '^https://github\.com/[^/]+/[^/]+/actions/runs/', + # The base the Pico 2 W firmware appends a package path to; alone it names no package, and jsDelivr answers 400. + '^https://cdn\.jsdelivr\.net/npm/?$', # Kubernetes API address; reachable only from inside the cluster. '^https://kubernetes\.default\.svc/?$', '^https://openhub\.net/p/edge-toolkit(/|$)', diff --git a/config/osv-scanner.toml b/config/osv-scanner.toml index 0115178c..fa40d4d8 100644 --- a/config/osv-scanner.toml +++ b/config/osv-scanner.toml @@ -1,39 +1,39 @@ # AUTO-GENERATED from config/deny.toml by `mise run gen:osv-scanner`. # The list spans every language, so "has unused ignores" warnings are expected: agents must ignore them. IgnoredVulns = [ - { id = "GHSA-5j98-2g5x-46v6", ignoreUntil = 2026-10-31 }, - { id = "GHSA-6f2x-v7q7-m7m5", ignoreUntil = 2026-10-31 }, - { id = "GHSA-qwgh-2vcv-g2f7", ignoreUntil = 2026-10-31 }, - { id = "RUSTSEC-2023-0071", ignoreUntil = 2026-10-31 }, - { id = "RUSTSEC-2024-0436", ignoreUntil = 2026-10-31 }, - { id = "RUSTSEC-2024-0449", ignoreUntil = 2026-10-31 }, - { id = "RUSTSEC-2025-0134", ignoreUntil = 2026-10-31 }, - { id = "RUSTSEC-2025-0141", ignoreUntil = 2026-10-31 }, - { id = "RUSTSEC-2026-0097", ignoreUntil = 2026-10-31 }, - { id = "RUSTSEC-2026-0110", ignoreUntil = 2026-10-31 }, - { id = "RUSTSEC-2026-0118", ignoreUntil = 2026-10-31 }, - { id = "RUSTSEC-2026-0119", ignoreUntil = 2026-10-31 }, - { id = "RUSTSEC-2026-0173", ignoreUntil = 2026-10-31 }, - { id = "RUSTSEC-2026-0249", ignoreUntil = 2026-10-31 }, - { id = "RUSTSEC-2026-0258", ignoreUntil = 2026-10-31 }, - { id = "RUSTSEC-2026-0285", ignoreUntil = 2026-10-31 }, - { id = "RUSTSEC-2026-0346", ignoreUntil = 2026-10-31 }, - { id = "RUSTSEC-2026-0348", ignoreUntil = 2026-10-31 }, - { id = "RUSTSEC-2026-0349", ignoreUntil = 2026-10-31 }, - { id = "RUSTSEC-2026-0381", ignoreUntil = 2026-10-31 }, - { id = "RUSTSEC-2026-0383", ignoreUntil = 2026-10-31 }, - { id = "RUSTSEC-2026-0387", ignoreUntil = 2026-10-31 }, - { id = "RUSTSEC-2026-0391", ignoreUntil = 2026-10-31 }, - { id = "RUSTSEC-2026-0395", ignoreUntil = 2026-10-31 }, - { id = "RUSTSEC-2026-0396", ignoreUntil = 2026-10-31 }, - { id = "RUSTSEC-2026-0397", ignoreUntil = 2026-10-31 }, - { id = "RUSTSEC-2026-0398", ignoreUntil = 2026-10-31 }, - { id = "RUSTSEC-2026-0399", ignoreUntil = 2026-10-31 }, - { id = "RUSTSEC-2026-0400", ignoreUntil = 2026-10-31 }, - { id = "RUSTSEC-2026-0401", ignoreUntil = 2026-10-31 }, - { id = "RUSTSEC-2026-0402", ignoreUntil = 2026-10-31 }, - { id = "RUSTSEC-2026-0403", ignoreUntil = 2026-10-31 }, - { id = "RUSTSEC-2026-0404", ignoreUntil = 2026-10-31 }, - { id = "RUSTSEC-2026-0405", ignoreUntil = 2026-10-31 }, - { id = "RUSTSEC-2026-0406", ignoreUntil = 2026-10-31 }, + { id = "GHSA-5j98-2g5x-46v6", ignoreUntil = 2026-11-13 }, + { id = "GHSA-6f2x-v7q7-m7m5", ignoreUntil = 2026-11-13 }, + { id = "GHSA-qwgh-2vcv-g2f7", ignoreUntil = 2026-11-13 }, + { id = "RUSTSEC-2023-0071", ignoreUntil = 2026-11-13 }, + { id = "RUSTSEC-2024-0436", ignoreUntil = 2026-11-13 }, + { id = "RUSTSEC-2024-0449", ignoreUntil = 2026-11-13 }, + { id = "RUSTSEC-2025-0134", ignoreUntil = 2026-11-13 }, + { id = "RUSTSEC-2025-0141", ignoreUntil = 2026-11-13 }, + { id = "RUSTSEC-2026-0097", ignoreUntil = 2026-11-13 }, + { id = "RUSTSEC-2026-0110", ignoreUntil = 2026-11-13 }, + { id = "RUSTSEC-2026-0118", ignoreUntil = 2026-11-13 }, + { id = "RUSTSEC-2026-0119", ignoreUntil = 2026-11-13 }, + { id = "RUSTSEC-2026-0173", ignoreUntil = 2026-11-13 }, + { id = "RUSTSEC-2026-0249", ignoreUntil = 2026-11-13 }, + { id = "RUSTSEC-2026-0258", ignoreUntil = 2026-11-13 }, + { id = "RUSTSEC-2026-0285", ignoreUntil = 2026-11-13 }, + { id = "RUSTSEC-2026-0346", ignoreUntil = 2026-11-13 }, + { id = "RUSTSEC-2026-0348", ignoreUntil = 2026-11-13 }, + { id = "RUSTSEC-2026-0349", ignoreUntil = 2026-11-13 }, + { id = "RUSTSEC-2026-0381", ignoreUntil = 2026-11-13 }, + { id = "RUSTSEC-2026-0383", ignoreUntil = 2026-11-13 }, + { id = "RUSTSEC-2026-0387", ignoreUntil = 2026-11-13 }, + { id = "RUSTSEC-2026-0391", ignoreUntil = 2026-11-13 }, + { id = "RUSTSEC-2026-0395", ignoreUntil = 2026-11-13 }, + { id = "RUSTSEC-2026-0396", ignoreUntil = 2026-11-13 }, + { id = "RUSTSEC-2026-0397", ignoreUntil = 2026-11-13 }, + { id = "RUSTSEC-2026-0398", ignoreUntil = 2026-11-13 }, + { id = "RUSTSEC-2026-0399", ignoreUntil = 2026-11-13 }, + { id = "RUSTSEC-2026-0400", ignoreUntil = 2026-11-13 }, + { id = "RUSTSEC-2026-0401", ignoreUntil = 2026-11-13 }, + { id = "RUSTSEC-2026-0402", ignoreUntil = 2026-11-13 }, + { id = "RUSTSEC-2026-0403", ignoreUntil = 2026-11-13 }, + { id = "RUSTSEC-2026-0404", ignoreUntil = 2026-11-13 }, + { id = "RUSTSEC-2026-0405", ignoreUntil = 2026-11-13 }, + { id = "RUSTSEC-2026-0406", ignoreUntil = 2026-11-13 }, ] diff --git a/libs/path/src/lib.rs b/libs/path/src/lib.rs index f5869d90..4501b28e 100644 --- a/libs/path/src/lib.rs +++ b/libs/path/src/lib.rs @@ -124,8 +124,8 @@ fn normalize_path(path: &Path) -> PathBuf { let mut normalized = PathBuf::new(); for component in path.components() { match component { - Component::Prefix(prefix) => normalized.push(prefix.as_os_str()), - Component::RootDir => normalized.push(Path::new("/")), + // The root as the platform spells it, so a Windows path keeps one separator throughout. + Component::Prefix(_) | Component::RootDir => normalized.push(component.as_os_str()), Component::CurDir => {} Component::ParentDir => { let _popped = normalized.pop(); diff --git a/libs/path/tests/absolute.rs b/libs/path/tests/absolute.rs new file mode 100644 index 00000000..5329ba8d --- /dev/null +++ b/libs/path/tests/absolute.rs @@ -0,0 +1,27 @@ +//! Covers `absolute_from`: a path already in normal form comes back exactly as the platform spells it. +//! +//! Compared as strings, since two `Path`s compare equal component by component, whichever separators they use. +#![cfg(test)] + +use std::env; +use std::path::Path; + +use et_path::absolute_from; + +#[test] +fn a_relative_path_joins_its_base_as_the_platform_would() { + let base = env::temp_dir().join("scenario"); + assert_eq!( + absolute_from(&base, Path::new("modules")).as_os_str(), + base.join("modules").as_os_str() + ); +} + +#[test] +fn an_absolute_path_is_kept_as_the_platform_spells_it() { + let path = env::temp_dir().join("scenario").join("modules"); + assert_eq!( + absolute_from(Path::new("elsewhere"), &path).as_os_str(), + path.as_os_str() + ); +} diff --git a/libs/sanitise/Cargo.toml b/libs/sanitise/Cargo.toml new file mode 100644 index 00000000..07110418 --- /dev/null +++ b/libs/sanitise/Cargo.toml @@ -0,0 +1,18 @@ +[package] +name = "et-sanitise" +publish = true +description = "Checks that a chosen name is safe in a generated shell command and on every filesystem" +version = "0.1.0" +edition.workspace = true +license.workspace = true +repository.workspace = true + +[lib] +doctest = false + +[dependencies] +sanitize-filename.workspace = true +shlex.workspace = true + +[lints] +workspace = true diff --git a/libs/sanitise/src/lib.rs b/libs/sanitise/src/lib.rs new file mode 100644 index 00000000..f0f1f914 --- /dev/null +++ b/libs/sanitise/src/lib.rs @@ -0,0 +1,56 @@ +//! Checks that a name someone chose can be used as written wherever the workspace puts it. +//! +//! A name is plain when it is ASCII and two libraries each accept it unchanged: sanitize-filename, with its Windows +//! rules on whatever the host, so a name holds on every filesystem the workspace supports; and shlex, so a generated +//! shell command can name it as a bare word, with nothing in it that a shell would expand, split or run. Checking +//! rather than quoting keeps every generated file readable, and leaves no generator to forget the quoting. +//! +//! A path nobody here chose -- the directory a scenario happens to sit in, such as a Windows temporary directory under +//! `RUNNER~1` -- is held to a narrower rule instead, which no library states: that it reads back unchanged inside a +//! double-quoted shell string, where every generated command puts it, and inside a comma-separated list. + +use std::borrow::Cow; +use std::path::{Component, Path}; + +use sanitize_filename::OptionsForCheck; + +/// Whether `name`, one file or directory name, is used as written on every filesystem and in every shell command. +/// +/// It has to be ASCII as well, which neither library asks: a non-ASCII name can be spelled in more than one Unicode +/// normal form, and filesystems disagree on whether those are the same name. +#[must_use] +pub fn is_plain_name(name: &str) -> bool { + let portable = OptionsForCheck { + windows: true, + truncate: true, + }; + name.is_ascii() + && sanitize_filename::is_sanitized_with_options(name, portable) + && matches!(shlex::try_quote(name), Ok(Cow::Borrowed(_))) +} + +/// Whether every name `path` is made of is plain, and valid UTF-8. +/// +/// The parts a platform supplies rather than a person -- a root, a Windows drive prefix, `.` and `..` -- are not names, +/// and are not checked. +#[must_use] +pub fn is_plain_path(path: &Path) -> bool { + path.components().all(|component| match component { + Component::Normal(name) => name.to_str().is_some_and(is_plain_name), + Component::Prefix(_) | Component::RootDir | Component::CurDir | Component::ParentDir => true, + }) +} + +/// Whether `path`, valid UTF-8, reads back unchanged inside a double-quoted shell string and a comma-separated list. +/// +/// Inside double quotes a shell still expands `$` and a backtick, and a `"` ends the string, so all three are refused; +/// with them gone, a `\` -- the Windows separator -- escapes nothing and stays. A `,` would split the path in two in a +/// list such as `MODULES_PATHS`, and a control character, a newline among them, ends the command. +#[must_use] +pub fn is_quotable_path(path: &Path) -> bool { + path.to_str().is_some_and(|text| { + !text + .chars() + .any(|character| matches!(character, '"' | '$' | '`' | ',') || character.is_control()) + }) +} diff --git a/libs/sanitise/tests/plain.rs b/libs/sanitise/tests/plain.rs new file mode 100644 index 00000000..6bf8a95f --- /dev/null +++ b/libs/sanitise/tests/plain.rs @@ -0,0 +1,56 @@ +//! Covers what this crate adds to its two libraries: that a name has to pass both, which parts of a path are names, +//! and the rule of its own that a path nobody here chose is held to. +//! +//! What each library rejects is its own tests' to prove; one name per library is enough to show it is consulted. +#![cfg(test)] + +use std::path::Path; + +use et_sanitise::{is_plain_name, is_plain_path, is_quotable_path}; + +#[test] +fn a_name_passes_only_when_both_libraries_accept_it() { + assert!(is_plain_name("math1-sender_v2.0")); + // sanitize-filename's Windows rules, applied whatever the host. + assert!(!is_plain_name("con")); + // shlex's bare words, which sanitize-filename alone would allow. + assert!(!is_plain_name("mod$(id)")); + // ASCII, which both libraries would allow. + assert!(!is_plain_name("r\u{e9}sum\u{e9}")); +} + +#[test] +fn a_path_is_checked_name_by_name() { + assert!(is_plain_path(Path::new("/modules/./mine/../math1"))); + assert!(!is_plain_path(Path::new("modules/my mod"))); +} + +#[test] +fn a_name_that_is_not_utf8_is_not_plain() { + // A lone byte that starts no UTF-8 sequence, or a lone surrogate, as each platform spells a name. + #[cfg(unix)] + let name: std::ffi::OsString = std::os::unix::ffi::OsStringExt::from_vec(vec![0xff]); + #[cfg(windows)] + let name: std::ffi::OsString = std::os::windows::ffi::OsStringExt::from_wide(&[0xD800]); + assert!(!is_plain_path(Path::new(&name))); + assert!(!is_quotable_path(Path::new(&name))); +} + +#[test] +fn a_quotable_path_keeps_what_double_quotes_leave_alone() { + // What a home or temporary directory holds, which a bare word could not: a space, a `~`, a Windows `\`. + for path in [ + r"C:\Users\RUNNER~1\AppData\Local\Temp", + "/home/Jo Smith/my mods (old)", + "/srv/~app/a;b'c", + ] { + assert!(is_quotable_path(Path::new(path))); + } +} + +#[test] +fn a_quotable_path_refuses_what_double_quotes_and_lists_do_not() { + for path in ["/case\";id;echo \"", "/srv/$HOME", "/srv/`id`", "/srv/a,b", "/srv/a\nb"] { + assert!(!is_quotable_path(Path::new(path))); + } +} diff --git a/services/firmware-modules/Cargo.toml b/services/firmware-modules/Cargo.toml new file mode 100644 index 00000000..d9ff4ec5 --- /dev/null +++ b/services/firmware-modules/Cargo.toml @@ -0,0 +1,22 @@ +[package] +name = "et-firmware-modules" +publish = true +description = "The source a firmware's build script writes to build ws-modules into its image" +version = "0.1.0" +edition.workspace = true +license.workspace = true +repository.workspace = true + +[lib] +doctest = false + +[dependencies] +fs-err.workspace = true +serde_json = { workspace = true, features = ["std"] } +thiserror = { workspace = true, features = ["std"] } + +[dev-dependencies] +tempfile.workspace = true + +[lints] +workspace = true diff --git a/services/firmware-modules/src/lib.rs b/services/firmware-modules/src/lib.rs new file mode 100644 index 00000000..940b330b --- /dev/null +++ b/services/firmware-modules/src/lib.rs @@ -0,0 +1,283 @@ +//! The source of a firmware's `modules.rs`: every ws-module package it serves, built in with `include_bytes!`. +//! +//! A firmware's build script reads the variables the `std` server's modules service reads -- the module directories, +//! the one served at `/`, and the packages served from elsewhere -- and writes what [`generate`] returns. Each +//! directory is a module the way that service finds one: its `pkg/` when it has one, otherwise the directory itself, +//! named by the `package.json` there. Its files are the ones npm would publish, each served as its extension says and +//! tagged with a hash of its contents. + +use std::fmt::Write as _; +use std::hash::{DefaultHasher, Hash as _, Hasher as _}; +use std::path::{Component, Path, PathBuf}; + +use fs_err as fs; + +/// The directory a module's build leaves its package in, when the module's own directory is its source. +const PACKAGE_DIR: &str = "pkg"; +/// The manifest naming a module, and listing its files. +const PACKAGE_JSON: &str = "package.json"; +/// Where the types the source builds its modules from live, named in full so a build with none imports nothing. +const MODULES_API: &str = "et_ws_server::picoserve_modules"; +/// The variable the module directories come from, which errors about them name. +const MODULES_PATHS: &str = "MODULES_PATHS"; + +/// Why a firmware's modules cannot be built in. +#[derive(Debug, thiserror::Error)] +#[non_exhaustive] +pub enum ModulesError { + #[error("{variable} must hold absolute paths, not {path}")] + RelativePath { variable: String, path: PathBuf }, + #[error("{variable} holds a path that is not valid UTF-8")] + NotUtf8 { variable: String }, + #[error(transparent)] + Io(#[from] std::io::Error), + #[error("{path} is not valid JSON")] + Json { + path: PathBuf, + #[source] + source: serde_json::Error, + }, + #[error("{0} declares no name")] + NoName(PathBuf), + #[error("{0} lists a file that is not a string")] + FileNotString(PathBuf), + #[error("{manifest} lists {entry}, which is outside the package")] + OutsidePackage { manifest: PathBuf, entry: String }, + #[error("{0} is a link to a file outside its package")] + LinksOutside(PathBuf), + #[error("MODULES_ROOT names {root}, which none of MODULES_PATHS provides; serving {names:?}")] + UnknownRoot { root: String, names: Vec }, + #[error("MODULES_ELSEWHERE needs MODULES_ELSEWHERE_URL to say where they are")] + ElsewhereWithoutUrl, +} + +/// The source of `modules.rs`, and every file it builds in, which the build has to rerun when one changes. +#[derive(Debug)] +#[non_exhaustive] +pub struct Generated { + /// The Rust source defining `static MODULES`. + pub source: String, + /// Every file `source` names in an `include_bytes!`. + pub files: Vec, +} + +/// The source building in the modules in `paths`, and serving the rest of the selection. +/// +/// It serves `root` at `/`, and redirects the modules in `elsewhere` to `elsewhere_url`. `paths` and `elsewhere` are +/// comma-separated lists, as the `MODULES_PATHS` and `MODULES_ELSEWHERE` variables hold them, and every one of them may +/// be empty. +/// +/// # Errors +/// +/// When a path is relative or not UTF-8, a package cannot be read or lists or links to a file outside itself, `root` +/// names none of the modules, or `elsewhere` names modules with no `elsewhere_url` to find them at. +#[expect( + clippy::use_debug, + reason = "a str's Debug form is the Rust literal the generated source needs" +)] +pub fn generate(paths: &str, root: &str, elsewhere: &str, elsewhere_url: &str) -> Result { + let mut embedded = String::default(); + let mut names = Vec::new(); + let mut built_in = Vec::new(); + for path in items(paths) { + let path = PathBuf::from(path); + // Checked before anything is read, so a relative entry is refused as one, wherever the build happens to run. + let _literal = include_path(MODULES_PATHS, &path)?; + let package_dir = if path.join(PACKAGE_DIR).is_dir() { + path.join(PACKAGE_DIR) + } else { + path + }; + let package = package(&package_dir)?; + let _written = writeln!(embedded, " {MODULES_API}::Module::new({:?}, &[", package.name); + for file in package.files { + // Checked whole first, so the name within the package, a part of it, is valid UTF-8 too. + let literal = include_path(MODULES_PATHS, &file)?; + // Every file was collected from under the package, so it always has the package as its prefix. + let relative = file + .strip_prefix(&package_dir) + .unwrap_or(&file) + .to_string_lossy() + .replace('\\', "/"); + let _written = writeln!( + embedded, + " ({relative:?}, {MODULES_API}::ModuleFile::new({:?}, include_bytes!({literal}), {:?})),", + content_type(&file), + etag(&file)?, + ); + built_in.push(file); + } + embedded.push_str(" ]),\n"); + names.push(package.name); + } + if !root.is_empty() && !names.iter().any(|name| name == root) { + return Err(ModulesError::UnknownRoot { + root: root.to_owned(), + names, + }); + } + let elsewhere: Vec<&str> = items(elsewhere).collect(); + if !elsewhere.is_empty() && elsewhere_url.is_empty() { + return Err(ModulesError::ElsewhereWithoutUrl); + } + let source = format!( + "/// The modules the firmware serves, as `MODULES_PATHS`, `MODULES_ROOT` and `MODULES_ELSEWHERE*` named them. +static MODULES: {MODULES_API}::Modules = {MODULES_API}::Modules::new( + &[ +{embedded} ], + {root:?}, + &{elsewhere:?}, + {elsewhere_url:?}, +); +" + ); + Ok(Generated { + source, + files: built_in, + }) +} + +/// `path` as the literal an `include_bytes!` names it by, which has to be absolute; `variable` is where it came from. +/// +/// A build script runs in its own crate's directory, so a relative path would quietly resolve against that. +/// +/// # Errors +/// +/// When `path` is relative, or is not valid UTF-8. +pub fn include_path(variable: &str, path: &Path) -> Result { + if !path.is_absolute() { + return Err(ModulesError::RelativePath { + variable: variable.to_owned(), + path: path.to_path_buf(), + }); + } + let path = path.to_str().ok_or_else(|| ModulesError::NotUtf8 { + variable: variable.to_owned(), + })?; + Ok(format!("{path:?}")) +} + +/// The comma-separated items of `list`, trimmed, with empty ones dropped. +fn items(list: &str) -> impl Iterator { + list.split(',').map(str::trim).filter(|item| !item.is_empty()) +} + +/// A module package: the name its `package.json` declares, and the files it publishes. +struct Package { + name: String, + files: Vec, +} + +/// The package in `dir`, with the files npm would publish from it. +/// +/// That is every file its `package.json` lists in `files`, and the manifest itself, or every file in `dir` when it +/// lists none. Either way, nothing under `node_modules` or hidden. +#[expect(clippy::single_call_fn, reason = "the per-module step of generate")] +fn package(dir: &Path) -> Result { + let manifest_path = dir.join(PACKAGE_JSON); + // Matched rather than converted, so the error can name the manifest that would not parse. + let manifest: serde_json::Value = match serde_json::from_str(&fs::read_to_string(&manifest_path)?) { + Ok(manifest) => manifest, + Err(source) => { + return Err(ModulesError::Json { + path: manifest_path, + source, + }); + } + }; + let name = manifest + .get("name") + .and_then(serde_json::Value::as_str) + .ok_or_else(|| ModulesError::NoName(manifest_path.clone()))? + .to_owned(); + let mut files = Vec::new(); + match manifest.get("files").and_then(serde_json::Value::as_array) { + Some(listed) => { + files.push(manifest_path.clone()); + for entry in listed { + let entry = entry + .as_str() + .ok_or_else(|| ModulesError::FileNotString(manifest_path.clone()))?; + // Only names within the package: a root, a drive or a `..` would build in a file from elsewhere. + if !Path::new(entry) + .components() + .all(|part| matches!(part, Component::Normal(_) | Component::CurDir)) + { + return Err(ModulesError::OutsidePackage { + manifest: manifest_path, + entry: entry.to_owned(), + }); + } + // Joined without its `.` parts, which would otherwise end up in the path the file is served at. + let within: PathBuf = Path::new(entry) + .components() + .filter(|part| matches!(part, Component::Normal(_))) + .collect(); + collect(&dir.join(within), &mut files)?; + } + } + None => collect(dir, &mut files)?, + } + // A link anywhere below the package root could build in a file from anywhere; one to the root itself is kept. + let root = fs::canonicalize(dir)?; + for file in &files { + if !fs::canonicalize(file)?.starts_with(&root) { + return Err(ModulesError::LinksOutside(file.clone())); + } + } + files.sort(); + files.dedup(); + Ok(Package { name, files }) +} + +/// Add `path` to `files`, or every file under it when it is a directory, skipping `node_modules` and hidden entries. +fn collect(path: &Path, files: &mut Vec) -> Result<(), ModulesError> { + if !path.is_dir() { + files.push(path.to_path_buf()); + return Ok(()); + } + for entry in fs::read_dir(path)? { + let entry = entry?; + let name = entry.file_name(); + let name = name.to_string_lossy(); + if name.starts_with('.') || name == "node_modules" { + continue; + } + collect(&entry.path(), files)?; + } + Ok(()) +} + +/// A tag for `file`'s contents, which changes whenever they do. +/// +/// Only one firmware ever compares it with itself, so a hash that a later toolchain computes differently costs a +/// browser one refetch after an update and nothing more. +#[expect( + clippy::single_call_fn, + reason = "the tagging step of generate, one of each file's three fields" +)] +fn etag(file: &Path) -> Result { + let mut hasher = DefaultHasher::new(); + fs::read(file)?.hash(&mut hasher); + Ok(format!("{:016x}", hasher.finish())) +} + +/// The content type `file` is served as, by its extension, as a static file server would guess it. +#[expect( + clippy::single_call_fn, + reason = "the typing step of generate, one of each file's three fields" +)] +fn content_type(file: &Path) -> &'static str { + match file.extension().and_then(|extension| extension.to_str()) { + Some("html" | "htm") => "text/html; charset=utf-8", + Some("js" | "mjs") => "application/javascript; charset=utf-8", + Some("css") => "text/css", + Some("json" | "map") => "application/json", + Some("wasm") => "application/wasm", + Some("png") => "image/png", + Some("svg") => "image/svg+xml", + Some("ico") => "image/x-icon", + Some("txt" | "md" | "py" | "ts") => "text/plain; charset=utf-8", + _ => "application/octet-stream", + } +} diff --git a/services/firmware-modules/tests/generate.rs b/services/firmware-modules/tests/generate.rs new file mode 100644 index 00000000..e8069f2b --- /dev/null +++ b/services/firmware-modules/tests/generate.rs @@ -0,0 +1,242 @@ +//! Covers the `modules.rs` source a firmware build writes: which files of which packages it builds in, how each is +//! served, and every way a module selection is refused. +#![cfg(test)] + +use std::path::{Path, PathBuf}; + +use et_firmware_modules::{Generated, ModulesError, generate, include_path}; +use fs_err as fs; +use tempfile::{TempDir, tempdir}; + +/// Write `contents` to `path` under `root`, making its directories. +fn write(root: &Path, path: &str, contents: &str) -> PathBuf { + let path = root.join(path); + fs::create_dir_all(path.parent().unwrap()).unwrap(); + fs::write(&path, contents).unwrap(); + path +} + +/// Make `link` a symbolic link to the file `target`. +fn link_to(target: &Path, link: &Path) -> PathBuf { + #[cfg(unix)] + std::os::unix::fs::symlink(target, link).unwrap(); + #[cfg(windows)] + std::os::windows::fs::symlink_file(target, link).unwrap(); + link.to_path_buf() +} + +/// A built module in `pkg/`, listing its files, beside a source module that lists none. +fn two_modules() -> TempDir { + let root = tempdir().unwrap(); + let listed = r#"{"name": "@scope/built", "files": ["app.js", "assets", "./index.html"]}"#; + let _manifest = write(root.path(), "built/pkg/package.json", listed); + let _app = write(root.path(), "built/pkg/app.js", "app"); + let _page = write(root.path(), "built/pkg/index.html", "

page

"); + let _model = write(root.path(), "built/pkg/assets/model.wasm", "\0asm"); + let _hidden = write(root.path(), "built/pkg/assets/.cache", "hidden"); + let _unlisted = write(root.path(), "built/pkg/notes.md", "unlisted"); + let _source = write(root.path(), "built/src/lib.rs", "outside pkg"); + let _manifest = write(root.path(), "plain/package.json", r#"{"name": "plain"}"#); + let _style = write(root.path(), "plain/style.css", "p {}"); + let _dependency = write(root.path(), "plain/node_modules/dep/index.js", "dependency"); + root +} + +/// The paths of the two modules, as `MODULES_PATHS` lists them. +fn paths(root: &TempDir) -> String { + format!( + "{}, {} ,", + root.path().join("built").display(), + root.path().join("plain").display() + ) +} + +/// The source `generate` returns for `paths`, which has to succeed. +fn generated(paths: &str, root: &str, elsewhere: &str, url: &str) -> Generated { + generate(paths, root, elsewhere, url).unwrap() +} + +#[test] +fn it_builds_in_what_each_package_publishes() { + let root = two_modules(); + let generated = generated( + &paths(&root), + "@scope/built", + "big, cdn-only", + "https://cdn.example/npm", + ); + let names: Vec = generated + .files + .iter() + .map(|file| { + file.strip_prefix(root.path()) + .unwrap() + .to_string_lossy() + .replace('\\', "/") + }) + .collect(); + assert_eq!( + names, + [ + "built/pkg/app.js", + "built/pkg/assets/model.wasm", + "built/pkg/index.html", + "built/pkg/package.json", + "plain/package.json", + "plain/style.css", + ] + ); + + let source = &generated.source; + assert!(source.contains(r#"Module::new("@scope/built", &["#)); + assert!( + source.contains(r#"("assets/model.wasm", et_ws_server::picoserve_modules::ModuleFile::new("application/wasm""#) + ); + assert!( + source + .contains(r#"("index.html", et_ws_server::picoserve_modules::ModuleFile::new("text/html; charset=utf-8""#) + ); + assert!(source.contains(r#"Module::new("plain", &["#)); + assert!(source.contains(" \"@scope/built\",\n")); + assert!(source.contains(r#"&["big", "cdn-only"],"#)); + assert!(source.contains(r#""https://cdn.example/npm","#)); +} + +#[test] +fn a_file_is_tagged_by_its_contents() { + let root = tempdir().unwrap(); + let _manifest = write( + root.path(), + "m/package.json", + r#"{"name": "m", "files": ["a.txt", "b.txt", "c.txt"]}"#, + ); + let _a = write(root.path(), "m/a.txt", "same"); + let _b = write(root.path(), "m/b.txt", "same"); + let _c = write(root.path(), "m/c.txt", "different"); + let source = generated(&root.path().join("m").display().to_string(), "", "", "").source; + let tags: Vec<&str> = source + .lines() + .filter(|line| line.contains(".txt\"")) + .map(|line| line.rsplit_once(", \"").unwrap().1) + .collect(); + assert_eq!(tags.len(), 3); + assert_eq!(tags[0], tags[1]); + assert_ne!(tags[0], tags[2]); +} + +#[test] +fn each_file_is_served_as_its_extension_says() { + let root = tempdir().unwrap(); + let extensions = [ + ("htm", "text/html; charset=utf-8"), + ("mjs", "application/javascript; charset=utf-8"), + ("map", "application/json"), + ("png", "image/png"), + ("svg", "image/svg+xml"), + ("ico", "image/x-icon"), + ("py", "text/plain; charset=utf-8"), + ("bin", "application/octet-stream"), + ]; + let _manifest = write(root.path(), "m/package.json", r#"{"name": "m"}"#); + for (extension, _) in extensions { + let _file = write(root.path(), &format!("m/file.{extension}"), extension); + } + let _bare = write(root.path(), "m/LICENSE", "no extension"); + let source = generated(&root.path().join("m").display().to_string(), "", "", "").source; + for (extension, content_type) in extensions { + assert!(source.contains(&format!( + r#"("file.{extension}", et_ws_server::picoserve_modules::ModuleFile::new("{content_type}""# + ))); + } + assert!( + source.contains(r#"("LICENSE", et_ws_server::picoserve_modules::ModuleFile::new("application/octet-stream""#) + ); +} + +#[test] +fn nothing_named_builds_in_nothing() { + let generated = generated(" , ", "", "", ""); + assert_eq!(generated.files, Vec::::new()); + assert!( + generated + .source + .contains("Modules::new(\n &[\n ],\n \"\",\n &[],\n \"\",\n);") + ); +} + +#[test] +fn a_selection_that_cannot_be_served_is_refused() { + let root = two_modules(); + let unknown_root = generate(&paths(&root), "absent", "", "").unwrap_err(); + assert!(matches!(&unknown_root, ModulesError::UnknownRoot { root, names } if root == "absent" && names.len() == 2)); + let no_url = generate("", "", "big", "").unwrap_err(); + assert!(matches!(no_url, ModulesError::ElsewhereWithoutUrl)); + let relative = generate("relative/module", "", "", "").unwrap_err(); + assert!(matches!(relative, ModulesError::RelativePath { .. })); +} + +#[test] +fn a_link_out_of_the_package_is_refused() { + let root = tempdir().unwrap(); + let secret = write(root.path(), "secret.txt", "secret"); + let _manifest = write(root.path(), "m/package.json", r#"{"name": "m"}"#); + let _inside = write(root.path(), "m/inside.txt", "inside"); + let link = link_to(&secret, &root.path().join("m/leak.txt")); + let refused = generate(&root.path().join("m").display().to_string(), "", "", "").unwrap_err(); + assert!(matches!(&refused, ModulesError::LinksOutside(file) if *file == link)); +} + +#[test] +fn a_link_within_the_package_is_built_in() { + let root = tempdir().unwrap(); + let _manifest = write(root.path(), "m/package.json", r#"{"name": "m"}"#); + let inside = write(root.path(), "m/inside.txt", "inside"); + let link = link_to(&inside, &root.path().join("m/alias.txt")); + let generated = generated(&root.path().join("m").display().to_string(), "", "", ""); + assert!(generated.files.contains(&link)); +} + +#[test] +fn a_package_that_cannot_be_read_is_refused() { + let root = tempdir().unwrap(); + let cases = [ + ("missing", None), + ("not-json", Some("{")), + ("nameless", Some(r#"{"files": []}"#)), + ("odd-file", Some(r#"{"name": "odd", "files": [7]}"#)), + ("escapes", Some(r#"{"name": "escapes", "files": ["/secret"]}"#)), + ]; + for (dir, manifest) in cases { + fs::create_dir_all(root.path().join(dir)).unwrap(); + if let Some(manifest) = manifest { + let _manifest = write(root.path(), &format!("{dir}/package.json"), manifest); + } + let refused = generate(&root.path().join(dir).display().to_string(), "", "", "").unwrap_err(); + let expected = match dir { + "missing" => matches!(refused, ModulesError::Io(_)), + "not-json" => matches!(refused, ModulesError::Json { .. }), + "nameless" => matches!(refused, ModulesError::NoName(_)), + "odd-file" => matches!(refused, ModulesError::FileNotString(_)), + _ => matches!(&refused, ModulesError::OutsidePackage { entry, .. } if entry == "/secret"), + }; + assert!(expected, "{dir}: {refused}"); + } +} + +#[test] +fn an_include_path_is_absolute_and_utf8() { + let root = tempdir().unwrap(); + let literal = include_path("VAR", root.path()).unwrap(); + assert_eq!(literal, format!("{:?}", root.path().to_str().unwrap())); + + let relative = include_path("VAR", Path::new("relative")).unwrap_err(); + assert_eq!(relative.to_string(), "VAR must hold absolute paths, not relative"); + + // A lone byte that starts no UTF-8 sequence, or a lone surrogate, as each platform spells a name. + #[cfg(unix)] + let name: std::ffi::OsString = std::os::unix::ffi::OsStringExt::from_vec(vec![0xff]); + #[cfg(windows)] + let name: std::ffi::OsString = std::os::windows::ffi::OsStringExt::from_wide(&[0xD800]); + let not_utf8 = include_path("VAR", &root.path().join(name)).unwrap_err(); + assert_eq!(not_utf8.to_string(), "VAR holds a path that is not valid UTF-8"); +} diff --git a/services/ws-server-pico2w/Cargo.toml b/services/ws-server-pico2w/Cargo.toml index 85d40a9d..3fe2d178 100644 --- a/services/ws-server-pico2w/Cargo.toml +++ b/services/ws-server-pico2w/Cargo.toml @@ -18,28 +18,42 @@ name = "et-ws-server-pico2w" path = "src/main.rs" test = false -# The firmware's dependencies are scoped to bare-metal Arm, and its body is gated the same way. -# On any other target -- the workspace's host builds, lints and tests -- the binary compiles to an empty `main`. -[target.'cfg(all(target_arch = "arm", target_os = "none"))'.dependencies] +# The hub, its settings and the modules built in are the same wherever the binary is built for. +[dependencies] chrono = { workspace = true, features = ["alloc"] } +edge-toolkit.workspace = true +et-ws-server = { workspace = true, features = ["picoserve-server"] } +et-ws-service = { workspace = true, features = ["picoserve-hub"] } +picoserve.workspace = true + +# The board's network stack and executor, scoped to bare-metal Arm as the firmware body is. +[target.'cfg(all(target_arch = "arm", target_os = "none"))'.dependencies] cortex-m-rt.workspace = true cyw43 = { workspace = true, features = ["defmt"] } cyw43-pio = { workspace = true, features = ["defmt"] } defmt.workspace = true defmt-rtt.workspace = true -edge-toolkit.workspace = true embassy-executor = { workspace = true, features = ["defmt", "executor-thread", "platform-cortex-m"] } embassy-net = { workspace = true, features = ["defmt", "dhcpv4", "medium-ethernet", "proto-ipv4", "tcp"] } embassy-rp = { workspace = true, features = ["critical-section-impl", "defmt", "rp235xa", "time-driver"] } embassy-time = { workspace = true, features = ["defmt"] } embedded-alloc.workspace = true -et-ws-server = { workspace = true, features = ["picoserve-server"] } -et-ws-service = { workspace = true, features = ["picoserve-hub"] } panic-probe = { workspace = true, features = ["defmt"] } picoserve = { workspace = true, features = ["embassy"] } static_cell.workspace = true +# Anywhere else, the host's: tokio, and a clock that knows the date. +[target.'cfg(not(all(target_arch = "arm", target_os = "none")))'.dependencies] +chrono = { workspace = true, features = ["std"] } +picoserve = { workspace = true, features = ["tokio"] } +tokio = { workspace = true, features = ["net", "rt", "time"] } + +[dev-dependencies] +serde_json = { workspace = true, features = ["std"] } +tungstenite.workspace = true + [build-dependencies] +et-firmware-modules.workspace = true fs-err.workspace = true [lints] diff --git a/services/ws-server-pico2w/README.md b/services/ws-server-pico2w/README.md index df5dc442..c13ad3c5 100644 --- a/services/ws-server-pico2w/README.md +++ b/services/ws-server-pico2w/README.md @@ -1,20 +1,29 @@ # et-int-ws-server-pico2w et-ws-server as firmware for the Raspberry Pi Pico 2 W, built as the `et-ws-server-pico2w` binary: the `no_std` -hub's `/ws`, with `/health` and `/favicon.ico` beside it, served over the board's own Wi-Fi on port 8080. It joins -the network it was built for, takes an address over DHCP, and serves four connections at once. +hub's `/ws`, with `/health`, `/favicon.ico` and the modules built into it beside it, served over the board's own +Wi-Fi on port 8080. It joins the network it was built for, takes an address over DHCP, and serves four connections +at once. It is a local hub, with no backplane, admin API, idle timeout or registry file. The board has no clock, so the times it stamps on messages are its uptime counted from the Unix epoch. ## Building -The build bakes in the network to join and the CYW43439 radio's firmware, so it needs three variables: +The build bakes in the network to join, from `WIFI_NETWORK` and `WIFI_PASSWORD`, and the CYW43439 radio's firmware. +The build task fetches the radio's `43439A0.bin`, `43439A0_clm.bin` and `nvram_rp2040.bin` from the embassy +repository's `cyw43-firmware` directory into `target/cyw43-firmware`, and points `CYW43_FIRMWARE_DIR` at them. +Infineon licenses those blobs separately from this project, so they are not part of it. -- `WIFI_NETWORK` and `WIFI_PASSWORD`: the network. -- `CYW43_FIRMWARE_DIR`: a directory holding the radio's `43439A0.bin`, `43439A0_clm.bin` and `nvram_rp2040.bin`, as - the `cyw43-firmware` directory of the embassy repository does. Infineon licenses those blobs separately from this - project, so they are not part of it. +It serves no modules unless told which to build in, through the variables the `std` server reads, each optional: + +- `MODULES_PATHS`: the module directories, comma-separated and absolute. Each is built in from its `pkg/` when it has + one, otherwise from the directory itself: every file its `package.json` lists under `files`, or every file there. +- `MODULES_ROOT`: the name of the module served at `/`, one of those. +- `MODULES_ELSEWHERE` and `MODULES_ELSEWHERE_URL`: module names too big for the board's 4 MiB of flash, listed with + the rest at `/modules/`, and the base URL a request for one of their files is redirected to. + +`et-cli generate-deployment --output-type firmware-pico2` writes the task that sets all of these for a scenario. ```sh MISE_ENV=rust mise run build-firmware-pico2w @@ -32,3 +41,16 @@ probe-rs run --chip RP235x target/thumbv8m.main-none-eabihf/release/et-ws-server ``` The log names the address it serves on, as `ws://
:8080/ws`. + +## Serving from the host + +Built for any other target, the binary is the same hub serving the same modules from the host over tokio, read from +the same `MODULES_*` at build time, so what a board would serve can be explored without one. It listens on the +address its one argument names, or on port 8080 on every interface, and prints where: + +```sh +cargo run -p et-int-ws-server-pico2w -- 127.0.0.1:8080 +``` + +The package's test starts it on a free port and fetches every module it lists, so run with the `MODULES_*` a +scenario's deployment sets, as its `firmware-host-test` task does, it checks what that board would serve. diff --git a/services/ws-server-pico2w/build.rs b/services/ws-server-pico2w/build.rs index 2d3620b6..3e5677fe 100644 --- a/services/ws-server-pico2w/build.rs +++ b/services/ws-server-pico2w/build.rs @@ -1,13 +1,26 @@ -//! Puts the Pico 2 W's memory layout where the linker looks, links the firmware with cortex-m-rt's and defmt's linker -//! scripts, and generates `board.rs`: what the firmware bakes in, read here from the build's environment. On any -//! other target the binary is empty and needs none of it. +//! Puts the Pico 2 W's memory layout where the linker looks, links the firmware with cortex-m-rt's and defmt's +//! linker scripts, and generates `board.rs` and `modules.rs`: what the firmware bakes in, read here from the build's +//! environment. On any other target the binary serves the same modules from the host, so only `modules.rs` is made. use std::env; use std::error::Error; use std::path::PathBuf; +use et_firmware_modules::{generate, include_path}; +use fs_err as fs; + /// The variables `board.rs` is generated from. const BOARD_VARIABLES: [&str; 3] = ["CYW43_FIRMWARE_DIR", "WIFI_NETWORK", "WIFI_PASSWORD"]; +/// The variables `modules.rs` is generated from, each named as the `std` server's modules service or `MODULES_*` reads. +const MODULE_VARIABLES: [&str; 4] = [MODULES_PATHS, MODULES_ROOT, MODULES_ELSEWHERE, MODULES_ELSEWHERE_URL]; +/// The module directories to build in, comma-separated, as the `std` server's `MODULES_PATHS` lists them. +const MODULES_PATHS: &str = "MODULES_PATHS"; +/// The name of the module served at `/`, as the `std` server's `MODULES_ROOT`. +const MODULES_ROOT: &str = "MODULES_ROOT"; +/// The names of the modules served from `MODULES_ELSEWHERE_URL` rather than built in, comma-separated. +const MODULES_ELSEWHERE: &str = "MODULES_ELSEWHERE"; +/// Where a request for a module in `MODULES_ELSEWHERE` is redirected, its path under `/modules` appended. +const MODULES_ELSEWHERE_URL: &str = "MODULES_ELSEWHERE_URL"; /// The operating system cargo is building for, `none` for the board. const CARGO_CFG_TARGET_OS: &str = "CARGO_CFG_TARGET_OS"; /// The directory cargo gives this build script for what it generates. @@ -15,15 +28,26 @@ const OUT_DIR: &str = "OUT_DIR"; fn main() -> Result<(), Box> { println!("cargo:rerun-if-changed=memory.x"); - for variable in BOARD_VARIABLES { + for variable in BOARD_VARIABLES.into_iter().chain(MODULE_VARIABLES) { println!("cargo:rerun-if-env-changed={variable}"); } + let out_dir = PathBuf::from(env::var(OUT_DIR)?); + // The modules are served on the host too, so what the board would serve can be explored without one. + let modules = generate( + &optional(MODULES_PATHS), + &optional(MODULES_ROOT), + &optional(MODULES_ELSEWHERE), + &optional(MODULES_ELSEWHERE_URL), + )?; + for file in &modules.files { + println!("cargo:rerun-if-changed={}", file.display()); + } + fs::write(out_dir.join("modules.rs"), modules.source)?; if env::var(CARGO_CFG_TARGET_OS)? != "none" { return Ok(()); } - let out_dir = PathBuf::from(env::var(OUT_DIR)?); - fs_err::write(out_dir.join("memory.x"), include_bytes!("memory.x"))?; - fs_err::write(out_dir.join("board.rs"), board()?)?; + fs::write(out_dir.join("memory.x"), include_bytes!("memory.x"))?; + fs::write(out_dir.join("board.rs"), board()?)?; println!("cargo:rustc-link-search={}", out_dir.display()); println!("cargo:rustc-link-arg-bins=--nmagic"); println!("cargo:rustc-link-arg-bins=-Tlink.x"); @@ -39,24 +63,27 @@ fn required(name: &str) -> Result> { Ok(value) } +/// The variable `name`, empty when it is unset, since every module variable may be. +fn optional(name: &str) -> String { + env::var(name).unwrap_or_default() +} + /// The source of `board.rs`: the network to join, and the radio's blobs from `CYW43_FIRMWARE_DIR`. #[expect( clippy::single_call_fn, - reason = "the generation step of main, kept apart from its linker directives" + reason = "the board generation step of main, kept apart from its linker directives" )] fn board() -> Result> { let firmware_dir = PathBuf::from(required("CYW43_FIRMWARE_DIR")?); - // A build script runs in its own crate's directory, so a relative path would quietly resolve against that. - if !firmware_dir.is_absolute() { - return Err("CYW43_FIRMWARE_DIR must be an absolute path".into()); - } let blob = |file: &str| -> Result> { let path = firmware_dir.join(file); if !path.is_file() { return Err(format!("CYW43_FIRMWARE_DIR holds no {file}").into()); } - let path = path.to_str().ok_or("CYW43_FIRMWARE_DIR is not valid UTF-8")?; - Ok(format!("include_bytes!({path:?})")) + Ok(format!( + "include_bytes!({})", + include_path("CYW43_FIRMWARE_DIR", &path)? + )) }; let network = required("WIFI_NETWORK")?; let password = required("WIFI_PASSWORD")?; diff --git a/services/ws-server-pico2w/src/firmware.rs b/services/ws-server-pico2w/src/firmware.rs index d840f453..d53d6071 100644 --- a/services/ws-server-pico2w/src/firmware.rs +++ b/services/ws-server-pico2w/src/firmware.rs @@ -16,7 +16,6 @@ use chrono::DateTime; use cyw43::JoinOptions; use cyw43_pio::{DEFAULT_CLOCK_DIVIDER, PioSpi}; use defmt::{Debug2Format, info, warn}; -use edge_toolkit::ports::Services; use edge_toolkit::ws_server::MessageRetention; use embassy_executor::Spawner; use embassy_net::tcp::TcpSocket; @@ -35,18 +34,12 @@ use picoserve::time::EmbassyTimer; use static_cell::StaticCell; use {defmt_rtt as _, panic_probe as _}; -/// The port the hub serves on, as the `std` server's plain-HTTP listener does. -const PORT: u16 = Services::InsecureWebSocketServer.port(); +use crate::{HTTP_BUFFER, MAX_FRAME_SIZE, MAX_QUEUED, MODULES, PORT}; + /// How many connections the hub serves at once: one task, and one socket's buffers, each. const CONNECTIONS: usize = 4; -/// The largest WebSocket frame the hub reads; each connection holds a buffer this size on the heap. -const MAX_FRAME_SIZE: usize = 8 * 1024; -/// How many messages for one connection the hub holds while it writes earlier ones, each up to `MAX_FRAME_SIZE`. -const MAX_QUEUED: usize = 2; /// Each connection's TCP receive and transmit buffer. const TCP_BUFFER: usize = 4 * 1024; -/// Each connection's buffer for the HTTP request head. -const HTTP_BUFFER: usize = 2048; /// The heap the hub's registry, routing and frame buffers allocate from. const HEAP_SIZE: usize = 160 * 1024; @@ -109,7 +102,16 @@ async fn connection_task(stack: Stack<'static>, hub: &'static Hub) let client_ip = socket .remote_endpoint() .map_or(IpAddr::V4(Ipv4Addr::UNSPECIFIED), |remote| remote.addr.into()); - if let Err(error) = serve(hub, socket, client_ip, EmbassyTimer, &config, &mut http_buffer).await { + let served = serve( + hub, + &MODULES, + socket, + client_ip, + EmbassyTimer, + &config, + &mut http_buffer, + ); + if let Err(error) = served.await { warn!( "Connection from {} ended: {}", Debug2Format(&client_ip), diff --git a/services/ws-server-pico2w/src/host.rs b/services/ws-server-pico2w/src/host.rs new file mode 100644 index 00000000..9cf813ab --- /dev/null +++ b/services/ws-server-pico2w/src/host.rs @@ -0,0 +1,78 @@ +//! The firmware's hub and modules, served from the host over tokio rather than from the board over its Wi-Fi. +//! +//! The hub, its settings and the modules built in are the board's own; only the network stack and the executor +//! differ. It listens on the address its one argument names, or on the board's port on every interface, and says +//! where on its first line of output, so a caller can ask for port 0 and learn the one it got. + +use std::error::Error; +use std::io::Write as _; +use std::net::SocketAddr; +use std::sync::Arc; +use std::time::SystemTime; + +use chrono::{DateTime, Utc}; +use edge_toolkit::ws_server::MessageRetention; +use et_ws_server::picoserve_server::serve; +use et_ws_service::hub::Stamps; +use et_ws_service::picoserve_hub::{self, Hub}; +use picoserve::time::TokioTimer; +use tokio::net::TcpListener; +use tokio::task::LocalSet; + +use crate::{HTTP_BUFFER, MAX_FRAME_SIZE, MAX_QUEUED, MODULES, PORT}; + +/// Ids `host-1`, `host-2`, ... in turn, and the host's clock, which unlike the board's knows the date. +#[derive(Default)] +struct HostStamps(u32); + +impl Stamps for HostStamps { + fn new_id(&mut self) -> String { + self.0 = self.0.wrapping_add(1); + format!("host-{}", self.0) + } + + fn now(&mut self) -> String { + DateTime::::from(SystemTime::now()).to_rfc3339() + } +} + +/// Serve the hub on the address the first argument names, or on [`PORT`] on every interface, until killed. +pub(crate) fn main() -> Result<(), Box> { + let address: SocketAddr = std::env::args() + .nth(1) + .unwrap_or_else(|| format!("0.0.0.0:{PORT}")) + .parse()?; + let runtime = tokio::runtime::Builder::new_current_thread().enable_all().build()?; + // One thread, as the board's executor has: picoserve's connections are not `Send`. + LocalSet::new().block_on(&runtime, Box::pin(accept(address))) +} + +/// Accept connections on `address`, serving each from its own task with the one hub they share. +#[expect( + clippy::future_not_send, + clippy::single_call_fn, + reason = "picoserve's futures hold Cells, run on one thread as on the board; a step apart from the runtime's setup" +)] +async fn accept(address: SocketAddr) -> Result<(), Box> { + let listener = TcpListener::bind(address).await?; + let mut stdout = std::io::stdout().lock(); + writeln!(stdout, "Serving http://{}/", listener.local_addr()?)?; + stdout.flush()?; + let hub = Arc::new(Hub::new( + HostStamps::default(), + MessageRetention::LatestPerSender, + MAX_FRAME_SIZE, + MAX_QUEUED, + )); + loop { + let (socket, peer) = listener.accept().await?; + let hub = Arc::clone(&hub); + let _serving = tokio::task::spawn_local(async move { + // Each connection's own, as each of the board's connection tasks holds one. + let config = picoserve_hub::config(); + let mut http_buffer = [0; HTTP_BUFFER]; + // A connection that ends badly is the client's to report; the hub carries on serving the others. + let _served = serve(&hub, &MODULES, socket, peer.ip(), TokioTimer, &config, &mut http_buffer).await; + }); + } +} diff --git a/services/ws-server-pico2w/src/main.rs b/services/ws-server-pico2w/src/main.rs index 12d763d6..28dcdc47 100644 --- a/services/ws-server-pico2w/src/main.rs +++ b/services/ws-server-pico2w/src/main.rs @@ -1,14 +1,31 @@ -//! et-ws-server as firmware for the Raspberry Pi Pico 2 W: the hub's `/ws`, `/health` and `/favicon.ico` on port -//! 8080, over the board's own Wi-Fi. +//! et-ws-server as firmware for the Raspberry Pi Pico 2 W: the hub's `/ws`, `/health`, `/favicon.ico` and the modules +//! built into it, on port 8080, over the board's own Wi-Fi. +//! +//! Built for any other target, the same hub serves the same modules from the host, over tokio, so what the board would +//! serve can be explored without one. #![cfg_attr(all(target_arch = "arm", target_os = "none"), no_std, no_main)] #![recursion_limit = "256"] #[cfg(all(target_arch = "arm", target_os = "none"))] extern crate alloc; +use edge_toolkit::ports::Services; + #[cfg(all(target_arch = "arm", target_os = "none"))] mod firmware; +#[cfg(not(all(target_arch = "arm", target_os = "none")))] +mod host; -/// Anywhere but on the board there is nothing to run; the binary exists there so the workspace's host builds compile. #[cfg(not(all(target_arch = "arm", target_os = "none")))] -fn main() {} +use self::host::main; + +/// The port the hub serves on, as the `std` server's plain-HTTP listener does. +const PORT: u16 = Services::InsecureWebSocketServer.port(); +/// The largest WebSocket frame the hub reads; each connection holds a buffer this size on the heap. +const MAX_FRAME_SIZE: usize = 8 * 1024; +/// How many messages for one connection the hub holds while it writes earlier ones, each up to `MAX_FRAME_SIZE`. +const MAX_QUEUED: usize = 2; +/// Each connection's buffer for the HTTP request head. +const HTTP_BUFFER: usize = 2048; + +include!(concat!(env!("OUT_DIR"), "/modules.rs")); diff --git a/services/ws-server-pico2w/tests/serves_modules.rs b/services/ws-server-pico2w/tests/serves_modules.rs new file mode 100644 index 00000000..246f4bb5 --- /dev/null +++ b/services/ws-server-pico2w/tests/serves_modules.rs @@ -0,0 +1,165 @@ +//! Explores the host build of the firmware the way a page or a runner would: every module it lists, fetched. +//! +//! It asks the server what it serves rather than being told, so it holds for whatever modules the build put in: none +//! in a plain test run, and a whole scenario's when that scenario's `firmware-host-test` task builds it. A module built +//! in has to serve its `package.json` and its `main`, and answer a revalidation without the body; one served elsewhere +//! has to redirect to the same path under its base, which is not fetched, since it is not this server's to serve. +#![cfg(test)] + +use std::io::{BufRead as _, BufReader, Read as _, Write as _}; +use std::net::{SocketAddr, TcpStream}; +use std::process::{Child, Stdio}; + +use edge_toolkit::ws::{ClientMessage, ServerMessage}; +use tungstenite::Message; + +/// The host binary, serving on a port of its choosing, killed when dropped. +struct Server { + child: Child, + address: SocketAddr, +} + +impl Server { + /// The binary, started on a free loopback port, once it has said which. + fn start() -> Self { + // The raw child, since the server is killed rather than left to exit with a status to check. + let mut child = std::process::Command::new(env!("CARGO_BIN_EXE_et-ws-server-pico2w")) + .arg("127.0.0.1:0") + .stdout(Stdio::piped()) + .spawn() + .unwrap(); + let mut line = String::default(); + let _read = BufReader::new(child.stdout.take().unwrap()) + .read_line(&mut line) + .unwrap(); + let address = line + .trim() + .strip_prefix("Serving http://") + .unwrap() + .trim_end_matches('/'); + Self { + address: address.parse().unwrap(), + child, + } + } + + /// The response to a `GET path` carrying `headers`, each a whole `Name: value\r\n` line. + fn get(&self, path: &str, headers: &str) -> Response { + let mut socket = TcpStream::connect(self.address).unwrap(); + let request = format!("GET {path} HTTP/1.1\r\nHost: board\r\n{headers}Connection: close\r\n\r\n"); + socket.write_all(request.as_bytes()).unwrap(); + let mut raw = Vec::new(); + let _read = socket.read_to_end(&mut raw).unwrap(); + let split = raw.windows(4).position(|window| window == b"\r\n\r\n").unwrap(); + let (head, body) = raw.split_at(split); + let head = String::from_utf8(head.to_vec()).unwrap(); + let status = head.split(' ').nth(1).unwrap().parse().unwrap(); + Response { + body: body.strip_prefix(b"\r\n\r\n").unwrap().to_vec(), + head, + status, + } + } +} + +impl Drop for Server { + fn drop(&mut self) { + let _killed = self.child.kill(); + let _waited = self.child.wait(); + } +} + +/// One HTTP response, its head kept as text. +struct Response { + body: Vec, + head: String, + status: u16, +} + +impl Response { + /// The value of the header `name`, or empty when the response has none. + fn header(&self, name: &str) -> &str { + let prefix = format!("{name}: "); + self.head + .lines() + .find_map(|line| line.strip_prefix(prefix.as_str())) + .unwrap_or_default() + } +} + +/// Fetch what a page loads of the module `name`, whose `package.json` came back as `package`. +#[expect( + clippy::single_call_fn, + reason = "the built-in branch of the exploration, kept apart from the walk" +)] +fn explore_embedded(server: &Server, name: &str, package: &Response) { + let manifest: serde_json::Value = serde_json::from_slice(&package.body).unwrap(); + if let Some(main) = manifest.get("main").and_then(serde_json::Value::as_str) { + let main = main.trim_start_matches(['.', '/']); + assert_eq!(server.get(&format!("/modules/{name}/{main}"), "").status, 200); + } + let revalidation = format!("If-None-Match: {}\r\n", package.header("ETag")); + let revalidated = server.get(&format!("/modules/{name}/package.json"), &revalidation); + assert_eq!(revalidated.status, 304); +} + +#[test] +fn it_serves_every_module_it_lists() { + let server = Server::start(); + assert_eq!(server.get("/health", "").status, 200); + + let list = server.get("/modules/", ""); + assert_eq!(list.status, 200); + let names: Vec = serde_json::from_slice(&list.body).unwrap(); + let mut built_in = 0_usize; + for name in &names { + let package = server.get(&format!("/modules/{name}/package.json"), ""); + if package.status == 307 { + let location = package.header("Location"); + assert!(location.starts_with("https://")); + assert!(location.ends_with(&format!("/{name}/package.json"))); + } else { + assert_eq!(package.status, 200); + explore_embedded(&server, name, &package); + built_in = built_in.saturating_add(1); + } + } + + // A deployment that builds modules in always names its page among them, and one that builds none serves no page. + let page = server.get("/", ""); + if built_in == 0 { + assert_eq!(page.status, 404); + } else { + assert_eq!(page.status, 200); + assert!(page.header("Content-Type").starts_with("text/html")); + } +} + +/// Send `message` to the hub over `socket`, and decode its reply. +fn exchange( + socket: &mut tungstenite::WebSocket>, + message: &ClientMessage, +) -> ServerMessage { + socket + .send(Message::text(serde_json::to_string(message).unwrap())) + .unwrap(); + ServerMessage::from_text_frame(socket.read().unwrap().to_text().unwrap()).unwrap() +} + +#[test] +fn an_agent_is_given_an_id_and_told_the_time() { + // The host's own ids and clock, which the board's hub stamps every agent and message with in its place. + let server = Server::start(); + let (mut socket, _response) = tungstenite::connect(format!("ws://{}/ws", server.address)).unwrap(); + + let connected = exchange(&mut socket, &ClientMessage::Connect { agent_id: None }); + assert!(matches!(connected, ServerMessage::ConnectAck { agent_id, .. } if agent_id == "host-1")); + let alive = ClientMessage::Alive { + timestamp: "now".to_owned(), + }; + let ServerMessage::Response { message } = exchange(&mut socket, &alive) else { + panic!("the alive message was not answered"); + }; + let (_said, time) = message.rsplit_once(" at ").unwrap(); + let _parsed = chrono::DateTime::parse_from_rfc3339(time).unwrap(); +} diff --git a/services/ws-server/Cargo.toml b/services/ws-server/Cargo.toml index 72cd8f91..8a64136f 100644 --- a/services/ws-server/Cargo.toml +++ b/services/ws-server/Cargo.toml @@ -78,7 +78,7 @@ std = [ "serde/std", "tracing/std", ] -## A `no_std` server for `/ws`, `/health` and `/favicon.ico` through picoserve, on any socket the embedder accepts. +## A `no_std` server for `/ws`, `/health`, `/favicon.ico` and built-in modules over picoserve, on any socket. picoserve-server = ["dep:picoserve", "et-ws-service/picoserve-hub"] ## Serves ws-module packages from disk under `/modules`, and the root module as the browser UI at `/`. modules = ["dep:et-modules-service", "std"] diff --git a/services/ws-server/src/lib.rs b/services/ws-server/src/lib.rs index 43ddc8b1..dbbdfe1e 100644 --- a/services/ws-server/src/lib.rs +++ b/services/ws-server/src/lib.rs @@ -20,6 +20,8 @@ pub mod config; #[cfg(feature = "std")] pub mod net; #[cfg(feature = "picoserve-server")] +pub mod picoserve_modules; +#[cfg(feature = "picoserve-server")] pub mod picoserve_server; pub mod routes; #[cfg(feature = "tls")] diff --git a/services/ws-server/src/picoserve_modules.rs b/services/ws-server/src/picoserve_modules.rs new file mode 100644 index 00000000..1f338607 --- /dev/null +++ b/services/ws-server/src/picoserve_modules.rs @@ -0,0 +1,313 @@ +//! ws-module packages built into the binary, served through picoserve where the `std` server serves them from disk. +//! +//! A board has no filesystem to scan, so its embedder hands [`serve`](crate::picoserve_server::serve) a [`Modules`] +//! whose every file is a `&'static [u8]`, usually through `include_bytes!` in generated code. Each answers a matching +//! `If-None-Match` with a 304, so a browser reloading the page does not fetch its wasm from the board again. The routes +//! are the `std` server's: `GET /modules/` lists the module names, `GET /modules/{name}/{path}` is one file of one +//! module, and the root module's files are served from `/`, `index.html` answering `/` itself. +//! +//! A package too big for the board's flash can be listed as served [`elsewhere`](Modules::elsewhere) instead: a request +//! for one of its files is redirected to the same path under [`elsewhere_url`](Modules::elsewhere_url), such as a CDN +//! that mirrors the npm registry, and its name is listed with the rest so a page cannot tell the difference. + +#[cfg(not(feature = "std"))] +use alloc::vec::Vec; + +use picoserve::ResponseSent; +use picoserve::io::{Read, Write}; +use picoserve::request::{Path, Request}; +use picoserve::response::{Content, IntoResponse as _, Json, Response, ResponseWriter, StatusCode}; +use picoserve::routing::{MethodNotAllowed, NotFound, PathRouter as _, PathRouterService, RequestHandler as _}; +use picoserve::url_encoded::UrlEncodedString; + +/// Where module files are served under, as the `std` server's modules service mounts them. +const MODULES_PREFIX: &str = "/modules"; +/// The file that answers a request for the root module's `/`. +const INDEX_FILE: &str = "index.html"; + +/// One file of a module, with what a browser needs to cache it. +/// +/// Its entity tag is worked out when the file is built in rather than here: hashing a module's wasm as a `const` would +/// cost the compiler minutes, and the board cycles it would cost at run time. +#[derive(Debug)] +#[non_exhaustive] +pub struct ModuleFile { + /// The `Content-Type` the file is served as. + pub content_type: &'static str, + /// The file's bytes. + pub body: &'static [u8], + /// A tag that changes whenever `body` does, sent unquoted as the file's `ETag`. + pub etag: &'static str, +} + +impl ModuleFile { + /// The file `body`, served as `content_type` and tagged `etag`. + #[must_use] + pub const fn new(content_type: &'static str, body: &'static [u8], etag: &'static str) -> Self { + Self { + content_type, + body, + etag, + } + } + + /// Whether an `If-None-Match` header of `tags` already names this file, so a 304 answers it. + fn is_matched_by(&self, tags: &[u8]) -> bool { + tags.split(|byte| *byte == b',').any(|tag| { + let tag = tag.trim_ascii(); + let tag = tag.strip_prefix(b"W/").unwrap_or(tag); + tag == b"*" || tag.strip_prefix(b"\"").and_then(|tag| tag.strip_suffix(b"\"")) == Some(self.etag.as_bytes()) + }) + } +} + +impl Content for &ModuleFile { + fn content_type(&self) -> &'static str { + self.content_type + } + + fn content_length(&self) -> usize { + self.body.len() + } + + async fn write_content(self, mut writer: W) -> Result<(), W::Error> + where + W: Write, + { + writer.write_all(self.body).await + } +} + +/// One module's files, each by its path within the module, such as `pkg.js` or `assets/model.bin`. +#[derive(Debug)] +#[non_exhaustive] +pub struct Module { + /// The module's name, exactly as its `package.json` declares it, scope and all. + pub name: &'static str, + /// Every file of the module, by its path relative to the module's root, without a leading `/`. + pub files: &'static [(&'static str, ModuleFile)], +} + +impl Module { + /// The module `name`, made of `files`. + #[must_use] + pub const fn new(name: &'static str, files: &'static [(&'static str, ModuleFile)]) -> Self { + Self { name, files } + } + + /// The file `path` names within this module, a request path relative to the module's root with its leading `/`. + /// + /// The module's own root is its `index.html` only when `indexed`, as only the root module's is. + fn file(&self, path: UrlEncodedString<'_>, indexed: bool) -> Target<'_> { + // Both callers hand over a path that starts with its `/`, so there is always one to take off. + let relative = path.strip_prefix("/").unwrap_or(path); + let relative = if indexed && relative.is_empty() { + UrlEncodedString(INDEX_FILE) + } else { + relative + }; + self.files + .iter() + .find(|(name, _)| relative == *name) + .map_or(Target::Missing, |(_, file)| Target::File(file)) + } +} + +/// Every module a board serves: those built into it, and those it redirects to [`elsewhere_url`](Self::elsewhere_url). +#[derive(Debug)] +#[non_exhaustive] +pub struct Modules { + /// The modules built into the binary. + pub embedded: &'static [Module], + /// The name of the embedded module served at `/`, or empty to serve nothing there, as `MODULES_ROOT` does. + pub root: &'static str, + /// The names of the modules served from [`elsewhere_url`](Self::elsewhere_url) rather than from the binary. + pub elsewhere: &'static [&'static str], + /// The base a request for a module in [`elsewhere`](Self::elsewhere) is redirected to, its path appended as is. + pub elsewhere_url: &'static str, +} + +impl Modules { + /// No modules at all: `/modules/` lists none and every other module path is not found. + pub const NONE: Self = Self::new(&[], "", &[], ""); + + /// The modules `embedded`, `root` among them, with `elsewhere` redirected to `elsewhere_url`. + #[must_use] + pub const fn new( + embedded: &'static [Module], + root: &'static str, + elsewhere: &'static [&'static str], + elsewhere_url: &'static str, + ) -> Self { + Self { + embedded, + root, + elsewhere, + elsewhere_url, + } + } + + /// Every module name, embedded or not, sorted as the `std` server lists them. + fn names(&self) -> Vec<&'static str> { + let mut names: Vec<&'static str> = self.embedded.iter().map(|module| module.name).collect(); + names.extend_from_slice(self.elsewhere); + names.sort_unstable(); + names + } + + /// The embedded file `path` names, a path under `/modules` that starts with its module's name. + fn embedded_file(&self, path: UrlEncodedString<'_>) -> Target<'_> { + let Some(path) = path.strip_prefix("/") else { + return Target::Missing; + }; + for module in self.embedded { + if let Some(rest) = path.strip_prefix(module.name) + && rest.0.starts_with('/') + { + return module.file(rest, false); + } + } + Target::Missing + } + + /// Whether `path` names a file of a module served elsewhere. + fn is_elsewhere(&self, path: UrlEncodedString<'_>) -> bool { + let Some(path) = path.strip_prefix("/") else { + return false; + }; + !self.elsewhere_url.is_empty() + && self + .elsewhere + .iter() + .any(|name| path.strip_prefix(name).is_some_and(|rest| rest.0.starts_with('/'))) + } +} + +/// The routes [`Modules`] answers, as the fallback of a server's other routes: a request none of them matched. +pub(crate) struct ModuleRoutes<'modules>(&'modules Modules); + +impl<'modules> ModuleRoutes<'modules> { + /// The routes `modules` answers. + #[expect( + clippy::single_call_fn, + reason = "the constructor that keeps the field private to this module, for the one server that mounts it" + )] + pub(crate) const fn new(modules: &'modules Modules) -> Self { + Self(modules) + } +} + +impl PathRouterService for ModuleRoutes<'_> { + #[expect( + clippy::future_not_send, + reason = "picoserve's futures hold Cells; its executors run each connection on one thread" + )] + async fn call_path_router_service( + &self, + state: &State, + current_path_parameters: CurrentPathParameters, + path: Path<'_>, + request: Request<'_, R>, + response_writer: W, + ) -> Result + where + R: Read, + W: ResponseWriter, + { + let modules = self.0; + let target = Target::named_by(modules, UrlEncodedString(path.encoded())); + if !matches!(target, Target::Missing) && !request.parts.method().eq_ignore_ascii_case("get") { + return MethodNotAllowed + .call_request_handler(state, current_path_parameters, request, response_writer) + .await; + } + match target { + Target::Missing => { + NotFound + .call_path_router(state, current_path_parameters, path, request, response_writer) + .await + } + Target::List => { + Json(modules.names()) + .write_to(request.body_connection.finalize().await?, response_writer) + .await + } + Target::File(file) => { + let etag = format_args!("\"{}\"", file.etag); + let cached = request + .parts + .headers() + .get("If-None-Match") + .is_some_and(|tags| file.is_matched_by(tags.as_raw())); + let connection = request.body_connection.finalize().await?; + if cached { + Response::new(StatusCode::NOT_MODIFIED, "") + .with_header("ETag", etag) + .write_to(connection, response_writer) + .await + } else { + Response::ok(file) + .with_header("ETag", etag) + .write_to(connection, response_writer) + .await + } + } + Target::Elsewhere(module_path) => { + Response::new(StatusCode::TEMPORARY_REDIRECT, "") + .with_header("Location", format_args!("{}{module_path}", modules.elsewhere_url)) + .write_to(request.body_connection.finalize().await?, response_writer) + .await + } + } + } +} + +/// What a request path names among [`Modules`]. +enum Target<'request> { + /// `/modules/`, the list of module names. + List, + /// One embedded file. + File(&'request ModuleFile), + /// A file of a module served elsewhere, by its path under `/modules`. + Elsewhere(&'request str), + /// Nothing these modules serve. + Missing, +} + +impl<'request> Target<'request> { + /// What `path`, a whole request path, names among `modules`. + #[expect( + clippy::single_call_fn, + reason = "the lookup step of ModuleRoutes, kept apart from writing what it finds" + )] + fn named_by(modules: &'request Modules, path: UrlEncodedString<'request>) -> Self { + let Some(module_path) = path.strip_prefix(MODULES_PREFIX) else { + return Self::root_file(modules, path); + }; + if module_path.is_empty() || module_path.0 == "/" { + return Self::List; + } + let file = modules.embedded_file(module_path); + if matches!(file, Self::File(_)) { + return file; + } + if modules.is_elsewhere(module_path) { + return Self::Elsewhere(module_path.0); + } + // Not a module path after all, such as `/modulesfoo`, which the root module may still hold. + if module_path.0.starts_with('/') { + Self::Missing + } else { + Self::root_file(modules, path) + } + } + + /// The root module's file at `path`. + fn root_file(modules: &'request Modules, path: UrlEncodedString<'request>) -> Self { + modules + .embedded + .iter() + .find(|module| !modules.root.is_empty() && module.name == modules.root) + .map_or(Self::Missing, |root| root.file(path, true)) + } +} diff --git a/services/ws-server/src/picoserve_server.rs b/services/ws-server/src/picoserve_server.rs index 84cf0659..2dd3d471 100644 --- a/services/ws-server/src/picoserve_server.rs +++ b/services/ws-server/src/picoserve_server.rs @@ -1,8 +1,10 @@ -//! The `no_std` ws-server: the hub's `/ws`, with `/health` and `/favicon.ico` beside it, served through picoserve. +//! The `no_std` ws-server: the hub's `/ws`, with `/health`, `/favicon.ico` and the modules beside it, served through +//! picoserve. //! //! As with [`Hub::serve`], the embedder owns the network stack and the executor, accepts each connection, and runs -//! [`serve`] for it with the one [`Hub`] every connection shares. The services the `std` server mounts besides the hub -//! -- storage, modules, websockify -- need a filesystem or an OS, and are not served here. +//! [`serve`] for it with the one [`Hub`] every connection shares. The modules are those built into the binary, as +//! [`Modules`] describes. The other services the `std` server mounts besides the hub -- storage, websockify -- need a +//! filesystem or an OS, and are not served here. use core::net::IpAddr; @@ -13,12 +15,13 @@ use picoserve::response::{Json, StatusCode}; use picoserve::routing::get; use picoserve::{Config, Router, Server, Timer}; +use crate::picoserve_modules::{ModuleRoutes, Modules}; use crate::routes::HealthResponse; /// Serve one accepted connection from `client_ip` until it closes. /// -/// `GET /health` answers what the `std` server's does, `GET /favicon.ico` answers 204, and `GET /ws` is `hub`'s, as -/// [`Hub::routes`] describes; every other request is answered 404. `config` is usually +/// `GET /health` answers what the `std` server's does, `GET /favicon.ico` answers 204, and `GET /ws` is `hub`'s, +/// as [`Hub::routes`] describes. Every other request is one of `modules`', or answered 404. `config` is usually /// [`et_ws_service::picoserve_hub::config`], and `http_buffer` holds the request head. /// /// As for [`Hub::serve`], the crate awaiting this one needs `#![recursion_limit = "256"]`. @@ -28,6 +31,7 @@ use crate::routes::HealthResponse; )] pub async fn serve( hub: &Hub, + modules: &Modules, socket: S, client_ip: IpAddr, timer: M, @@ -40,7 +44,7 @@ where S: Socket, { let app = hub.routes( - Router::new() + Router::from_service(ModuleRoutes::new(modules)) .route("/favicon.ico", get(async || StatusCode::NO_CONTENT)) .route("/health", get(async || Json(HealthResponse::healthy()))), client_ip, diff --git a/services/ws-server/tests/picoserve_server.rs b/services/ws-server/tests/picoserve_server.rs index 384e9f5e..dc06fd48 100644 --- a/services/ws-server/tests/picoserve_server.rs +++ b/services/ws-server/tests/picoserve_server.rs @@ -1,5 +1,6 @@ //! The `no_std` ws-server over real TCP sockets, with tokio standing in for an embedded executor and network stack: -//! `/health` and `/favicon.ico` answer as the `std` server's do, and `/ws` is the hub. +//! `/health` and `/favicon.ico` answer as the `std` server's do, `/ws` is the hub, and the modules built into the +//! binary are served from `/modules` and `/`. #![cfg(test)] #![recursion_limit = "256"] @@ -9,6 +10,7 @@ use std::time::Duration; use edge_toolkit::ws::{ClientMessage, ConnectStatus, ServerMessage}; use edge_toolkit::ws_server::MessageRetention; +use et_ws_server::picoserve_modules::{Module, ModuleFile, Modules}; use et_ws_server::picoserve_server::serve; use et_ws_server::routes::HealthResponse; use et_ws_service::hub::Stamps; @@ -24,6 +26,29 @@ use tokio_tungstenite::tungstenite::Message; /// How long a test waits for an answer before failing rather than hanging. const PATIENCE: Duration = Duration::from_secs(5); +/// A page served at `/`, a scoped module beside it, and a package redirected to a CDN. +static MODULES: Modules = Modules::new( + &[ + Module::new( + "page", + &[ + ("index.html", ModuleFile::new("text/html", b"

page

", "index")), + ("app.js", ModuleFile::new("text/javascript", b"app", "app")), + ], + ), + Module::new( + "@scope/math", + &[ + ("package.json", ModuleFile::new("application/json", b"{}", "package")), + ("pkg/math.wasm", ModuleFile::new("application/wasm", b"\0asm", "wasm")), + ], + ), + ], + "page", + &["big"], + "https://cdn.example/npm", +); + /// Ids `id-1`, `id-2`, ... in turn, and a fixed time. #[derive(Default)] struct Counter(u32); @@ -39,12 +64,8 @@ impl Stamps for Counter { } } -/// The server on a fresh loopback port, one task per connection as an embedder would, and its address. -#[expect( - clippy::single_call_fn, - reason = "the embedder's accept loop, kept apart from what the test asserts" -)] -async fn served() -> SocketAddr { +/// The server of `modules` on a fresh loopback port, one task per connection as an embedder would, and its address. +async fn served(modules: &'static Modules) -> SocketAddr { let hub = Arc::new(Hub::new(Counter::default(), MessageRetention::All, 1024, 16)); let listener = TcpListener::bind((Ipv4Addr::LOCALHOST, 0)).await.unwrap(); let address = listener.local_addr().unwrap(); @@ -54,7 +75,16 @@ async fn served() -> SocketAddr { let hub = Arc::clone(&hub); let _serving = tokio::task::spawn_local(async move { let mut http_buffer = [0; 2048]; - let _served = serve(&hub, socket, peer.ip(), TokioTimer, &config(), &mut http_buffer).await; + let _served = serve( + &hub, + modules, + socket, + peer.ip(), + TokioTimer, + &config(), + &mut http_buffer, + ) + .await; }); } }); @@ -63,8 +93,18 @@ async fn served() -> SocketAddr { /// The whole response to a bodiless `GET path`, headers and body, as text. async fn get(address: SocketAddr, path: &str) -> String { + request(address, "GET", path).await +} + +/// The whole response to a bodiless `method path`, headers and body, as text. +async fn request(address: SocketAddr, method: &str, path: &str) -> String { + send(address, &format!("{method} {path} HTTP/1.1\r\nHost: hub\r\n")).await +} + +/// The whole response to `head`, a request line and headers to which the closing ones are added. +async fn send(address: SocketAddr, head: &str) -> String { let mut socket = TcpStream::connect(address).await.unwrap(); - let request = format!("GET {path} HTTP/1.1\r\nHost: hub\r\nConnection: close\r\n\r\n"); + let request = format!("{head}Connection: close\r\n\r\n"); socket.write_all(request.as_bytes()).await.unwrap(); let mut response = String::default(); let _read = tokio::time::timeout(PATIENCE, socket.read_to_string(&mut response)) @@ -76,11 +116,11 @@ async fn get(address: SocketAddr, path: &str) -> String { #[tokio::test] async fn it_serves_health_favicon_and_the_hub() { - // Boxed because its size depends on how the build unifies features: over 16 KiB with the whole workspace, under - // it alone, so an expectation of `clippy::large_futures` could hold in only one of the two. + // Boxed because its size depends on how the build unifies features: over 16 KiB with the whole workspace, under it + // alone, so an expectation of `clippy::large_futures` could hold in only one of the two. LocalSet::new() .run_until(Box::pin(async { - let address = served().await; + let address = served(&MODULES).await; let health = get(address, "/health").await; assert!(health.starts_with("HTTP/1.1 200")); @@ -108,3 +148,83 @@ async fn it_serves_health_favicon_and_the_hub() { })) .await; } + +/// The body of `response`, after its head. +fn body(response: &str) -> &str { + response.split_once("\r\n\r\n").unwrap().1 +} + +#[tokio::test] +async fn it_serves_the_embedded_modules() { + LocalSet::new() + .run_until(Box::pin(async { + let address = served(&MODULES).await; + + let list = get(address, "/modules/").await; + assert!(list.starts_with("HTTP/1.1 200")); + let names: Vec = serde_json::from_str(body(&list)).unwrap(); + assert_eq!(names, ["@scope/math", "big", "page"]); + assert!(get(address, "/modules").await.starts_with("HTTP/1.1 200")); + + let page = get(address, "/").await; + assert!(page.contains("Content-Type: text/html")); + assert_eq!(body(&page), "

page

"); + assert_eq!(body(&get(address, "/app.js").await), "app"); + assert_eq!(body(&get(address, "/modules/page/app.js").await), "app"); + assert!(get(address, "/modules/page/").await.starts_with("HTTP/1.1 404")); + + let wasm = get(address, "/modules/@scope/math/pkg/math.wasm").await; + assert!(wasm.contains("Content-Type: application/wasm")); + assert!(wasm.contains("ETag: \"wasm\"\r\n")); + assert_eq!(body(&wasm), "\0asm"); + let wasm_path = "GET /modules/@scope/math/pkg/math.wasm HTTP/1.1\r\nHost: hub\r\n"; + for tags in ["\"wasm\"", "\"other\", W/\"wasm\"", "*"] { + let revalidated = send(address, &format!("{wasm_path}If-None-Match: {tags}\r\n")).await; + assert!(revalidated.starts_with("HTTP/1.1 304")); + } + let changed = send(address, &format!("{wasm_path}If-None-Match: \"other\", wasm\r\n")).await; + assert_eq!(body(&changed), "\0asm"); + let encoded = get(address, "/modules/%40scope/math/package.json").await; + assert_eq!(body(&encoded), "{}"); + assert!( + get(address, "/modules/@scope/math/missing.js") + .await + .starts_with("HTTP/1.1 404") + ); + assert!( + get(address, "/modules/@scope/mathematics/x") + .await + .starts_with("HTTP/1.1 404") + ); + assert!(get(address, "/modulesx").await.starts_with("HTTP/1.1 404")); + + let redirect = get(address, "/modules/big/dist/big.js").await; + assert!(redirect.starts_with("HTTP/1.1 307")); + assert!(redirect.contains("Location: https://cdn.example/npm/big/dist/big.js\r\n")); + assert!(get(address, "/modules/bigger/x.js").await.starts_with("HTTP/1.1 404")); + + assert!(request(address, "POST", "/app.js").await.starts_with("HTTP/1.1 405")); + assert!(request(address, "POST", "/missing").await.starts_with("HTTP/1.1 404")); + })) + .await; +} + +#[tokio::test] +async fn modules_assembled_at_run_time_serve_as_built_in_ones_do() { + // Read from somewhere only known once running, rather than built in, and so leaked to live as long as the server. + let file = ModuleFile::new("text/plain", Box::leak(b"run".to_vec().into_boxed_slice()), "run"); + let files: &'static [(&'static str, ModuleFile)] = Box::leak(Box::new([("a.txt", file)])); + let module = Module::new("assembled", files); + // No page at `/`, and a module named as served elsewhere with nowhere to send it. + let modules: &'static Modules = Box::leak(Box::new(Modules::new(Box::leak(Box::new([module])), "", &["far"], ""))); + LocalSet::new() + .run_until(Box::pin(async { + let address = served(modules).await; + assert_eq!(body(&get(address, "/modules/assembled/a.txt").await), "run"); + assert!(get(address, "/").await.starts_with("HTTP/1.1 404")); + assert!(get(address, "/modules/far/x.js").await.starts_with("HTTP/1.1 404")); + let names: Vec = serde_json::from_str(body(&get(address, "/modules/").await)).unwrap(); + assert_eq!(names, ["assembled", "far"]); + })) + .await; +} diff --git a/utilities/cli/Cargo.toml b/utilities/cli/Cargo.toml index c1d731d1..930c25fa 100644 --- a/utilities/cli/Cargo.toml +++ b/utilities/cli/Cargo.toml @@ -23,6 +23,7 @@ edge-toolkit = { workspace = true, features = ["std"] } et-org.workspace = true et-path.workspace = true et-recovery-supervisor.workspace = true +et-sanitise.workspace = true fake.workspace = true fs-err.workspace = true jiff.workspace = true diff --git a/utilities/cli/HELP.md b/utilities/cli/HELP.md index fa243c66..f11bf904 100644 --- a/utilities/cli/HELP.md +++ b/utilities/cli/HELP.md @@ -39,7 +39,14 @@ Generate deployment config from a cluster input YAML Default value: `mise` - Possible values: `mise`, `dekit`, `process-compose`, `docker-compose`, `k3s` + Possible values: + - `mise` + - `dekit` + - `process-compose` + - `docker-compose` + - `k3s` + - `firmware-pico2`: + The hub alone, as firmware for a Raspberry Pi Pico 2 W ## `et-cli regen-verification` diff --git a/utilities/cli/src/deployment_types/firmware.rs b/utilities/cli/src/deployment_types/firmware.rs new file mode 100644 index 00000000..9ed138c1 --- /dev/null +++ b/utilities/cli/src/deployment_types/firmware.rs @@ -0,0 +1,144 @@ +use std::path::Path; + +use et_path::{absolute_from, relative_path_from}; +use fs_err as fs; +use toml::{Table, Value}; + +use super::mise::{mise_task, registry_and_modules, wrap_module_paths}; +use crate::error::CliError; +use crate::input::{ArtifactSource, Board, ClusterInput}; +use crate::{ModuleSource, ScenarioModules, cluster_module_names, resolve_cluster_modules}; + +/// The task that builds the firmware with the scenario's modules in it. +const BUILD_TASK: &str = "firmware-build"; +/// The task that loads the firmware onto the board. +pub(crate) const FLASH_TASK: &str = "firmware-flash"; +/// The task that serves the same modules from the host, and fetches each, with no board needed. +pub(crate) const HOST_TEST_TASK: &str = "firmware-host-test"; +/// Where npm packages too big for the board are fetched from instead: a CDN mirroring the npm registry by package path. +pub(crate) const ELSEWHERE_URL: &str = "https://cdn.jsdelivr.net/npm"; + +/// Refuse a scenario `board` cannot run: it holds one hub and the modules built into it, and nothing else. +/// +/// A runner needs an OS to run on, a second machine a backplane, and a published deployment modules staged from a +/// release, none of which the firmware has. A second hub needs a second machine, so is refused with it. Checked before +/// anything is written, as every format's refusals are. +pub(crate) fn validate_firmware(cluster: &ClusterInput, board: Board) -> Result<(), CliError> { + let reason = if cluster.agents.iter().any(|agent| agent.runner.is_some()) { + "starts a runner" + } else if !cluster.machines.is_empty() { + "spans several machines" + } else if matches!(cluster.artifact_source, ArtifactSource::Published) { + "uses published artifacts, since the firmware is built from this repository" + } else { + return Ok(()); + }; + Err(CliError::BoardCannotRun { + board: board.spec().title, + reason, + }) +} + +/// Write the `mise.firmware-.toml` that builds the scenario's hub as `board` firmware and loads it onto one. +/// +/// The board serves the modules the scenario names, and what they depend on, from flash: each module from this +/// repository or the scenario's own `module_paths:` is built into the firmware, and each npm package a mise tool +/// stages is redirected to a CDN instead, since onnxruntime-web alone is bigger than a board's flash. Selected with +/// `MISE_ENV=firmware-`, so it sits beside a `mise.toml` without either loading the other's tasks unasked. +pub(crate) fn generate_firmware_deployment( + cluster: &ClusterInput, + output_dir: &Path, + board: Board, +) -> Result<(), CliError> { + validate_firmware(cluster, board)?; + let board = board.spec(); + let workspace_root = edge_toolkit::config::get_project_root(); + let output_abs = absolute_from(&workspace_root, output_dir); + let workspace_rel = relative_path_from(&output_abs, &workspace_root); + let (modules_paths, env) = module_env(cluster, &workspace_root)?; + // Both run at the repository root, where `$PWD` names the module paths absolutely, as the build needs. + let module_task = |description: &str, command: &str| { + let run = format!("{modules_paths}export MODULES_PATHS\n{command}\n"); + mise_task( + None, + Some(description), + Some(&workspace_rel), + Some(&run), + None, + Some(env.clone()), + ) + }; + let build = module_task( + &format!("Build the {} firmware serving {}", board.title, cluster.cluster_name), + &format!("mise -E rust run {}", board.build_task), + ); + let host_test = module_task( + &format!( + "Serve {}'s modules from this host as the board would, and fetch each", + cluster.cluster_name + ), + &format!("mise run cargo-test-pkg {}", board.package), + ); + + let mut depends = Table::new(); + let _previous: Option = depends.insert( + "depends".to_owned(), + Value::Array(vec![Value::String(BUILD_TASK.to_owned())]), + ); + let flash_run = format!("{} {}", board.load_command, board.image); + let flash_description = format!("Load the firmware onto a {}, and start it", board.title); + let flash = mise_task( + None, + Some(&flash_description), + Some(&workspace_rel), + Some(&flash_run), + Some(depends), + None, + ); + + let mut tasks = Table::new(); + let _previous: Option = tasks.insert(BUILD_TASK.to_owned(), Value::Table(build)); + let _previous: Option = tasks.insert(FLASH_TASK.to_owned(), Value::Table(flash)); + let _previous: Option = tasks.insert(HOST_TEST_TASK.to_owned(), Value::Table(host_test)); + let mut root = Table::new(); + let _previous: Option = root.insert("tasks".to_owned(), Value::Table(tasks)); + fs::write( + output_dir.join(board.output_type.output_file_name()), + toml::to_string(&Value::Table(root))?, + )?; + Ok(()) +} + +/// The `MODULES_PATHS` assignments naming the modules built in, and the rest of the `MODULES_*` environment. +#[expect( + clippy::single_call_fn, + reason = "the module-resolving step of generate_firmware_deployment, kept apart from writing the file" +)] +fn module_env(cluster: &ClusterInput, workspace_root: &Path) -> Result<(String, Table), CliError> { + let ws_server_dir = workspace_root.join("services/ws-server"); + let module_names = cluster_module_names(cluster); + // With no runner allowed, every agent is one the page runs, so the board always serves the page. + let scenario = ScenarioModules::new(&ws_server_dir, &module_names, true).with_module_paths(&cluster.module_paths); + + let (registry, modules) = registry_and_modules(&scenario); + let mut embedded = Vec::new(); + let mut elsewhere = Vec::new(); + for entry in resolve_cluster_modules(®istry, &modules)? { + match entry.source { + ModuleSource::Repo(path) => embedded.push(format!("$PWD/{path}")), + ModuleSource::Scenario { dir, .. } => embedded.push(dir.display().to_string()), + ModuleSource::MiseTool { package, .. } => elsewhere.push(package), + } + } + embedded.sort(); + elsewhere.sort(); + + let mut env = Table::new(); + let mut set = |name: &str, value: String| { + let _previous: Option = env.insert(name.to_owned(), Value::String(value)); + }; + set("MODULES_ROOT", super::hub_root_module(&ws_server_dir)); + set("MODULES_ELSEWHERE", elsewhere.join(", ")); + set("MODULES_ELSEWHERE_URL", ELSEWHERE_URL.to_owned()); + Ok((wrap_module_paths(&embedded), env)) +} diff --git a/utilities/cli/src/deployment_types/mise.rs b/utilities/cli/src/deployment_types/mise.rs index 20873a95..525d631d 100644 --- a/utilities/cli/src/deployment_types/mise.rs +++ b/utilities/cli/src/deployment_types/mise.rs @@ -17,8 +17,8 @@ use crate::input::{ArtifactSource, ClusterInput}; use crate::placement::{hubs, shares_hubs}; use crate::{ COLLECTOR_SETTINGS, COLLECTOR_USERNAME, MODULE_SCOPE, ModuleRegistryEntry, ModuleSource, RunnerInstance, - cluster_module_names, escape_for_double_quotes, module_registry, resolve_cluster_modules, resolve_module_paths, - runner_crate, scenario_runners, + cluster_module_names, module_registry, resolve_cluster_modules, resolve_module_paths, runner_crate, + scenario_runners, }; /// Crate whose binary serves the hub, which a published deployment installs in place of building it. @@ -474,7 +474,7 @@ fn published_module_paths(scenario: &ScenarioModules<'_>, output_abs: &Path) -> let mut paths: Vec = resolved .into_iter() .filter_map(|entry| match entry.source { - ModuleSource::Scenario { dir, .. } => Some(escape_for_double_quotes(&relative_path_from(output_abs, &dir))), + ModuleSource::Scenario { dir, .. } => Some(relative_path_from(output_abs, &dir)), ModuleSource::MiseTool { tool, .. } if !tool.starts_with(STAGED_PREFIX) => Some(entry.mise_path), ModuleSource::Repo(_) | ModuleSource::MiseTool { .. } => None, }) @@ -519,7 +519,9 @@ impl<'scenario> ScenarioModules<'scenario> { } /// The registry to resolve against, and the module list to resolve -- what both resolutions here start from. -fn registry_and_modules(scenario: &ScenarioModules<'_>) -> (BTreeMap, Vec) { +pub(super) fn registry_and_modules( + scenario: &ScenarioModules<'_>, +) -> (BTreeMap, Vec) { let project_root = edge_toolkit::config::get_project_root(); ( module_registry(&project_root, scenario.ws_server_dir, scenario.module_paths), @@ -540,7 +542,7 @@ pub fn scenario_module_paths(scenario: &ScenarioModules<'_>) -> Result String { +pub(super) fn wrap_module_paths(paths: &[String]) -> String { #[derive(Debug)] struct PathFragment<'path> { path: &'path str, @@ -585,7 +587,7 @@ fn wrap_module_paths(paths: &[String]) -> String { out } -fn mise_task( +pub(super) fn mise_task( alias: Option<&str>, description: Option<&str>, dir: Option<&str>, diff --git a/utilities/cli/src/deployment_types/mod.rs b/utilities/cli/src/deployment_types/mod.rs index 435b2f64..174bfb2d 100644 --- a/utilities/cli/src/deployment_types/mod.rs +++ b/utilities/cli/src/deployment_types/mod.rs @@ -1,5 +1,6 @@ mod dekit; mod docker_compose; +mod firmware; mod hub; mod hub_sharing; mod k3s; @@ -12,6 +13,10 @@ mod yaml; pub use self::dekit::generate_dekit_deployment; pub(crate) use self::docker_compose::{HUB_IMAGE_BUILD, SCHEDULER_SERVICE, scheduler_service}; pub use self::docker_compose::{docker_image_module_paths, generate_docker_compose_deployment}; +pub(crate) use self::firmware::{ + ELSEWHERE_URL as FIRMWARE_ELSEWHERE_URL, FLASH_TASK as FIRMWARE_FLASH_TASK, + HOST_TEST_TASK as FIRMWARE_HOST_TEST_TASK, generate_firmware_deployment, validate_firmware, +}; pub(crate) use self::hub::{hub_root_module, serves_a_page}; pub(crate) use self::hub_sharing::{BACKPLANE_TASK, SHARED_STORAGE_TASK, STORAGE_SECRET_ENVS}; pub use self::k3s::generate_k3s_deployment; diff --git a/utilities/cli/src/error.rs b/utilities/cli/src/error.rs index c63aea4b..5bdd8879 100644 --- a/utilities/cli/src/error.rs +++ b/utilities/cli/src/error.rs @@ -172,6 +172,12 @@ pub enum CliError { )] ScheduleUnsupported { agent: String, output_type: String }, + #[error("The {board} firmware runs one hub and nothing else, so it cannot deploy a scenario that {reason}")] + BoardCannotRun { board: &'static str, reason: &'static str }, + + #[error("Module path {0:?} has a name a shell would interpret or a filesystem would not hold; rename it")] + UnsafeModulePath(PathBuf), + #[error("Agent {agent:?} is listed on both machine {first:?} and machine {second:?}; an agent runs on one")] AgentOnTwoMachines { agent: String, diff --git a/utilities/cli/src/input.rs b/utilities/cli/src/input.rs index 0b4b02d2..ad04ce38 100644 --- a/utilities/cli/src/input.rs +++ b/utilities/cli/src/input.rs @@ -39,6 +39,65 @@ pub enum OutputType { #[serde(rename = "docker-compose", alias = "docker_compose")] DockerCompose, K3s, + /// The hub alone, as firmware for a Raspberry Pi Pico 2 W. + #[serde(rename = "firmware-pico2", alias = "firmware_pico2")] + FirmwarePico2, +} + +/// A board a `firmware-*` format builds the hub for, each the board of exactly one format. +/// +/// Only boards with a network of their own are supported, so a board's name leaves out the suffix its vendor gives +/// the networked variant: `pico2` is the Pico 2 W. +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub(crate) enum Board { + /// The Raspberry Pi Pico 2 W, over its own Wi-Fi. + Pico2, +} + +/// What building, loading and describing one board's firmware takes. +#[derive(Clone, Copy, Debug)] +pub(crate) struct BoardSpec { + /// The format that builds firmware for the board. + pub output_type: OutputType, + /// What a person calls the board. + pub title: &'static str, + /// The repository task that builds the firmware, reading the modules from `MODULES_*`. + pub build_task: &'static str, + /// The firmware's package, whose build for the host serves the same modules from there instead of the board. + pub package: &'static str, + /// The firmware image that task leaves, relative to the repository root. + pub image: &'static str, + /// The command that loads `image` onto the board, followed by its path. + pub load_command: &'static str, + /// How the board is made ready for `load_command`. + pub load_preparation: &'static str, + /// What the build needs told about the network the board joins, as a sentence. + pub network_setup: &'static str, + /// How big the board's flash is, which bounds what can be built in. + pub flash: &'static str, +} + +impl Board { + /// Every board, each with a `firmware-*` format of its own. + pub(crate) const ALL: &'static [Self] = &[Self::Pico2]; + + /// What building, loading and describing this board's firmware takes. + #[must_use] + pub(crate) const fn spec(self) -> BoardSpec { + match self { + Self::Pico2 => BoardSpec { + output_type: OutputType::FirmwarePico2, + title: "Raspberry Pi Pico 2 W", + build_task: "build-firmware-pico2w", + package: "et-int-ws-server-pico2w", + image: "target/thumbv8m.main-none-eabihf/release/et-ws-server-pico2w", + load_command: "picotool load -x -t elf", + load_preparation: "hold the board's BOOTSEL button while plugging it in", + network_setup: "Set `WIFI_NETWORK` and `WIFI_PASSWORD` to the Wi-Fi network the board joins.", + flash: "4 MiB", + }, + } + } } impl OutputType { @@ -46,13 +105,15 @@ impl OutputType { /// /// The order is what running each takes and what it offers. The three that need no container runtime come first: /// mise, a plain CLI; dekit, a TUI; then process-compose, a TUI that can also schedule. Compose follows because it - /// needs Docker, and k3s comes last because it needs a Kubernetes cluster such as Rancher Desktop's. + /// needs Docker, and k3s because it needs a Kubernetes cluster such as Rancher Desktop's. The firmware is last, + /// because it needs a board, and because it deploys only the hub. pub const ALL: &'static [Self] = &[ Self::Mise, Self::Dekit, Self::ProcessCompose, Self::DockerCompose, Self::K3s, + Self::FirmwarePico2, ]; /// The file this format is written to, beside the others in the deployment directory. @@ -64,6 +125,7 @@ impl OutputType { Self::K3s => "k3s.yaml", Self::Dekit => "dekit.yaml", Self::ProcessCompose => "process-compose.yaml", + Self::FirmwarePico2 => "mise.firmware-pico2.toml", } } @@ -76,15 +138,17 @@ impl OutputType { Self::K3s => "k3s", Self::Dekit => "dekit", Self::ProcessCompose => "process-compose", + Self::FirmwarePico2 => "firmware-pico2", } } /// Whether this format can start an agent on its `schedule:`. /// - /// mise and dekit have no scheduler of their own, so a scheduled agent is refused rather than run continuously. + /// mise and dekit have no scheduler of their own, so a scheduled agent is refused rather than run continuously, and + /// the board runs no agents at all. #[must_use] pub const fn runs_schedules(self) -> bool { - !matches!(self, Self::Mise | Self::Dekit) + !matches!(self, Self::Mise | Self::Dekit | Self::FirmwarePico2) } /// Whether this format runs the tasks of a `mise.toml`, so needs what that file names installed first. diff --git a/utilities/cli/src/lib.rs b/utilities/cli/src/lib.rs index 9d284c8a..3c9456ed 100644 --- a/utilities/cli/src/lib.rs +++ b/utilities/cli/src/lib.rs @@ -10,6 +10,7 @@ use std::slice; use edge_toolkit::ports::Services as Ports; use et_path::{absolute_from, relative_path_from}; +use et_sanitise::{is_plain_path, is_quotable_path}; use fs_err as fs; use serde::Deserialize; @@ -35,16 +36,16 @@ mod schedule; pub use self::admin::{EvictedAgent, block_ip, evict_agent}; pub use self::deployment_types::{ScenarioModules, docker_image_module_paths, scenario_module_paths}; pub(crate) use self::deployment_types::{ - generate_dekit_deployment, generate_docker_compose_deployment, generate_k3s_deployment, generate_mise_deployment, - generate_process_compose_deployment, generate_scenario_image, + generate_dekit_deployment, generate_docker_compose_deployment, generate_firmware_deployment, + generate_k3s_deployment, generate_mise_deployment, generate_process_compose_deployment, generate_scenario_image, }; pub use self::error::CliError; pub use self::hub_ws_url::{hub_service_ws_url, hub_service_ws_url_named, hub_ws_url, hub_ws_url_at}; -pub(crate) use self::input::running_in_this_repository; pub use self::input::{ ArtifactSource, ClusterInput, DEFAULT_CLUSTER_NAME, OutputType, Service, Services, infer_artifact_source, manifest_declares_this_repository, }; +pub(crate) use self::input::{Board, BoardSpec, running_in_this_repository}; pub use self::limits::{Cpus, Limits, LimitsError, Ulimit}; pub use self::module_package_json::generate_module_package_json; pub use self::recovery::{Recovery, RecoverySettings}; @@ -433,11 +434,13 @@ pub(crate) fn load_cluster_input(input_file: &Path) -> Result<(ClusterInput, u64 // filesystem root, which is its own directory. let input_abs = std::path::absolute(input_file)?; let input_dir = input_abs.parent().unwrap_or(&input_abs); - cluster.module_paths = cluster + let resolved: Vec = cluster .module_paths .iter() .map(|path| absolute_from(input_dir, path)) .collect(); + validate_module_paths(&cluster.module_paths, &resolved)?; + cluster.module_paths = resolved; Ok((cluster, scenario_seed(&content))) } @@ -522,6 +525,12 @@ pub fn regenerate_verification( .copied() .filter(|output_type| !scheduled || output_type.runs_schedules()) .filter(|output_type| cluster.recovery.is_unset() || *output_type == OutputType::K3s) + .filter(|output_type| { + Board::ALL.iter().all(|board| { + board.spec().output_type != *output_type + || deployment_types::validate_firmware(&cluster, *board).is_ok() + }) + }) .collect(); if output_types.is_empty() { continue; @@ -555,7 +564,15 @@ const fn deployment_summary( fn validate_deployment_outputs(cluster: &ClusterInput, output_types: &[OutputType]) -> Result<(), CliError> { refuse_unschedulable_formats(cluster, output_types)?; recovery::validate_formats(cluster, output_types)?; - recovery::validate(cluster, &scenario_runners(cluster)?) + recovery::validate(cluster, &scenario_runners(cluster)?)?; + // After the runners resolve, so a scenario naming a runner that does not exist is told that rather than only that + // the board runs none. + for board in Board::ALL { + if output_types.contains(&board.spec().output_type) { + deployment_types::validate_firmware(cluster, *board)?; + } + } + Ok(()) } /// Write every file a deployment of `output_types` needs into `output_dir`, with its credential and README. @@ -636,6 +653,7 @@ fn generate_format( generate_mise_deployment(cluster, output_dir, with_process_compose)?; generate_process_compose_deployment(cluster, output_dir) } + OutputType::FirmwarePico2 => generate_firmware_deployment(cluster, output_dir, Board::Pico2), } } @@ -869,12 +887,10 @@ fn generated_readme( }; let output_summary = output_summary(cluster, output_types); - // Relative to where the README's `docker build` runs from, and escaped because each lands inside a double-quoted - // shell argument. + // Relative to where the README's `docker build` runs from. let build_contexts = module_path_contexts(&cluster.module_paths, &image_build.context_root) .iter() .fold(String::default(), |mut flags, (name, path)| { - let path = escape_for_double_quotes(path); let _write_result = write!(flags, " --build-context \"{name}={path}\""); flags }); @@ -1109,9 +1125,45 @@ fn generated_run_instructions( OutputType::ProcessCompose => process_compose_run_instructions(artifacts).to_string(), OutputType::DockerCompose => docker_compose_run_instructions(cluster, artifacts), OutputType::K3s => k3s_run_instructions(cluster, image_build, runners, artifacts, build_contexts), + OutputType::FirmwarePico2 => firmware_run_instructions(Board::Pico2.spec()), } } +/// Render the README section that builds the hub as firmware for `board` and loads it onto one. +#[expect(clippy::single_call_fn, reason = "the firmware step of generated_run_instructions")] +fn firmware_run_instructions(board: BoardSpec) -> String { + format!( + concat!( + "## Run As Firmware On A {title}\n\n", + "The board runs the hub alone and serves this scenario's modules from its {flash} of flash.\n", + "Build those modules first, since the firmware takes each one's built package.\n", + "{network}\n", + "Then {preparation}.\n", + "From this directory, build the firmware and load it with:\n\n", + "```bash\n", + "MISE_ENV={env} mise run {task}\n", + "```\n\n", + "To check what the board will serve without one, serve the same modules from this host and fetch\n", + "each, as a page would:\n\n", + "```bash\n", + "MISE_ENV={env} mise run {host_test}\n", + "```\n\n", + "The board takes an address over DHCP and serves the page at port 8080 of it. npm packages the\n", + "modules use that are too big for its flash, such as onnxruntime-web, are not built in: the board\n", + "redirects a request for one to the same path under `{elsewhere}`, which the browser has to be\n", + "able to reach.\n" + ), + title = board.title, + flash = board.flash, + network = board.network_setup, + preparation = board.load_preparation, + env = board.output_type.name(), + task = deployment_types::FIRMWARE_FLASH_TASK, + host_test = deployment_types::FIRMWARE_HOST_TEST_TASK, + elsewhere = deployment_types::FIRMWARE_ELSEWHERE_URL, + ) +} + /// Render the README section that runs the Docker Compose deployment on one host. #[expect( clippy::single_call_fn, @@ -1213,6 +1265,8 @@ fn machine_instructions(output_type: OutputType, cluster: &ClusterInput) -> Stri |machine| format!("docker compose --profile {machine} up --build"), ), OutputType::K3s => return k3s_machine_instructions(cluster, server), + // Refused before anything is written: the board is one machine. + OutputType::FirmwarePico2 => return String::default(), }; let roles = machine_roles(cluster, server); let loopback = loopback_note(cluster); @@ -1738,20 +1792,56 @@ fn is_module_dir(path: &Path) -> bool { || path.join("pyproject.toml").is_file() } -/// A literal path spelled so a shell reads it back unchanged inside double quotes, as a generated command puts it. +/// Refuse a `module_paths:` directory whose name a generated deployment could not use as written. /// -/// Applied to a `module_paths:` module's path only. Every other entry is this repository's own path or a command -/// substitution the generator writes on purpose, whereas a `module_paths:` directory is named by whoever laid out the -/// tree the scenario points at, and a `$(...)` in that name would otherwise run when the deployment starts. -pub(crate) fn escape_for_double_quotes(text: &str) -> String { - let mut escaped = String::with_capacity(text.len()); - for character in text.chars() { - if matches!(character, '\\' | '"' | '$' | '`') { - escaped.push('\\'); +/// Every generator puts these paths into shell commands, so a directory called `$(id)` would otherwise run `id` when +/// the deployment starts. Rather than escape the path for each generator, a name that is not plain is refused while +/// the scenario is read. What the scenario's author chose has to be plain: each entry as `written`, and each module +/// directory found under an entry, by its name. The directory the input sits in is wherever the scenario happens to +/// be, such as a Windows temporary directory under `RUNNER~1`, so the whole `resolved` path is held only to what the +/// generated commands need of it: that it reads back unchanged inside the double quotes and lists they put it in. +fn validate_module_paths(written: &[PathBuf], resolved: &[PathBuf]) -> Result<(), CliError> { + for (index, (entry, path)) in written.iter().zip(resolved).enumerate() { + if !is_plain_path(entry) { + return Err(CliError::UnsafeModulePath(entry.clone())); + } + if !is_quotable_path(path) { + return Err(CliError::UnsafeModulePath(path.clone())); + } + for (dir, ..) in scenario_module_dirs(index, path) { + if dir != *path && !is_plain_path(Path::new(dir.file_name().unwrap_or_default())) { + return Err(CliError::UnsafeModulePath(dir)); + } } - escaped.push(character); } - escaped + Ok(()) +} + +/// The module directories the `module_paths:` entry `path`, at `index`, names. +/// +/// Each comes with its build-context path and the path an image holds it at. The entry is a module directory itself, or +/// a parent of several; a path that is neither names none. +fn scenario_module_dirs(index: usize, path: &Path) -> Vec<(PathBuf, String, String)> { + let context_root = format!("/app/module-paths/{index}"); + if is_module_dir(path) { + // The context is the module itself, so the whole of it is copied: its root, spelled as the one-character path. + return vec![(path.to_path_buf(), '.'.to_string(), context_root)]; + } + let Ok(entries) = fs::read_dir(path) else { + return Vec::new(); + }; + let mut dirs: Vec<(PathBuf, String, String)> = entries + .flatten() + .map(|entry| entry.path()) + .filter(|child| child.is_dir() && is_module_dir(child)) + .filter_map(|child| { + let name = child.file_name()?.to_str()?.to_string(); + let docker_path = format!("{context_root}/{name}"); + Some((child, name, docker_path)) + }) + .collect(); + dirs.sort(); + dirs } /// Register the `module_paths:` entry at `index`, which is a module directory or a parent of several. @@ -1765,33 +1855,11 @@ fn register_scenario_paths( path: &Path, ws_server_dir: &Path, ) { - let context_root = format!("/app/module-paths/{index}"); - let module_dirs: Vec<(PathBuf, String, String)> = if is_module_dir(path) { - // The context is the module itself, so the whole of it is copied: its root, spelled as the one-character path. - vec![(path.to_path_buf(), '.'.to_string(), context_root)] - } else { - let Ok(entries) = fs::read_dir(path) else { - return; - }; - let mut dirs: Vec<(PathBuf, String, String)> = entries - .flatten() - .map(|entry| entry.path()) - .filter(|child| child.is_dir() && is_module_dir(child)) - .filter_map(|child| { - let name = child.file_name()?.to_str()?.to_string(); - let docker_path = format!("{context_root}/{name}"); - Some((child, name, docker_path)) - }) - .collect(); - dirs.sort(); - dirs - }; - for (module_path, context_path, docker_path) in module_dirs { + for (module_path, context_path, docker_path) in scenario_module_dirs(index, path) { let Some(directory_name) = module_path.file_name().and_then(|name| name.to_str()) else { continue; }; let mut entry = module_entry(&module_path, ws_server_dir, &docker_path); - entry.mise_path = escape_for_double_quotes(&entry.mise_path); entry.source = ModuleSource::Scenario { dir: module_path.clone(), context: module_path_context(index), diff --git a/utilities/cli/src/limits.rs b/utilities/cli/src/limits.rs index 67f7151e..d7c74e8c 100644 --- a/utilities/cli/src/limits.rs +++ b/utilities/cli/src/limits.rs @@ -4,13 +4,13 @@ //! k3s runs on. Compose also accepts `mem_swappiness`, `oom_kill_disable` and the `cpu_rt_*` pair, but Docker discards //! the first two there with a warning and refuses the third outright, so they are not features a scenario could rely //! on. Per-device block I/O limits and device reservations name host devices, which a scenario written once for every -//! machine cannot know. `oom_score_adj` is a property of the process rather than of its cgroup, and the kubelet sets -//! it from each pod's `QoS` class, so no container runtime plugin can hold it to a scenario's value. +//! machine cannot know. `oom_score_adj` is a property of the process rather than of its cgroup, and the kubelet sets it +//! from each pod's `QoS` class, so no container runtime plugin can hold it to a scenario's value. //! //! Each field resolves to the same cgroup value in both formats, even where they spell it differently. k3s applies the -//! ones the Pod API carries natively and has two container runtime plugins write the rest straight into the -//! container's cgroup, so a field never means one thing under compose and another under k3s. The mise and dekit -//! deployments run each process directly on the host, outside any container, and apply none of them. +//! ones the Pod API carries natively and has two container runtime plugins write the rest straight into the container's +//! cgroup, so a field never means one thing under compose and another under k3s. The mise and dekit deployments run +//! each process directly on the host, outside any container, and apply none of them. use std::collections::BTreeMap; use std::fmt::{self, Write as _}; @@ -103,8 +103,8 @@ impl Cpus { /// The CPU weight this amount reserves, in the shares the kernel's cgroup v1 interface and Docker speak. /// - /// The kubelet's own conversion of a request, `millicores * 1024 / 1000` with a floor of two shares, so a - /// compose process reserving this amount gets the weight a Kubernetes container requesting it does. + /// The kubelet's own conversion of a request, `millicores * 1024 / 1000` with a floor of two shares, so a compose + /// process reserving this amount gets the weight a Kubernetes container requesting it does. #[must_use] pub fn shares(self) -> u64 { (u64::from(self.0).saturating_mul(1024).div_euclid(1000)).max(2) @@ -540,7 +540,8 @@ pub(crate) fn limits_note(output_type: OutputType, cluster: &ClusterInput) -> St // Named per format, because every format's section lands in the one README and markdownlint rejects a repeated // heading. let heading = match output_type { - OutputType::DockerCompose | OutputType::K3s => return String::default(), + // The board runs no agent, and only the one hub, which has nothing else to be limited against. + OutputType::DockerCompose | OutputType::K3s | OutputType::FirmwarePico2 => return String::default(), OutputType::Mise => "Mise", OutputType::Dekit => "dekit", OutputType::ProcessCompose => "process-compose", diff --git a/utilities/cli/tests/firmware.rs b/utilities/cli/tests/firmware.rs new file mode 100644 index 00000000..47c13cf6 --- /dev/null +++ b/utilities/cli/tests/firmware.rs @@ -0,0 +1,161 @@ +//! Covers the firmware formats through `firmware-pico2`: which modules a firmware build embeds and which it +//! redirects, and the scenarios it refuses because a board runs one hub and nothing else. +#![cfg(test)] + +use std::path::PathBuf; + +use et_cli::{OutputType, generate_deployment, regenerate_verification}; +use fs_err as fs; +use tempfile::{TempDir, tempdir}; + +/// A browser agent running math1, which the page loads from the board. +const BROWSER: &str = r#"cluster_name: "board" +agents: + - name: "math1-twin" + resources: + - type: "math1" +"#; + +/// The file `firmware-pico2` writes. +const CONFIG_FILE: &str = "mise.firmware-pico2.toml"; + +/// The generated firmware config, parsed. +#[derive(Debug, serde::Deserialize)] +struct FirmwareConfig { + tasks: Tasks, +} + +#[derive(Debug, serde::Deserialize)] +struct Tasks { + #[serde(rename = "firmware-build")] + build: Task, + #[serde(rename = "firmware-flash")] + flash: Task, + #[serde(rename = "firmware-host-test")] + host_test: Task, +} + +#[derive(Debug, Default, serde::Deserialize)] +#[serde(default)] +struct Task { + depends: Vec, + dir: String, + env: std::collections::BTreeMap, + run: String, +} + +/// What generating `input` in `test_root` as `firmware-pico2` returns, the error's text or empty, and its directory. +fn generate_in(test_root: TempDir, input: &str) -> (TempDir, PathBuf, String) { + let input_file = test_root.path().join("board.yaml"); + fs::write(&input_file, input).unwrap(); + let output_dir = test_root.path().join("output"); + let outcome = generate_deployment(&input_file, &output_dir, Some(OutputType::FirmwarePico2)) + .map_or_else(|error| error.to_string(), |_summary| String::default()); + (test_root, output_dir, outcome) +} + +/// The deployment `input` generates, which has to succeed. +fn generated(input: &str) -> (TempDir, FirmwareConfig, String) { + generated_in(tempdir().unwrap(), input) +} + +/// The deployment `input`, written to a scenario in `test_root`, generates, which has to succeed. +fn generated_in(test_root: TempDir, input: &str) -> (TempDir, FirmwareConfig, String) { + let (test_root, output_dir, outcome) = generate_in(test_root, input); + assert_eq!(outcome, ""); + assert_eq!(OutputType::FirmwarePico2.output_file_name(), CONFIG_FILE); + let config = toml::from_str(&fs::read_to_string(output_dir.join(CONFIG_FILE)).unwrap()).unwrap(); + let readme = fs::read_to_string(output_dir.join("README.md")).unwrap(); + (test_root, config, readme) +} + +#[test] +fn it_embeds_the_repository_modules_and_redirects_the_npm_packages() { + let (_root, config, readme) = generated(BROWSER); + let build = &config.tasks.build; + for module in [ + "services/ws-modules/math1", + "services/ws-server/static", + "services/ws-wasm-agent", + ] { + assert!(build.run.contains(&format!("$PWD/{module}"))); + } + assert!(!build.run.contains("onnxruntime-web")); + assert!(build.run.contains("mise -E rust run build-firmware-pico2w")); + assert_eq!(build.env["MODULES_ROOT"], "@edge-toolkit/et-ws-server-static"); + assert_eq!(build.env["MODULES_ELSEWHERE"], "onnxruntime-web, stats-gl"); + assert_eq!(build.env["MODULES_ELSEWHERE_URL"], "https://cdn.jsdelivr.net/npm"); + + assert_eq!(config.tasks.flash.depends, ["firmware-build"]); + assert_eq!(config.tasks.flash.dir, build.dir); + assert!(config.tasks.flash.run.starts_with("picotool load")); + assert!(readme.contains("## Run As Firmware On A Raspberry Pi Pico 2 W")); + assert!(readme.contains("MISE_ENV=firmware-pico2 mise run firmware-flash")); +} + +#[test] +fn its_host_test_serves_the_modules_the_build_embeds() { + let (_root, config, readme) = generated(BROWSER); + let (build, host_test) = (&config.tasks.build, &config.tasks.host_test); + assert_eq!(host_test.env, build.env); + assert_eq!(host_test.dir, build.dir); + let (paths, _) = build.run.split_once("mise -E rust").unwrap(); + let (host_paths, command) = host_test.run.split_once("mise run").unwrap(); + assert_eq!(host_paths, paths); + assert_eq!(command, " cargo-test-pkg et-int-ws-server-pico2w\n"); + assert!(readme.contains("MISE_ENV=firmware-pico2 mise run firmware-host-test")); +} + +#[test] +fn it_embeds_a_module_the_scenario_keeps_beside_it() { + let test_root = tempdir().unwrap(); + let module_dir = test_root.path().join("mine"); + fs::create_dir_all(&module_dir).unwrap(); + fs::write( + module_dir.join("package.json"), + r#"{"name": "mine", "main": "mine.js"}"#, + ) + .unwrap(); + // Named relative to the scenario, as `module_paths:` is meant to be; the temporary directory holding both, which + // on Windows runs through `RUNNER~1`, is not the scenario author's to name. + let input = format!("{BROWSER}module_paths: [\"mine\"]\n").replace(r#"type: "math1""#, r#"type: "mine""#); + + let module_path = module_dir.display().to_string(); + let (_root, config, _readme) = generated_in(test_root, &input); + assert!(config.tasks.build.run.contains(&module_path)); +} + +#[test] +fn it_refuses_what_the_board_cannot_run() { + let runner = BROWSER.replace(" resources:", " runner: \"web\"\n resources:"); + let machines = format!("{BROWSER}machines:\n - name: \"server\"\n address: \"10.0.0.1\"\n"); + let published = format!("{BROWSER}artifact_source: \"published\"\n"); + for (input, reason) in [ + (runner, "starts a runner"), + (machines, "spans several machines"), + (published, "uses published artifacts"), + ] { + let (_root, output_dir, outcome) = generate_in(tempdir().unwrap(), &input); + assert!(outcome.starts_with("The Raspberry Pi Pico 2 W firmware runs one hub and nothing else")); + assert!(outcome.contains(reason)); + assert!(!output_dir.join(CONFIG_FILE).exists()); + } +} + +#[test] +fn verification_renders_it_only_for_a_scenario_the_board_can_run() { + let test_root = tempdir().unwrap(); + let verification_root = test_root.path().join("verification"); + let input_dir = verification_root.join("local/input"); + fs::create_dir_all(&input_dir).unwrap(); + fs::write(input_dir.join("browser.yaml"), BROWSER).unwrap(); + let headless = BROWSER.replace(" resources:", " runner: \"web\"\n resources:"); + fs::write(input_dir.join("headless.yaml"), headless).unwrap(); + + let _regenerated = regenerate_verification(&verification_root, None).unwrap(); + let output = verification_root.join("local/output"); + assert!(output.join("browser").join(CONFIG_FILE).is_file()); + assert!(output.join("browser/mise.toml").is_file()); + assert!(!output.join("headless").join(CONFIG_FILE).exists()); + assert!(output.join("headless/mise.toml").is_file()); +} diff --git a/utilities/cli/tests/input.rs b/utilities/cli/tests/input.rs index f29e6b54..775e685b 100644 --- a/utilities/cli/tests/input.rs +++ b/utilities/cli/tests/input.rs @@ -1,9 +1,9 @@ //! Covers how a scenario input deserializes, and what each field answers when the input leaves it out. //! -//! Every default here is load-bearing, because `verification/*/input/default.yaml` is an input that states -//! nothing but its name and whose committed output is whatever these produce. The one that most repays a -//! test is `artifact_source`, which is not a fixed default at all: it is settled against the tree the -//! process is running in, so the same input means "build it" in this repository and "pull it" anywhere else. +//! Every default here is load-bearing, because `verification/*/input/default.yaml` is an input that states nothing but +//! its name and whose committed output is whatever these produce. The one that most repays a test is `artifact_source`, +//! which is not a fixed default at all: it is settled against the tree the process is running in, so the same input +//! means "build it" in this repository and "pull it" anywhere else. #![cfg(test)] use et_cli::{ @@ -39,8 +39,8 @@ fn an_input_stating_only_a_name_takes_every_other_field_from_its_default() { #[test] fn the_default_cluster_input_is_one_that_could_be_generated_from() { - // A derived `Default` would give it the empty name that scenario validation rejects, so the value would - // exist only to be invalid. This is the same cluster `ONLY_A_NAME` parses to, apart from the source. + // A derived `Default` would give it the empty name that scenario validation rejects, so the value would exist only + // to be invalid. This is the same cluster `ONLY_A_NAME` parses to, apart from the source. let cluster = ClusterInput::default(); assert_eq!(cluster.cluster_name, DEFAULT_CLUSTER_NAME); @@ -57,16 +57,16 @@ fn published_is_the_artifact_source_with_nothing_to_infer_from() { #[test] fn an_unstated_artifact_source_infers_local_inside_this_repository() { - // These tests run from a checkout, which is the proof the inference looks for. Were this to answer - // `Published`, every scenario under `verification/local` would quietly start naming the registry. + // These tests run from a checkout, which is the proof the inference looks for. Were this to answer `Published`, + // every scenario under `verification/local` would quietly start naming the registry. assert_eq!(infer_artifact_source(), ArtifactSource::Local); assert_eq!(parse(ONLY_A_NAME).artifact_source, ArtifactSource::Local); } #[test] fn a_stated_artifact_source_is_honoured_over_what_the_tree_would_infer() { - // In both directions, including a `published` stated here, where the tree could have supplied - // everything: that input describes a deployment somewhere else. + // In both directions, including a `published` stated here, where the tree could have supplied everything: that + // input describes a deployment somewhere else. let local = parse(&format!("{ONLY_A_NAME}artifact_source: local\n")); let published = parse(&format!("{ONLY_A_NAME}artifact_source: published\n")); @@ -109,6 +109,8 @@ fn every_deployment_type_spelling_deserializes_to_its_own_output_file() { ("dekit", OutputType::Dekit, "dekit.yaml"), ("process-compose", OutputType::ProcessCompose, "process-compose.yaml"), ("process_compose", OutputType::ProcessCompose, "process-compose.yaml"), + ("firmware-pico2", OutputType::FirmwarePico2, "mise.firmware-pico2.toml"), + ("firmware_pico2", OutputType::FirmwarePico2, "mise.firmware-pico2.toml"), ] { let cluster = parse(&format!("{ONLY_A_NAME}deployment_type: {spelling}\n")); @@ -117,10 +119,13 @@ fn every_deployment_type_spelling_deserializes_to_its_own_output_file() { assert_eq!(cluster.deployment_type.name(), spelling.replace('_', "-")); assert_eq!( cluster.deployment_type.runs_mise_tasks(), - !matches!(expected, OutputType::DockerCompose | OutputType::K3s) + !matches!( + expected, + OutputType::DockerCompose | OutputType::K3s | OutputType::FirmwarePico2 + ) ); } - assert_eq!(OutputType::ALL.len(), 5, "and the list of them is complete"); + assert_eq!(OutputType::ALL.len(), 6, "and the list of them is complete"); } #[test] @@ -136,8 +141,8 @@ fn an_unknown_deployment_type_is_rejected() { #[test] fn an_unknown_artifact_source_is_rejected() { - // Rather than falling back to a default, which would turn a typo into a deployment addressing the wrong - // artifacts and report nothing at generation time. + // Rather than falling back to a default, which would turn a typo into a deployment addressing the wrong artifacts + // and report nothing at generation time. let parsed: Result = serde_yaml::from_str(&format!("{ONLY_A_NAME}artifact_source: registry\n")); let _rejected = parsed.unwrap_err(); @@ -160,8 +165,8 @@ fn an_agent_keeps_its_runner_and_resources() { #[test] fn a_cluster_input_round_trips_through_yaml() { - // Serialization is a public surface of its own: `et-cli` writes a scenario back out, and a `skip` - // attribute on the wrong field would drop it silently rather than failing to compile. + // Serialization is a public surface of its own: `et-cli` writes a scenario back out, and a `skip` attribute on the + // wrong field would drop it silently rather than failing to compile. let cluster = parse(&format!("{WITH_AN_AGENT}artifact_source: published\n")); let rendered = serde_yaml::to_string(&cluster).unwrap(); diff --git a/utilities/cli/tests/machines.rs b/utilities/cli/tests/machines.rs index f53816ca..c07151e9 100644 --- a/utilities/cli/tests/machines.rs +++ b/utilities/cli/tests/machines.rs @@ -287,10 +287,12 @@ fn the_readme_says_how_to_run_each_machine_share() { let (_root, dir) = generated(&format!("{AGENTS}{MACHINES}")); let readme = read(&dir, "README.md"); + // Every format but the board's, which refuses a scenario spanning machines. assert_eq!( readme.matches("### Run Across Machines With").count(), - OutputType::ALL.len() + OutputType::ALL.len() - 1 ); + assert!(!dir.join(OutputType::FirmwarePico2.output_file_name()).exists()); assert!(readme.contains("mise run machine-edge")); assert!(readme.contains("docker compose --profile edge up --build")); assert!(readme.contains("dekit start +edge")); diff --git a/utilities/cli/tests/recovery_regeneration.rs b/utilities/cli/tests/recovery_regeneration.rs index 9b002964..faa7e6ae 100644 --- a/utilities/cli/tests/recovery_regeneration.rs +++ b/utilities/cli/tests/recovery_regeneration.rs @@ -38,6 +38,13 @@ fn verification_generation_limits_recovery_scenarios_to_supported_formats() { assert!(readme.contains("services/recovery-supervisor/Dockerfile")); } +/// Every format a scenario with runners renders: all but the board's, which runs none. +fn runner_formats() -> impl Iterator { + OutputType::ALL + .iter() + .filter(|kind| **kind != OutputType::FirmwarePico2) +} + /// Start with actual outputs from every generator, rather than fabricated files that may not match their layout. fn verification_without_recovery() -> (tempfile::TempDir, PathBuf) { let root = tempfile::tempdir().unwrap(); @@ -48,7 +55,7 @@ fn verification_without_recovery() -> (tempfile::TempDir, PathBuf) { let _file = write_input(&input_dir, &input); let generated = regenerate_verification(root.path(), None).unwrap(); let output = generated.first().unwrap().output_dir.clone(); - for kind in OutputType::ALL { + for kind in runner_formats() { assert!(output.join(kind.output_file_name()).is_file()); } (root, output) @@ -56,8 +63,7 @@ fn verification_without_recovery() -> (tempfile::TempDir, PathBuf) { /// Snapshot the launcher files that a recovery-free K3s-only regeneration must leave untouched. fn non_k3s_snapshot(output: &Path) -> Vec<(PathBuf, Vec)> { - OutputType::ALL - .iter() + runner_formats() .filter(|kind| **kind != OutputType::K3s) .map(|kind| { let path = output.join(kind.output_file_name()); diff --git a/utilities/cli/tests/scenario_generation.rs b/utilities/cli/tests/scenario_generation.rs index 33b978da..8e985cc6 100644 --- a/utilities/cli/tests/scenario_generation.rs +++ b/utilities/cli/tests/scenario_generation.rs @@ -922,27 +922,92 @@ fn scenario_module(dir: &std::path::Path, name: &str, loader: &str) { } #[test] -fn module_paths_modules_keep_their_own_identity_and_cannot_inject_shell() { +fn a_module_paths_entry_holding_no_module_adds_none() { + // One entry that is not there at all, and a parent whose only other child is a file rather than a module. let test_root = tempdir().unwrap(); - // Two directories of one name under different parents, which resolution used to fold into one module, and a - // directory whose name is shell syntax the generated task would otherwise run. + scenario_module(&test_root.path().join("parent/module"), "@ext/only", ""); + fs::write(test_root.path().join("parent/notes.txt"), "not a module").unwrap(); + let input_file = test_root.path().join("cluster.yaml"); + let input = "cluster_name: \"external\"\nmodule_paths: [\"absent\", \"parent\"]\n"; + fs::write(&input_file, input).unwrap(); + + let _summary = generate_deployment(&input_file, &test_root.path().join("output"), None).unwrap(); + let ws_server_dir = edge_toolkit::config::get_project_root().join("services/ws-server"); + let module_paths = [test_root.path().join("absent"), test_root.path().join("parent")]; + let modules = ["@ext/only".to_string()]; + let scenario = ScenarioModules::new(&ws_server_dir, &modules, false).with_module_paths(&module_paths); + let paths = scenario_module_paths(&scenario).unwrap(); + // The module, and the pyodide it depends on; nothing from the absent entry or the file. + assert!(paths.iter().any(|path| path.ends_with("parent/module")), "{paths:?}"); + assert!( + !paths + .iter() + .any(|path| path.contains("absent") || path.contains("notes")), + "{paths:?}" + ); +} + +#[test] +fn module_paths_modules_keep_their_own_identity() { + let test_root = tempdir().unwrap(); + // Two directories of one name under different parents, which resolution used to fold into one module. scenario_module(&test_root.path().join("a/module"), "@ext/a", ""); scenario_module(&test_root.path().join("b/module"), "@ext/b", ""); - scenario_module(&test_root.path().join("c/mod$(id)"), "@ext/c", ""); - let module_paths = [ - test_root.path().join("a/module"), - test_root.path().join("b/module"), - test_root.path().join("c"), - ]; + let module_paths = [test_root.path().join("a/module"), test_root.path().join("b/module")]; let ws_server_dir = edge_toolkit::config::get_project_root().join("services/ws-server"); - let modules = ["@ext/a".to_string(), "@ext/b".to_string(), "@ext/c".to_string()]; + let modules = ["@ext/a".to_string(), "@ext/b".to_string()]; let scenario = ScenarioModules::new(&ws_server_dir, &modules, false).with_module_paths(&module_paths); let paths = scenario_module_paths(&scenario).unwrap(); assert!(paths.iter().any(|path| path.ends_with("a/module")), "{paths:?}"); assert!(paths.iter().any(|path| path.ends_with("b/module")), "{paths:?}"); - assert!(paths.iter().any(|path| path.ends_with(r"c/mod\$(id)")), "{paths:?}"); - assert!(!paths.iter().any(|path| path.ends_with("c/mod$(id)")), "{paths:?}"); +} + +#[test] +fn a_module_path_that_is_not_plain_is_refused() { + // Named outright, reported as written, and found under a parent the scenario names, reported where it was found. + for (module_dir, entry, reported_under_root) in [("c/mod$(id)", "c/mod$(id)", false), ("d/my mod", "d", true)] { + let test_root = tempdir().unwrap(); + scenario_module(&test_root.path().join(module_dir), "@ext/c", ""); + let input_file = test_root.path().join("cluster.yaml"); + let input = format!("cluster_name: \"external\"\nmodule_paths: [{entry:?}]\n"); + fs::write(&input_file, input).unwrap(); + + let refused = generate_deployment(&input_file, &test_root.path().join("output"), None).unwrap_err(); + // Joined a name at a time, as the directory listing it was found by spells it. + let reported = if reported_under_root { + module_dir + .split('/') + .fold(test_root.path().to_path_buf(), |path, name| path.join(name)) + } else { + module_dir.into() + }; + assert_eq!( + refused.to_string(), + format!( + "Module path {reported:?} has a name a shell would interpret or a filesystem would not hold; rename it" + ) + ); + } +} + +#[test] +fn a_plain_module_path_under_a_directory_a_shell_would_expand_is_refused() { + // The entry is a plain name, but the scenario sits where it happens to, and that lands in the generated commands. + let test_root = tempdir().unwrap(); + let scenario_dir = test_root.path().join("case$HOME"); + scenario_module(&scenario_dir.join("modules"), "@ext/c", ""); + let input_file = scenario_dir.join("cluster.yaml"); + fs::write(&input_file, "cluster_name: \"external\"\nmodule_paths: [\"modules\"]\n").unwrap(); + + let refused = generate_deployment(&input_file, &test_root.path().join("output"), None).unwrap_err(); + let reported = scenario_dir.join("modules"); + assert_eq!( + refused.to_string(), + format!( + "Module path {reported:?} has a name a shell would interpret or a filesystem would not hold; rename it" + ) + ); } #[test] @@ -1021,19 +1086,6 @@ fn a_compose_deployment_builds_module_paths_modules_into_its_image() { assert!(dockerfile.contains(copy), "{dockerfile}"); } -#[test] -fn a_module_paths_directory_name_stays_one_operand_and_one_shell_word() { - // A space splits a plain-form `COPY` operand, and a `$` is substitution to both Docker and the shell: the entry - // carries the `$`, which the README's build context names, and the module under it the space. - let (_test_root, output_dir) = module_paths_deployment("k3s", "mods$HOME", "my mod"); - let dockerfile = fs::read_to_string(output_dir.join("Dockerfile")).unwrap(); - let readme = fs::read_to_string(output_dir.join("README.md")).unwrap(); - - let copy = r#"COPY --from=module-path-0 --chown=10001:10001 ["my mod", "/app/module-paths/0/my mod"]"#; - assert!(dockerfile.contains(copy), "{dockerfile}"); - assert!(readme.contains(r#"/mods\$HOME""#), "{readme}"); -} - #[test] fn a_k3s_deployment_names_the_module_paths_context_its_image_build_needs() { let (_test_root, output_dir) = module_paths_deployment("k3s", "modules", "mine"); diff --git a/verification/local/output/default/README.md b/verification/local/output/default/README.md index 0fbbd2b5..964ec8bc 100644 --- a/verification/local/output/default/README.md +++ b/verification/local/output/default/README.md @@ -1,7 +1,7 @@ # default This directory contains generated deployment configs for the `default` scenario. -Files: `mise.toml`, `dekit.yaml`, `process-compose.yaml`, `compose.yaml`, `k3s.yaml`. +Files: `mise.toml`, `dekit.yaml`, `process-compose.yaml`, `compose.yaml`, `k3s.yaml`, `mise.firmware-pico2.toml`. No workflow modules were selected in the scenario input. @@ -152,3 +152,27 @@ kubectl get pods -n "$ns" --watch Nothing outside the cluster can reach `ws-server`, so no browser can open its page. List `ws` and `wss` under `services.hub.expose` in the scenario input and regenerate to publish them. + +## Run As Firmware On A Raspberry Pi Pico 2 W + +The board runs the hub alone and serves this scenario's modules from its 4 MiB of flash. +Build those modules first, since the firmware takes each one's built package. +Set `WIFI_NETWORK` and `WIFI_PASSWORD` to the Wi-Fi network the board joins. +Then hold the board's BOOTSEL button while plugging it in. +From this directory, build the firmware and load it with: + +```bash +MISE_ENV=firmware-pico2 mise run firmware-flash +``` + +To check what the board will serve without one, serve the same modules from this host and fetch +each, as a page would: + +```bash +MISE_ENV=firmware-pico2 mise run firmware-host-test +``` + +The board takes an address over DHCP and serves the page at port 8080 of it. npm packages the +modules use that are too big for its flash, such as onnxruntime-web, are not built in: the board +redirects a request for one to the same path under `https://cdn.jsdelivr.net/npm`, which the browser has to be +able to reach. diff --git a/verification/local/output/default/mise.firmware-pico2.toml b/verification/local/output/default/mise.firmware-pico2.toml new file mode 100644 index 00000000..02048ffe --- /dev/null +++ b/verification/local/output/default/mise.firmware-pico2.toml @@ -0,0 +1,33 @@ +[tasks.firmware-build] +description = "Build the Raspberry Pi Pico 2 W firmware serving default" +dir = "../../../.." +run = """ +MODULES_PATHS="$PWD/services/ws-server/static, $PWD/services/ws-wasm-agent" +export MODULES_PATHS +mise -E rust run build-firmware-pico2w +""" + +[tasks.firmware-build.env] +MODULES_ELSEWHERE = "onnxruntime-web, stats-gl" +MODULES_ELSEWHERE_URL = "https://cdn.jsdelivr.net/npm" +MODULES_ROOT = "@edge-toolkit/et-ws-server-static" + +[tasks.firmware-flash] +depends = ["firmware-build"] +description = "Load the firmware onto a Raspberry Pi Pico 2 W, and start it" +dir = "../../../.." +run = "picotool load -x -t elf target/thumbv8m.main-none-eabihf/release/et-ws-server-pico2w" + +[tasks.firmware-host-test] +description = "Serve default's modules from this host as the board would, and fetch each" +dir = "../../../.." +run = """ +MODULES_PATHS="$PWD/services/ws-server/static, $PWD/services/ws-wasm-agent" +export MODULES_PATHS +mise run cargo-test-pkg et-int-ws-server-pico2w +""" + +[tasks.firmware-host-test.env] +MODULES_ELSEWHERE = "onnxruntime-web, stats-gl" +MODULES_ELSEWHERE_URL = "https://cdn.jsdelivr.net/npm" +MODULES_ROOT = "@edge-toolkit/et-ws-server-static" diff --git a/verification/local/output/facility-security-scenario/README.md b/verification/local/output/facility-security-scenario/README.md index 3ded8b4d..f9924ba8 100644 --- a/verification/local/output/facility-security-scenario/README.md +++ b/verification/local/output/facility-security-scenario/README.md @@ -1,7 +1,7 @@ # facility-security-scenario This directory contains generated deployment configs for the `facility-security-scenario` scenario. -Files: `mise.toml`, `dekit.yaml`, `process-compose.yaml`, `compose.yaml`, `k3s.yaml`. +Files: `mise.toml`, `dekit.yaml`, `process-compose.yaml`, `compose.yaml`, `k3s.yaml`, `mise.firmware-pico2.toml`. The scenario exposes these workflow modules: face-detection, har1, pyface1. @@ -152,3 +152,27 @@ kubectl get pods -n "$ns" --watch Nothing outside the cluster can reach `ws-server`, so no browser can open its page. List `ws` and `wss` under `services.hub.expose` in the scenario input and regenerate to publish them. + +## Run As Firmware On A Raspberry Pi Pico 2 W + +The board runs the hub alone and serves this scenario's modules from its 4 MiB of flash. +Build those modules first, since the firmware takes each one's built package. +Set `WIFI_NETWORK` and `WIFI_PASSWORD` to the Wi-Fi network the board joins. +Then hold the board's BOOTSEL button while plugging it in. +From this directory, build the firmware and load it with: + +```bash +MISE_ENV=firmware-pico2 mise run firmware-flash +``` + +To check what the board will serve without one, serve the same modules from this host and fetch +each, as a page would: + +```bash +MISE_ENV=firmware-pico2 mise run firmware-host-test +``` + +The board takes an address over DHCP and serves the page at port 8080 of it. npm packages the +modules use that are too big for its flash, such as onnxruntime-web, are not built in: the board +redirects a request for one to the same path under `https://cdn.jsdelivr.net/npm`, which the browser has to be +able to reach. diff --git a/verification/local/output/facility-security-scenario/mise.firmware-pico2.toml b/verification/local/output/facility-security-scenario/mise.firmware-pico2.toml new file mode 100644 index 00000000..66724077 --- /dev/null +++ b/verification/local/output/facility-security-scenario/mise.firmware-pico2.toml @@ -0,0 +1,41 @@ +[tasks.firmware-build] +description = "Build the Raspberry Pi Pico 2 W firmware serving facility-security-scenario" +dir = "../../../.." +run = """ +MODULES_PATHS="$PWD/data/model-modules/model-face1, $PWD/data/model-modules/model-har-motion1, $PWD/generated/python-ws" +MODULES_PATHS="$MODULES_PATHS, $PWD/libs/python-edge-toolkit, $PWD/libs/python-org" +MODULES_PATHS="$MODULES_PATHS, $PWD/services/ws-modules/face-detection, $PWD/services/ws-modules/har1" +MODULES_PATHS="$MODULES_PATHS, $PWD/services/ws-modules/pyface1, $PWD/services/ws-server/static" +MODULES_PATHS="$MODULES_PATHS, $PWD/services/ws-wasm-agent" +export MODULES_PATHS +mise -E rust run build-firmware-pico2w +""" + +[tasks.firmware-build.env] +MODULES_ELSEWHERE = "onnxruntime-web, pyodide, stats-gl" +MODULES_ELSEWHERE_URL = "https://cdn.jsdelivr.net/npm" +MODULES_ROOT = "@edge-toolkit/et-ws-server-static" + +[tasks.firmware-flash] +depends = ["firmware-build"] +description = "Load the firmware onto a Raspberry Pi Pico 2 W, and start it" +dir = "../../../.." +run = "picotool load -x -t elf target/thumbv8m.main-none-eabihf/release/et-ws-server-pico2w" + +[tasks.firmware-host-test] +description = "Serve facility-security-scenario's modules from this host as the board would, and fetch each" +dir = "../../../.." +run = """ +MODULES_PATHS="$PWD/data/model-modules/model-face1, $PWD/data/model-modules/model-har-motion1, $PWD/generated/python-ws" +MODULES_PATHS="$MODULES_PATHS, $PWD/libs/python-edge-toolkit, $PWD/libs/python-org" +MODULES_PATHS="$MODULES_PATHS, $PWD/services/ws-modules/face-detection, $PWD/services/ws-modules/har1" +MODULES_PATHS="$MODULES_PATHS, $PWD/services/ws-modules/pyface1, $PWD/services/ws-server/static" +MODULES_PATHS="$MODULES_PATHS, $PWD/services/ws-wasm-agent" +export MODULES_PATHS +mise run cargo-test-pkg et-int-ws-server-pico2w +""" + +[tasks.firmware-host-test.env] +MODULES_ELSEWHERE = "onnxruntime-web, pyodide, stats-gl" +MODULES_ELSEWHERE_URL = "https://cdn.jsdelivr.net/npm" +MODULES_ROOT = "@edge-toolkit/et-ws-server-static"