From d767378b8e3fc619f14dfb4f2e6eb60b19656a17 Mon Sep 17 00:00:00 2001 From: John Vandenberg Date: Wed, 4 Nov 2026 01:52:40 +0800 Subject: [PATCH] Dont use --context --- .dockerignore | 2 + .github/actions/k3s-scenario/action.yaml | 33 +++++++ .github/workflows/coverage.yaml | 4 +- .github/workflows/k3s.yaml | 31 +++++++ .github/workflows/test.yaml | 18 ++-- .gitignore | 2 + .mise/config.toml | 83 +++++++++++++++--- AGENTS.md | 25 ++++++ CHECKS.md | 9 +- README.md | 10 ++- config/conftest/policy/gha/gha.rego | 61 +++++++++++++ config/conftest/policy/gha/gha_test.rego | 86 +++++++++++++++++++ config/gitleaks.toml | 1 + config/lychee.toml | 2 +- .../no-repo-mentions-in-verification.md.yaml | 17 ++++ .../no-repo-mentions-in-verification.yaml | 14 +++ libs/test-recovery/src/lib.rs | 5 +- services/recovery-supervisor/HELP.md | 6 -- services/recovery-supervisor/README.md | 20 +++-- services/recovery-supervisor/src/config.rs | 9 +- .../recovery-supervisor/src/kubernetes.rs | 64 ++++++++++++-- services/recovery-supervisor/src/main.rs | 3 +- services/recovery-supervisor/tests/config.rs | 3 +- .../recovery-supervisor/tests/environment.rs | 53 +++++++++++- .../tests/fixtures/kubeconfig.yaml | 2 +- services/recovery-supervisor/tests/runtime.rs | 4 +- .../recovery-supervisor/tests/shutdown.rs | 16 ++-- services/ws-modules/README.md | 77 +++++++++++++++++ services/ws-modules/bluetooth/README.md | 43 ++++++++++ services/ws-server/Dockerfile | 4 +- .../cli/src/deployment_types/k3s_recovery.rs | 57 ++---------- utilities/cli/tests/recovery.rs | 30 +++++-- .../output/default/Dockerfile.dockerignore | 2 + .../output/expose/Dockerfile.dockerignore | 2 + .../Dockerfile.dockerignore | 2 + .../output/math1-cron/Dockerfile.dockerignore | 2 + .../math1-limits/Dockerfile.dockerignore | 2 + .../math1-multi-hub/Dockerfile.dockerignore | 2 + .../math1-multi/Dockerfile.dockerignore | 2 + .../output/math1/Dockerfile.dockerignore | 2 + .../Dockerfile.dockerignore | 2 + .../local/output/pyo3-math1-recovery/k3s.yaml | 35 -------- .../output/pyo3-math1/Dockerfile.dockerignore | 2 + .../output/wasi-math1/Dockerfile.dockerignore | 2 + 44 files changed, 678 insertions(+), 173 deletions(-) create mode 100644 .github/actions/k3s-scenario/action.yaml create mode 100644 config/semgrep-tests/no-repo-mentions-in-verification.md.yaml create mode 100644 config/semgrep/no-repo-mentions-in-verification.yaml create mode 100644 services/ws-modules/README.md create mode 100644 services/ws-modules/bluetooth/README.md diff --git a/.dockerignore b/.dockerignore index bbc3f37c..38ed41e0 100644 --- a/.dockerignore +++ b/.dockerignore @@ -53,6 +53,8 @@ lcov.info # Dart pub tool dir, regenerated by `dart pub get`; one global rule for the whole workspace (root + members). **/.dart_tool/ services/ws-server/storage/ +# One-off Markdown an agent writes on request, such as an email draft or notes, which is never committed. +**/*.local.md # Lock files a generated scenario leaves behind when it is run. # `lockfile = true` makes mise write one beside whichever config it resolved tools for, so running a scenario # drops `mise.lock` into that scenario's output directory. It is a by-product of running the deployment rather diff --git a/.github/actions/k3s-scenario/action.yaml b/.github/actions/k3s-scenario/action.yaml new file mode 100644 index 00000000..5a9e4cf4 --- /dev/null +++ b/.github/actions/k3s-scenario/action.yaml @@ -0,0 +1,33 @@ +--- +name: Verify a k3s scenario +description: >- + Installs the mise tools, builds every image a local scenario's k3s manifests name, and applies them to a + throwaway k3d cluster with `k3s-verify`. One job per scenario calls it after `actions/checkout`, which a local + action cannot do itself: GitHub reads this file from the checked-out tree. The calling job's `timeout-minutes` + bounds the whole of it, since a composite action's steps take no timeout of their own. + +inputs: + scenario: + description: >- + The scenario under verification/local/output to build and apply. Its module packages must already be built + when they are wasm-pack artifacts; committed module sources need nothing. + required: true + +runs: + using: composite + steps: + - name: Install mise + tools + uses: $/.github/actions/install-mise-tools + + - name: Build the scenario's images + shell: bash --noprofile --norc -euo pipefail {0} + env: + SCENARIO: ${{ inputs.scenario }} + run: mise run k3s-images --scenario "$SCENARIO" + + - name: Apply the manifests and verify the deployment + shell: bash --noprofile --norc -euo pipefail {0} + env: + CARGO_NET_RETRY: "5" + SCENARIO: ${{ inputs.scenario }} + run: mise run k3s-verify --scenario "$SCENARIO" diff --git a/.github/workflows/coverage.yaml b/.github/workflows/coverage.yaml index 1a594421..dba46719 100644 --- a/.github/workflows/coverage.yaml +++ b/.github/workflows/coverage.yaml @@ -42,12 +42,12 @@ jobs: - os: ubuntu-latest os_flag: linux timeout: 150 - # Longer than the Linux lane, despite doing less. + # As long as the Linux lane, despite doing less. # The macOS runners are slower per-core, and this one still pays the full instrumented rebuild even # though it skips the wasm/browser steps. - os: macos-latest os_flag: macos - timeout: 180 + timeout: 150 env: MISE_ENV: dart,dotnet,java,js,kotlin,python,r,rust,zig,coverage ET_TEST_COVERAGE: "true" diff --git a/.github/workflows/k3s.yaml b/.github/workflows/k3s.yaml index 83208ece..1f1d21d9 100644 --- a/.github/workflows/k3s.yaml +++ b/.github/workflows/k3s.yaml @@ -17,6 +17,7 @@ name: k3s - .github/actions/add-swap/** - .github/actions/free-disk-space/** - .github/actions/install-mise-tools/** + - .github/actions/k3s-scenario/** - .github/workflows/k3s.yaml # The `k3s-verify` task itself lives here. # A change to it must re-trigger this job rather than letting an edit to the verification steps ship @@ -52,6 +53,13 @@ name: k3s - verification/published/input/math1-multi.yaml - verification/published/output/math1-multi/Dockerfile - verification/published/output/math1-multi/k3s.yaml + # What the `recovery` job builds and applies: the supervisor and how it reaches the API, its runner target. + - services/recovery-supervisor/** + - services/ws-pyo3-runner/Dockerfile + - utilities/cli/src/recovery.rs + - verification/local/input/pyo3-math1-recovery.yaml + - verification/local/output/pyo3-math1-recovery/Dockerfile + - verification/local/output/pyo3-math1-recovery/k3s.yaml workflow_dispatch: permissions: @@ -176,6 +184,29 @@ jobs: CARGO_NET_RETRY: "5" run: mise run compose-verify --scenario math1-limits + # The recovery supervisor reaching the Kubernetes API through its pod's service account, with no kubeconfig. + # Only a live pod has that service account, so no unit test can show it. Its own job, so its three release image + # builds -- the hub, the native-Python runner and the supervisor -- run beside `verify` rather than after it. + # The scenario's one module is committed Python source, so nothing is built before the images. + recovery: + runs-on: ubuntu-latest + # Read-only pull of the maintainer-published ghcr.io mise-tools store, as in `verify`. + permissions: + contents: read + packages: read + timeout-minutes: 150 + steps: + - name: Checkout + uses: actions/checkout@v7 + with: + fetch-depth: 1 + persist-credentials: false + + - name: Verify the recovery scenario + uses: $/.github/actions/k3s-scenario + with: + scenario: pyo3-math1-recovery + # Whether what is on crates.io and GitHub Packages actually runs, which no other job asks. # Every sibling check reads this working tree; this one installs the released hub, runner and module packages # and runs the math1 exchange on them, so it is the only signal that a release is usable. diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index a7fbc93d..299ca681 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -96,7 +96,7 @@ jobs: '{"os":"ubuntu-26.04","timeout":40}', '{"os":"ubuntu-24.04-arm","timeout":50}', '{"os":"macos-latest","timeout":75}', - '{"os":"macos-26-intel","timeout":225,"install_timeout":120}', + '{"os":"macos-26-intel","timeout":150,"install_timeout":120}', '{"os":"windows-latest","timeout":120}', '{"os":"windows-11-arm","timeout":60}' ) @@ -105,7 +105,7 @@ jobs: || github.event.inputs.os == 'ubuntu-24.04-arm' && '[{"os":"ubuntu-24.04-arm","timeout":50}]' || github.event.inputs.os == 'macos-latest' && '[{"os":"macos-latest","timeout":75}]' || github.event.inputs.os == 'macos-26-intel' - && '[{"os":"macos-26-intel","timeout":225,"install_timeout":120}]' + && '[{"os":"macos-26-intel","timeout":150,"install_timeout":120}]' || github.event.inputs.os == 'windows-latest' && '[{"os":"windows-latest","timeout":120}]' || github.event.inputs.os == 'windows-11-arm' && '[{"os":"windows-11-arm","timeout":60}]' ) @@ -199,11 +199,11 @@ jobs: packages: read id-token: write runs-on: windows-latest - # Covers the step ceilings this job declares, which now sum past the old 120. - # Install 60 + prefetch 20 + build-modules 25 + tests 55 is 160, so at 120 a slow setup could have the - # job cancelled while the test step still had budget left -- the test step's own limit would never apply. - # In practice msvc runs ~68m end to end; this only binds when something is already pathological. - timeout-minutes: 165 + # Covers every step ceiling this job declares. + # Install 60 + prefetch 15 + build-modules 20 + tests 55 is 150, so a slow setup cannot have the job cancelled + # while the test step still has budget left. In practice msvc runs ~68m end to end; this only binds when + # something is already pathological. + timeout-minutes: 150 strategy: fail-fast: false matrix: @@ -245,7 +245,7 @@ jobs: key: portable-msvc-${{ hashFiles('.mise/config.msvc.toml') }} - name: Prefetch dependencies - timeout-minutes: 20 + timeout-minutes: 15 env: GITHUB_TOKEN: ${{ github.token }} run: mise run prefetch-ci @@ -255,7 +255,7 @@ jobs: run: echo "MISE_JOBS=1" >> "$GITHUB_ENV" - name: Build WASM modules - timeout-minutes: 25 + timeout-minutes: 20 env: GITHUB_TOKEN: ${{ github.token }} run: mise run build-modules diff --git a/.gitignore b/.gitignore index 81ecabd2..1ce5e3bd 100644 --- a/.gitignore +++ b/.gitignore @@ -49,6 +49,8 @@ coverage-python.xml # Dart pub tool dir, regenerated by `dart pub get`; one global rule for the whole workspace (root + members). **/.dart_tool/ services/ws-server/storage/ +# One-off Markdown an agent writes on request, such as an email draft or notes, which is never committed. +*.local.md # Lock files a generated scenario leaves behind when it is run. # `lockfile = true` makes mise write one beside whichever config it resolved tools for, so running a scenario # drops `mise.lock` into that scenario's output directory. It is a by-product of running the deployment rather diff --git a/.mise/config.toml b/.mise/config.toml index 3808cdca..a94d650a 100644 --- a/.mise/config.toml +++ b/.mise/config.toml @@ -804,6 +804,8 @@ toml_files = "git ls-files -z --cached --others --exclude-standard '*.toml'" task_shell_trace = 'bash -xeuo pipefail "{{ config_root }}/.mise/task-shell.sh"' # Unguarded variant, for a body that checks its own exit codes and must not abort on the first failure. task_shell_plain = 'bash "{{ config_root }}/.mise/task-shell.sh"' +# A yq program printing every image a k3s manifest names without a registry: the ones this repository builds. +k3s_local_images = '.. | select(tag == "!!map" and has("image")) | .image | select(test("/") | not)' # OpenObserve docker image for the openobserve/o2 dev-observability task. o2_image = "openobserve/openobserve:v1.1.2" # Port publish spec for that task, bound to loopback rather than every interface. @@ -1906,17 +1908,22 @@ shell = "{{ vars.task_shell }}" # stands up a throwaway k3s cluster (k3d runs one in Docker, so no root and no systemd), applies the scenario, # and asserts the twin stored the same model the mise and compose deployments are held to. # -# The images are the caller's to build: they come from this repository rather than a registry, and building -# them dwarfs the run itself, so a missing one fails immediately rather than starting a deployment whose pods -# can only sit in ErrImagePull. Deliberately not part of the `check` aggregate -- it needs Docker and those -# images, which no other check does. +# The images are the caller's to build, with `k3s-images`: they come from this repository rather than a +# registry, and building them dwarfs the run itself, so a missing one fails immediately rather than starting a +# deployment whose pods can only sit in ErrImagePull. Deliberately not part of the `check` aggregate -- it needs +# Docker and those images, which no other check does. +# +# A scenario with a `recovery:` block has no math1 exchange, so what it is held to instead is the supervisor's +# own access to the cluster. Its rollout completes only once a supervision cycle has, and every cycle asks the +# API about each workload through the pod's service account. A refused or unreachable request does not stop the +# cycle; it is counted in the report's `unknown_observations`, so that count staying at zero is the assertion. # # `--scenario` picks another math1-shaped scenario. One that names machines gets a k3d node per machine, the # first as the server and the rest as agents, each carrying the node label the manifests pin its pods to, and # the run then checks every pod landed on its own machine's node. k3d's nodes are full k3s nodes joined to the # server by token, so the scheduling and the cross-node traffic are real even though one host runs them all. [tasks.k3s-verify] -description = "Apply a generated math1 scenario's k3s manifests to a throwaway cluster and verify the stored model" +description = "Apply a generated scenario's k3s manifests to a throwaway cluster and verify the deployment works" run = """ scenario="${usage_scenario:-math1}" source="${usage_source:-local}" @@ -1925,6 +1932,7 @@ cluster="et-$scenario-verify" out="verification/$source/output/$scenario" input="verification/$source/input/$scenario.yaml" hub_image="et-ws-server-$scenario:latest" +recovery=$(yq '.recovery != null' "$input") manifest_wait=300s machine_label=edge-toolkit/machine # One name per line; each is an RFC 1123 label, so the unquoted word-splitting loops over it below are safe. @@ -1950,13 +1958,16 @@ if [ "$source" = "published" ]; then # on https://github.com/edge-toolkit/core/commit/bf8e71a0565e58c8fabb4d53640970eb181c5726 at # https://github.com/edge-toolkit/core/actions/runs/41288288633/job/124794004602, the first ubuntu-26.04 runners. printf 'FROM %s\\n' "$runner_image" | docker build --provenance=false -t "$runner_image" - + images="$hub_image $runner_image" else - runner_image="et-ws-web-runner:latest" + # The hub, each runner kind and a recovery supervisor are whichever the manifest names, so none is assumed. + images=$(yq -N '{{ vars.k3s_local_images }}' "$out/k3s.yaml" | coreutils sort -u) fi -for image in "$hub_image" "$runner_image"; do +# Word-split on purpose: an image reference holds no whitespace. +for image in $images; do if ! docker image inspect "$image" >/dev/null 2>&1; then - echo "k3s-verify: $image is not built; $out/README.md has the build and import steps" >&2 + echo "k3s-verify: $image is not built; $out/README.md has the steps, or for a local scenario, k3s-images" >&2 exit 1 fi done @@ -1995,7 +2006,7 @@ trap 'status=$?; k3d cluster delete "$cluster" >/dev/null 2>&1 || true coreutils rm -rf "$secrets_dir"' EXIT -cargo run -q -p et-cli -- generate-deployment --input-file "$input" --output-dir "$secrets_dir" +cargo run -q -p et-cli -- generate-deployment --input-file "$input" --output-dir "$secrets_dir" --output-type k3s # The first machine is the server node and every other one an agent. # Each is labelled as the README's join commands label them. A scenario naming no machines gets no flags, which @@ -2042,13 +2053,15 @@ for _ in $(coreutils seq 1 60); do done kubectl wait --for=condition=Ready node --all --timeout="$manifest_wait" # `k3d image import` loads every node, so a pod finds its image whichever machine it is pinned to. -k3d image import -c "$cluster" "$hub_image" "$runner_image" -# Every node is checked for both, since the import reports success even when a node's containerd drops an image. +# $images is a word-split list of references by design; do not quote it. +# shellcheck disable=SC2086 +k3d image import -c "$cluster" $images +# Every node is checked for each, since the import reports success even when a node's containerd drops an image. # k3d names each node's container as Kubernetes names the node, and a bare image name is docker.io's library one. for node in $(kubectl get nodes -o name); do node=${node#node/} held=$(docker exec "$node" ctr -n k8s.io images ls -q) - for image in "$hub_image" "$runner_image"; do + for image in $images; do case "$image" in */*) ref=$image ;; *) ref="docker.io/library/$image" ;; @@ -2071,10 +2084,27 @@ if [ -n "$other_hubs" ]; then deployments="$deployments ws-server-$machine" done fi +if [ "$recovery" = true ]; then + # The supervisor's report counts failed API requests, not unready workloads, so each workload is rolled out too. + deployments="$deployments recovery-supervisor $(yq '.recovery.workloads[].deployment' "$input")" +fi for deployment in $deployments; do kubectl rollout status "deploy/$deployment" -n "$ns" --timeout="$manifest_wait" done +if [ "$recovery" = true ]; then + # The supervisor image's own `cat` reads its state file; no mise-managed coreutils exists there. + # nosemgrep: config.semgrep.mise-task-coreutils-prefix + report=$(kubectl exec -n "$ns" deploy/recovery-supervisor -- cat /app/state/recovery.json) + unknown=$(printf '%s' "$report" | jaq '.unknown_observations') + if [ "$unknown" != 0 ]; then + echo "k3s-verify: $unknown supervisor requests to the Kubernetes API failed" >&2 + kubectl logs -n "$ns" deploy/recovery-supervisor --tail=40 >&2 || true + exit 1 + fi + exit 0 +fi + # The runners restart until the hub answers, so the model is what says the exchange ran -- not pod readiness. # # Only the first bucket is read. Both agents register and each gets its own storage bucket, so the glob matches @@ -2153,7 +2183,34 @@ done shell = "{{ vars.task_shell }}" usage = """ arg "[source]" help="Which verification tree to apply: local (default) or published" -flag "--scenario " help="Which math1-shaped scenario to apply" default="math1" +flag "--scenario " help="Which math1-shaped or recovery scenario to apply" default="math1" +""" + +# Build every image a local scenario's k3s manifest names from this repository, so `k3s-verify` can apply it. +# The hub image comes first because the scenario image takes it as the `hub` named build context. Every other +# image is `et-`, built from `services//Dockerfile`. The scenario image stages its modules' +# built `pkg/` directories, so a module that needs building must be built beforehand. +[tasks.k3s-images] +description = "Build the images a local scenario's k3s manifests name" +# docker is a host daemon rather than a mise tool, and these images install nothing through mise. +run = """ +scenario="${usage_scenario:-math1}" +out="verification/local/output/$scenario" +hub=et-ws-server-hub:latest +scenario_image="et-ws-server-$scenario:latest" +docker build -t "$hub" -f services/ws-server/Dockerfile . +for image in $(yq -N '{{ vars.k3s_local_images }}' "$out/k3s.yaml" | coreutils sort -u); do + if [ "$image" = "$scenario_image" ]; then + docker build --build-context "hub=docker-image://$hub" -t "$image" -f "$out/Dockerfile" . + else + name=${image%:*} + docker build -t "$image" -f "services/${name#et-}/Dockerfile" . + fi +done +""" +shell = "{{ vars.task_shell }}" +usage = """ +flag "--scenario " help="Which local scenario's images to build" default="math1" """ # End-to-end check that a generated compose deployment runs the math1 exchange, one machine at a time. diff --git a/AGENTS.md b/AGENTS.md index 06ebf531..7bec8481 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -83,6 +83,23 @@ outside the tree on purpose), then simply never point at it from inside. Creatin repo is fine when the operator wants that -- this rule is **only** about never referring to out-of-repo files from within the repo. +## Misc Markdown files are not committed + +Never create a `docs/` directory. When asked for a miscellaneous `.md` file related to this repo -- an email draft, +notes, a write-up -- write it at the top level of the repo as `.local.md`. `.gitignore` covers that suffix, so +it is never committed and no linter reads it. It needs no formatting either, and the line-length limit does not +apply. Write it for its reader instead: text meant to be pasted elsewhere, such as an email, keeps each paragraph +on one line so it pastes without hard breaks. Once it is written, offer to open it in a Markdown editor; on macOS, +`open` on the file launches the default app for `.md`. + +## Finding a Curtin staff member's details + +A Curtin staff member's email address is `firstname.lastname@curtin.edu.au`, so build it from their name rather than +searching for it. Their honorific, position and school are public in Curtin Profiles, which takes a surname at +`https://profiles.curtin.edu.au/search?by=text&type=user&v=`; it renders client-side, so read it in the +browser pane rather than fetching it. Web search finds neither, so do not spend searches on a name. The older +`https://staffportal.curtin.edu.au/staff/profile/view/` profile URLs redirect to Curtin Profiles. + ## On macOS: use homebrew bash for ad-hoc agent commands On macOS, every ad-hoc command the agent runs that is **not** a `mise run ` invocation -- investigations, @@ -137,6 +154,14 @@ back to the host binary -- surface it. Recommend to the user that the tool be ad (prebuilt-binary backend, cross-platform, per the "Tools must work on every OS" rule), and let them decide. A tool worth using in this repo is worth pinning. +## Only free tools with a GitHub repository + +When evaluating a tool -- for the repo, or to install on the operator's machine -- consider only free, open-source +tools, and confirm each has a GitHub repository before recommending or installing it. Never install a paid or +trial-licensed tool, even where it is the best of its kind; a free trial is still a non-free tool. Read the licence +in the repository rather than inferring it from a package listing: a Homebrew cask or an app-store page says nothing +about whether the tool is free -- the Marked Markdown viewer was installed that way and turned out to be paid. + ## Kill any command past an hour unless it is visibly progressing A command that has run for an hour is presumed hung and must be killed, unless there is positive evidence it is diff --git a/CHECKS.md b/CHECKS.md index 8b5005b5..6a8e481b 100644 --- a/CHECKS.md +++ b/CHECKS.md @@ -1,11 +1,11 @@ # Checks -- **Generated:** 2026-11-03 +- **Generated:** 2026-11-05 - **Checks:** 82 - **Tools:** 48 - **Shell-script checks:** 10 - **Checks with a repair pass:** 22 -- **Rules:** 363 custom local rules, or non-default strict linter settings +- **Rules:** 366 custom local rules, or non-default strict linter settings - **Meta rules:** 20 of those rules are rules about rules Every check is listed across every `MISE_ENV`, filed under the tool it runs. Each entry names the env whose config @@ -672,6 +672,8 @@ Rules, from `config/conftest/policy`: - Every actions/upload-artifact step must set with.if-no-files-found: error. - Scalar lists: `matrix.: [a, b, c]`. - `include:` entries, which is where the per-combination keys live. + - Every job must say how long it gets. + - No job may be given more than 150 minutes. - `gha_action` (`config/conftest/policy/gha_action/gha_action.rego`): - Composite-action `run:` steps require an explicit `shell:`. - Every actions/upload-artifact step must set with.if-no-files-found: error. @@ -1046,6 +1048,9 @@ Rules, from `config/semgrep`: own directory under `utilities/` with a `README.md`. - `no-regal-mentions` -- Do not name the Rego linter outside its own config. A policy is written the way Rego reads best; if one of its idioms is deliberate, say what the idiom does rather than which linter asked for it. +- `no-repo-mentions-in-verification` -- A scenario's generated docs must not mention git or a checkout. They are read by + whoever deploys the scenario, on machines that hold the generated directory and nothing else, so an instruction that + assumes the source repository is there is one that reader cannot follow. Change the et-cli text that renders it. - `no-roslynator-mentions` -- Do not name the C# analyzer outside the tasks and images that install it. A `#pragma warning disable` already names the diagnostic, which is the part a reader can look up. - `no-ruff-mentions` -- Do not name the Python linter/formatter outside its config and the configs that defer to it. A diff --git a/README.md b/README.md index 2907a9bf..c2132ff4 100644 --- a/README.md +++ b/README.md @@ -22,10 +22,12 @@ edge-toolkit is a WebSocket-based edge-computing framework that runs AI on hardware you control, so nothing has to leave your network. A lightweight server acts as a hub that serves small AI modules -- written in Rust, Python, Dart, C#, Java, Kotlin and more, each compiled to WebAssembly or transpiled to JavaScript -- straight to a browser, where they run -locally -and can reach the browser's own Web APIs (camera, microphone, geolocation, motion sensors, Bluetooth, NFC) to sense the -real world directly. The same framework also drives larger models on local GPU hardware through standardised WebAssembly -interfaces, so one toolkit spans on-device and server inference without changing the protocol. +locally and can reach the browser's own Web APIs (camera, microphone, geolocation, motion sensors, Bluetooth, NFC, +USB devices and more, listed by browser in [services/ws-modules/README.md](services/ws-modules/README.md)) to sense +the real world directly. Bluetooth is missing from Safari and every other mainstream iPhone browser; the +[bluetooth module's README](services/ws-modules/bluetooth/README.md) has the evidence and the Bluefy workaround. The +same framework also drives larger models on local GPU hardware through standardised WebAssembly interfaces, so one +toolkit spans on-device and server inference without changing the protocol. The result is AI that protects privacy and data sovereignty: sensitive camera, audio and research data stay on the device or your own network, never sent to an external cloud service. diff --git a/config/conftest/policy/gha/gha.rego b/config/conftest/policy/gha/gha.rego index d5ca41be..a397d8b9 100644 --- a/config/conftest/policy/gha/gha.rego +++ b/config/conftest/policy/gha/gha.rego @@ -217,3 +217,64 @@ deny contains msg if { [name, value, key, count(value), max_matrix_value_len], ) } + +# The longest a job may be given, in minutes. +max_job_timeout := 150 + +# Every job must say how long it gets. +# GitHub's own default for a job with no `timeout-minutes` is 360, so leaving it unset is the longest bound of all. +# A job that calls a reusable workflow (`uses:`) cannot set one, and is bounded by the jobs inside that workflow. +deny contains msg if { + some name, job in input.jobs + not job.uses + not job["timeout-minutes"] + msg := $"job \"{name}\" sets no timeout-minutes; GitHub would give it 360 (max {max_job_timeout})" +} + +# No job may be given more than 150 minutes. +# A job past that bound is doing too much for one lane: split it into jobs, which also run in parallel. +deny contains msg if { + some name, job in input.jobs + some minutes in job_timeouts(job) + minutes > max_job_timeout + msg := $"job \"{name}\": timeout-minutes can be {minutes} (max {max_job_timeout}); split the job" +} + +# Every value a job's `timeout-minutes` can take. +# A literal is itself. An expression is every number literal in it, such as codeql's `&& 60 || 15`; digits inside a +# quoted string such as `'windows-2022'` or a name such as `matrix.node-18` are not ones it can return. When it +# reads `matrix.timeout` (or `matrix['timeout']`), it is also every `timeout` the matrix declares at the top level +# or on an `include` entry: as a scalar or list under that key, or inside a JSON string, as test.yaml's `fromJSON` +# include is written. `install_timeout` and keys nested deeper are not the job's `timeout`. +job_timeouts(job) := ({job["timeout-minutes"] | + is_number(job["timeout-minutes"]) +} | {to_number(match[1]) | + is_string(job["timeout-minutes"]) + unquoted := regex.replace(job["timeout-minutes"], `'[^']*'`, "''") + some match in regex.find_all_string_submatch_n(`(?:^|[^\w.-])(\d+)(?:[^\w.-]|$)`, unquoted, -1) +}) | matrix_timeouts(job) + +matrix_timeouts(job) := {minutes | + reads_matrix_timeout(job) + walk(object.get(job, ["strategy", "matrix"], {}), [path, value]) + matrix_timeout_path(path) + some minutes in scalar_or_list(value) +} | {to_number(match[1]) | + reads_matrix_timeout(job) + walk(object.get(job, ["strategy", "matrix"], {}), [_, value]) + is_string(value) + some match in regex.find_all_string_submatch_n(`"timeout"\s*:\s*(\d+)`, value, -1) +} + +matrix_timeout_path(["timeout"]) + +matrix_timeout_path(["include", _, "timeout"]) + +reads_matrix_timeout(job) if { + is_string(job["timeout-minutes"]) + regex.match(`matrix(\.timeout\b|\[\s*'timeout'\s*\])`, job["timeout-minutes"]) +} + +scalar_or_list(value) := value if is_array(value) + +else := [value] diff --git a/config/conftest/policy/gha/gha_test.rego b/config/conftest/policy/gha/gha_test.rego index 747d7b96..13a2c224 100644 --- a/config/conftest/policy/gha/gha_test.rego +++ b/config/conftest/policy/gha/gha_test.rego @@ -26,3 +26,89 @@ test_quoted_on_key_is_read_too if { quoted := object.union(object.remove(workflow(["src/**"]), ["true"]), {"on": {"pull_request": {"paths": ["src/**"]}}}) untriggered(gha.deny) with input as quoted } + +# A workflow whose one job is `job`. +timed(job) := {"name": "probe", "jobs": {"build": job}} + +over_cap(msgs) if { + some msg in msgs + contains(msg, "timeout-minutes can be") +} + +unbounded(msgs) if { + some msg in msgs + contains(msg, "sets no timeout-minutes") +} + +test_job_at_the_cap_is_allowed if { + not over_cap(gha.deny) with input as timed({"timeout-minutes": 150}) +} + +test_job_over_the_cap_is_denied if { + over_cap(gha.deny) with input as timed({"timeout-minutes": 151}) +} + +test_job_without_a_timeout_is_denied if { + unbounded(gha.deny) with input as timed({"steps": []}) +} + +test_reusable_workflow_job_needs_no_timeout if { + not unbounded(gha.deny) with input as timed({"uses": "./.github/workflows/other.yaml"}) +} + +test_number_in_an_expression_is_read if { + over_cap(gha.deny) with input as timed({"timeout-minutes": "${{ matrix.language == 'rust' && 160 || 15 }}"}) +} + +test_expression_within_the_cap_is_allowed if { + not over_cap(gha.deny) with input as timed({"timeout-minutes": "${{ matrix.language == 'rust' && 60 || 15 }}"}) +} + +test_matrix_include_timeout_over_the_cap_is_denied if { + job := { + "timeout-minutes": "${{ matrix.timeout }}", + "strategy": {"matrix": {"include": [{"os": "a", "timeout": 40}, {"os": "b", "timeout": 180}]}}, + } + over_cap(gha.deny) with input as timed(job) +} + +test_matrix_list_timeout_over_the_cap_is_denied if { + job := {"timeout-minutes": "${{ matrix.timeout }}", "strategy": {"matrix": {"timeout": [40, 200]}}} + over_cap(gha.deny) with input as timed(job) +} + +test_matrix_json_string_timeout_over_the_cap_is_denied if { + include := `${{ fromJSON('[{"os":"a","timeout":225}]') }}` + job := {"timeout-minutes": "${{ matrix.timeout }}", "strategy": {"matrix": {"include": include}}} + over_cap(gha.deny) with input as timed(job) +} + +test_other_matrix_timeout_keys_are_not_the_job_timeout if { + include := `${{ fromJSON('[{"os":"a","timeout":150,"install_timeout":200}]') }}` + job := {"timeout-minutes": "${{ matrix.timeout }}", "strategy": {"matrix": {"include": include}}} + not over_cap(gha.deny) with input as timed(job) +} + +test_digits_in_a_quoted_runner_label_are_not_a_timeout if { + not over_cap(gha.deny) with input as timed({"timeout-minutes": "${{ matrix.os == 'windows-2022' && 90 || 60 }}"}) +} + +test_digits_in_a_hyphenated_name_are_not_a_timeout if { + not over_cap(gha.deny) with input as timed({"timeout-minutes": "${{ matrix.node-180 && 90 || 60 }}"}) +} + +test_nested_matrix_timeout_is_not_the_job_timeout if { + entry := {"os": "a", "timeout": 30, "config": {"timeout": 200}} + job := {"timeout-minutes": "${{ matrix.timeout }}", "strategy": {"matrix": {"include": [entry]}}} + not over_cap(gha.deny) with input as timed(job) +} + +test_bracket_matrix_timeout_over_the_cap_is_denied if { + job := {"timeout-minutes": "${{ matrix['timeout'] }}", "strategy": {"matrix": {"timeout": [200]}}} + over_cap(gha.deny) with input as timed(job) +} + +test_matrix_timeouts_are_ignored_when_the_job_does_not_read_them if { + job := {"timeout-minutes": 30, "strategy": {"matrix": {"include": [{"os": "a", "timeout": 500}]}}} + not over_cap(gha.deny) with input as timed(job) +} diff --git a/config/gitleaks.toml b/config/gitleaks.toml index 183c3913..71dfba8f 100644 --- a/config/gitleaks.toml +++ b/config/gitleaks.toml @@ -59,6 +59,7 @@ paths = [ description = "Untracked artifacts matched by extension rather than location, per .gitignore's file patterns." paths = [ '''\.coverage$''', + '''\.local\.md$''', '''\.o$''', '''\.onnx$''', '''\.pem$''', diff --git a/config/lychee.toml b/config/lychee.toml index 7c28e905..5b35b7b2 100644 --- a/config/lychee.toml +++ b/config/lychee.toml @@ -26,7 +26,7 @@ exclude_private = true # those paths). # Each entry is an unanchored regex over the path, so it is anchored to a whole directory name: a bare "data" also # dropped every file under the `*-data1` modules. -exclude_path = ['(^|/)node_modules/', '(^|/)target/', '^(\./)?data/', '^(\./)?generated/'] +exclude_path = ['(^|/)node_modules/', '(^|/)target/', '\.local\.md$', '^(\./)?data/', '^(\./)?generated/'] # URL patterns (regex) to skip, none of which are real links. # doc/test placeholder hosts (e.g. http://host:8080/) and `{...}` format-string templates diff --git a/config/semgrep-tests/no-repo-mentions-in-verification.md.yaml b/config/semgrep-tests/no-repo-mentions-in-verification.md.yaml new file mode 100644 index 00000000..4b63071f --- /dev/null +++ b/config/semgrep-tests/no-repo-mentions-in-verification.md.yaml @@ -0,0 +1,17 @@ +code: | + # Scenario + + + Copy this directory into a checkout of the repository. + + + Run git pull on each machine first. + + + Keep both Checkouts at the same path. + + + The modules are published on GitHub, and `.gitignore` keeps the build output out. + + + Copy this directory to each machine and start that machine's share there. diff --git a/config/semgrep/no-repo-mentions-in-verification.yaml b/config/semgrep/no-repo-mentions-in-verification.yaml new file mode 100644 index 00000000..aba81b91 --- /dev/null +++ b/config/semgrep/no-repo-mentions-in-verification.yaml @@ -0,0 +1,14 @@ +rules: + - id: no-repo-mentions-in-verification + languages: [generic] + paths: + include: + - "/verification/**/*.md" + # Whole words only, so `GitHub` and `.gitignore` are left alone. + # Check-only: dropping the word leaves a sentence to be rewritten around it, in the generator that wrote it. + pattern-regex: '(?i)\b(?:git|checkouts?)\b' + message: >- + A scenario's generated docs must not mention git or a checkout. They are read by whoever deploys the + scenario, on machines that hold the generated directory and nothing else, so an instruction that assumes the + source repository is there is one that reader cannot follow. Change the et-cli text that renders it. + severity: ERROR diff --git a/libs/test-recovery/src/lib.rs b/libs/test-recovery/src/lib.rs index 461d0f07..20253580 100644 --- a/libs/test-recovery/src/lib.rs +++ b/libs/test-recovery/src/lib.rs @@ -24,9 +24,12 @@ pub fn config() -> Config { #[must_use] pub fn environment() -> BTreeMap { let values = [ + // Kubelet sets these in every pod, and the supervisor refuses to start without them. + ("KUBERNETES_SERVICE_HOST", "10.43.0.1"), + ("KUBERNETES_SERVICE_PORT", "443"), + ("KUBERNETES_SERVICE_PORT_HTTPS", "443"), ("RECOVERY_AGENT_ID", "recovery-supervisor"), ("RECOVERY_COMMAND_TIMEOUT_SECONDS", "10"), - ("RECOVERY_CONTEXT", "k3d-example"), ("RECOVERY_JOB_ID", "example-job"), ("RECOVERY_MODE", "observe-only"), ("RECOVERY_NAMESPACE", "example"), diff --git a/services/recovery-supervisor/HELP.md b/services/recovery-supervisor/HELP.md index ff3849a3..30cb9d0c 100644 --- a/services/recovery-supervisor/HELP.md +++ b/services/recovery-supervisor/HELP.md @@ -16,12 +16,6 @@ Type: integer. Required. Bound on each kubectl process, including credential-plugin execution. -### `RECOVERY_CONTEXT` - -Type: string. Required. - -Explicit Kubernetes context; never relies on the operator's current context. - ### `RECOVERY_JOB_ID` Type: string. Required. diff --git a/services/recovery-supervisor/README.md b/services/recovery-supervisor/README.md index bd7361a9..9ae39b7c 100644 --- a/services/recovery-supervisor/README.md +++ b/services/recovery-supervisor/README.md @@ -43,8 +43,8 @@ recovery: ``` Generate and deploy the scenario using the normal CLI workflow. The output includes the supervisor Deployment, -`RECOVERY_*` settings, ServiceAccount, named-Deployment Role and RoleBinding, projected-token kubeconfig and persistent -state claim. Observe-only grants `get`; controller-assisted also grants `patch`. No host kubeconfig is mounted. +`RECOVERY_*` settings, ServiceAccount, named-Deployment Role and RoleBinding, and persistent state claim. Observe-only +grants `get`; controller-assisted also grants `patch`. No kubeconfig is mounted: kubectl uses the pod's service account. The generated README includes image preparation and deployment commands. Recovery currently requires k3s output; other formats are rejected rather than silently omitting supervision. Optional `recovery.limits` uses the standard scenario resource-limit fields. @@ -88,16 +88,15 @@ health check. A replacement restores recovery context, retaining an explicit gap ## Standalone example The binary reads environment settings through serde-env and takes no configuration-file argument. Supply real identities -and a surviving state directory, then start it separately from the application: +and a surviving state directory, then start it in its own pod, separately from the application: ```sh export RECOVERY_AGENT_ID=recovery-supervisor -export RECOVERY_WS_URL=ws://127.0.0.1:8080/ws -export RECOVERY_CONTEXT=k3d-example +export RECOVERY_WS_URL=ws://ws-server:8080/ws export RECOVERY_NAMESPACE=example export RECOVERY_JOB_ID=example-job export RECOVERY_MODE=observe-only -export RECOVERY_STATE_FILE=target/recovery-state.json +export RECOVERY_STATE_FILE=/app/state/recovery.json export RECOVERY_POLL_SECONDS=2 export RECOVERY_COMMAND_TIMEOUT_SECONDS=10 export RECOVERY_POLICY_DETECTION_GRACE_SECONDS=3 @@ -108,11 +107,14 @@ export RECOVERY_POLICY_MAX_RESTARTS=2 export RECOVERY_WORKLOADS='[ {"deployment":"example-worker","agent_id":"example-worker","source":"/modules/example-worker"} ]' -mise run recovery-supervisor +et-recovery-supervisor ``` -`kubectl` must be on PATH with credentials for the explicitly configured context. Standalone startup does not apply -deployment manifests. The state directory must already exist and survive supervisor restarts. +`kubectl` must be on PATH. The supervisor passes it no context, so kubectl uses its in-cluster service-account +credentials: run the supervisor in a pod whose ServiceAccount has the permissions below. Startup is refused unless +`KUBERNETES_SERVICE_HOST` and `KUBERNETES_SERVICE_PORT` are set, as kubelet sets them in every pod, and refused when +`KUBECONFIG` is set or `$HOME/.kube/config` exists, since kubectl would prefer either to the service account. Standalone +startup does not apply deployment manifests. The state directory must already exist and survive supervisor restarts. `RECOVERY_WORKLOADS` is a JSON array of objects with `deployment`, `agent_id`, and `source` fields. It remains an array for a single workload. The loader explicitly decodes this JSON; malformed entries and unknown workload fields fail diff --git a/services/recovery-supervisor/src/config.rs b/services/recovery-supervisor/src/config.rs index a156b82d..29fcfe70 100644 --- a/services/recovery-supervisor/src/config.rs +++ b/services/recovery-supervisor/src/config.rs @@ -64,8 +64,6 @@ pub struct Config { pub agent_id: String, /// Application job whose lifecycle events are eligible. pub job_id: String, - /// Explicit Kubernetes context; never relies on the operator's current context. - pub context: String, /// Namespace containing every allowlisted deployment. pub namespace: String, /// Observe-only unless explicitly enabled. @@ -146,7 +144,6 @@ impl Config { ws_url, agent_id: "recovery-supervisor".into(), job_id, - context: "in-cluster".into(), namespace, mode: Mode::ObserveOnly, poll_seconds: 2, @@ -183,15 +180,13 @@ impl Config { /// Reject ambiguous job, deployment namespace and durable-state identities before connecting. fn validate_identity(&self) -> Result<(), RecoveryError> { - if self.context.is_empty() - || self.context.starts_with('-') - || !valid_name(&self.namespace) + if !valid_name(&self.namespace) || self.agent_id.is_empty() || self.job_id.is_empty() || self.state_file.as_os_str().is_empty() { return Err(RecoveryError::Invalid( - "invalid hub, job, namespace, context, or state location".into(), + "invalid hub, job, namespace, or state location".into(), )); } validate_endpoint(&self.ws_url) diff --git a/services/recovery-supervisor/src/kubernetes.rs b/services/recovery-supervisor/src/kubernetes.rs index 0c145b0a..742a58f8 100644 --- a/services/recovery-supervisor/src/kubernetes.rs +++ b/services/recovery-supervisor/src/kubernetes.rs @@ -1,5 +1,6 @@ //! Namespace-scoped Kubernetes operations; targets come only from validated operator configuration. +use std::path::Path; use std::time::Duration; use serde::Deserialize; @@ -17,7 +18,54 @@ pub trait Backend: Send { fn restart(&mut self, workload: &Workload) -> impl Future> + Send; } -/// Kubernetes backend using the installed kubectl credentials and an explicit context and namespace. +/// The environment that decides whether kubectl uses the pod's service account or a kubeconfig. +#[derive(Debug, Deserialize)] +#[non_exhaustive] +pub struct Pod { + /// `HOME`, whose `.kube/config` kubectl reads when `KUBECONFIG` is unset. + #[serde(default)] + home: String, + /// `KUBECONFIG`, which kubectl reads in preference to the service account. + #[serde(default)] + kubeconfig: String, + /// `KUBERNETES_SERVICE_HOST`, unset outside a pod. + #[serde(default)] + kubernetes_service_host: String, + /// `KUBERNETES_SERVICE_PORT`, unset outside a pod. + #[serde(default)] + kubernetes_service_port: String, +} + +impl Pod { + /// Refuse to supervise unless kubectl will use the pod's service account, as no context pins it to one cluster. + pub fn validate(&self) -> Result<(), RecoveryError> { + if self.kubernetes_service_host.is_empty() || self.kubernetes_service_port.is_empty() { + return Err(RecoveryError::Invalid( + concat!( + "refusing to start outside a pod: KUBERNETES_SERVICE_HOST or KUBERNETES_SERVICE_PORT is unset, ", + "so kubectl would use the current kubeconfig context rather than the pod's service account", + ) + .into(), + )); + } + if !self.kubeconfig.is_empty() { + return Err(RecoveryError::Invalid( + "refusing to start: kubectl would use KUBECONFIG rather than the pod's service account".into(), + )); + } + // A path that cannot be checked is let through: kubectl then fails to read it rather than reaching a cluster. + let home_kubeconfig = Path::new(&self.home).join(".kube").join("config"); + if home_kubeconfig.exists() { + return Err(RecoveryError::Invalid(format!( + "refusing to start: kubectl would use {} rather than the pod's service account", + home_kubeconfig.display() + ))); + } + Ok(()) + } +} + +/// Kubernetes backend using kubectl's in-cluster service-account credentials and the configured namespace. #[derive(Debug)] pub struct Kubernetes { config: Config, @@ -30,7 +78,7 @@ impl Kubernetes { Ok(Self { config }) } - /// Execute kubectl directly, without shell interpolation, bounding both the API request and child lifetime. + /// Execute kubectl directly, without shell interpolation, bounding the child's lifetime. pub async fn execute(&self, workload: &Workload, arguments: &[&str]) -> Result, RecoveryError> { let supervised = self .config @@ -44,14 +92,12 @@ impl Kubernetes { } let target = format!("deployment/{}", workload.deployment); let mut command = Command::new("kubectl"); + // Only the namespace is overridden; the process deadline below bounds the request instead of a flag. + // kubectl falls back to its in-cluster credentials only while the client config it built equals the bare + // default, and `--request-timeout` is part of that config, so with it every request in a pod went to + // The connection to the server localhost:8080 was refused - did you specify the right host or port? let _command = command - .args([ - "--context", - &self.config.context, - "--namespace", - &self.config.namespace, - "--request-timeout=5s", - ]) + .args(["--namespace", &self.config.namespace]) .args(arguments) .arg(target); let output = diff --git a/services/recovery-supervisor/src/main.rs b/services/recovery-supervisor/src/main.rs index 621fe98e..3663630b 100644 --- a/services/recovery-supervisor/src/main.rs +++ b/services/recovery-supervisor/src/main.rs @@ -1,7 +1,7 @@ //! Recovery service entrypoint; policy and backend behavior are exercised through the library. use et_recovery_supervisor::config::EnvConfig; -use et_recovery_supervisor::kubernetes::Kubernetes; +use et_recovery_supervisor::kubernetes::{Kubernetes, Pod}; /// Load environment configuration and run recovery supervision. #[tokio::main] @@ -11,6 +11,7 @@ async fn main() -> Result<(), Box> { .with_env_filter(tracing_subscriber::EnvFilter::from_default_env()) .try_init()?; let backend = Kubernetes::new(config.clone())?; + serde_env::from_env::()?.validate()?; et_recovery_supervisor::runtime::run(config, backend).await?; Ok(()) } diff --git a/services/recovery-supervisor/tests/config.rs b/services/recovery-supervisor/tests/config.rs index 5804f082..ffa9f18d 100644 --- a/services/recovery-supervisor/tests/config.rs +++ b/services/recovery-supervisor/tests/config.rs @@ -97,7 +97,6 @@ fn resilience_configuration_round_trips_and_rejects_invalid_dependency_graphs() #[test] fn typed_in_cluster_settings_match_the_expected_runtime_configuration() { let mut expected = config(); - expected.context = "in-cluster".into(); expected.state_file = "/app/state/recovery.json".into(); let actual = Config::in_cluster( expected.ws_url.clone(), @@ -147,7 +146,7 @@ fn targets_and_identities_are_explicit_and_unique() { } assert!(!valid_name(&"a".repeat(64))); assert!(valid_name("worker-1")); - for field in ["context", "namespace", "agent_id", "job_id", "state_file", "ws_url"] { + for field in ["namespace", "agent_id", "job_id", "state_file", "ws_url"] { let mut value = config_for_serialization(); value[field] = serde_json::json!(""); let invalid: et_recovery_supervisor::config::Config = serde_json::from_value(value).unwrap(); diff --git a/services/recovery-supervisor/tests/environment.rs b/services/recovery-supervisor/tests/environment.rs index b3a263bc..a8b62d42 100644 --- a/services/recovery-supervisor/tests/environment.rs +++ b/services/recovery-supervisor/tests/environment.rs @@ -2,7 +2,9 @@ #![cfg(test)] use et_recovery_supervisor::config::{Config, EnvConfig, Mode}; +use et_recovery_supervisor::kubernetes::Pod; use et_test_recovery::environment; +use fs_err as fs; /// Keep recovery opt-in while allowing unrelated service settings in the process environment. #[test] @@ -79,7 +81,7 @@ fn malformed_environment_values_and_missing_fields_fail_deserialization() { fn parsed_environment_still_requires_semantic_validation() { for (key, value) in [ ("RECOVERY_WORKLOADS", "[]"), - ("RECOVERY_CONTEXT", "-unsafe"), + ("RECOVERY_NAMESPACE", "-unsafe"), ("RECOVERY_POLICY_DEADLINE_SECONDS", "1"), ("RECOVERY_POLL_SECONDS", "0"), ] { @@ -90,6 +92,55 @@ fn parsed_environment_still_requires_semantic_validation() { } } +/// Accept the API-service variables kubelet sets in a pod, and refuse to supervise when either is missing. +#[test] +fn supervision_requires_both_api_service_variables_a_pod_has() { + let pod: Pod = serde_env::from_iter(environment()).unwrap(); + pod.validate().unwrap(); + for key in ["KUBERNETES_SERVICE_HOST", "KUBERNETES_SERVICE_PORT"] { + let mut values = environment(); + let _removed = values.remove(key); + let pod: Pod = serde_env::from_iter(values).unwrap(); + assert!(pod.validate().unwrap_err().to_string().contains("outside a pod")); + } +} + +/// Refuse a kubeconfig kubectl would prefer to the service account, whether named or in the home directory. +#[test] +fn supervision_refuses_a_kubeconfig_that_kubectl_would_prefer() { + let home = tempfile::tempdir().unwrap(); + let mut values = environment(); + let _previous = values.insert("HOME".into(), home.path().to_str().unwrap().into()); + let pod: Pod = serde_env::from_iter(values.clone()).unwrap(); + pod.validate().unwrap(); + let mut named = values.clone(); + let _previous = named.insert("KUBECONFIG".into(), "kubeconfig.yaml".into()); + let pod: Pod = serde_env::from_iter(named).unwrap(); + assert!(pod.validate().unwrap_err().to_string().contains("KUBECONFIG")); + let directory = home.path().join(".kube"); + fs::create_dir(&directory).unwrap(); + fs::write(directory.join("config"), "").unwrap(); + let pod: Pod = serde_env::from_iter(values).unwrap(); + assert!(pod.validate().unwrap_err().to_string().contains(".kube")); +} + +/// Refuse to start outside a pod before creating state, even when every recovery setting is valid. +#[test] +fn binary_refuses_to_start_outside_a_pod() { + let directory = tempfile::tempdir().unwrap(); + let state = directory.path().join("state.json"); + // Raw output is intentional: rejection before any external action is the outcome under test. + let output = std::process::Command::new(env!("CARGO_BIN_EXE_et-recovery-supervisor")) + .envs(environment()) + .env("RECOVERY_STATE_FILE", &state) + .env_remove("KUBERNETES_SERVICE_HOST") + .output() + .unwrap(); + assert!(!output.status.success()); + assert!(String::from_utf8(output.stderr).unwrap().contains("outside a pod")); + assert!(!state.exists()); +} + /// Fail startup before writing state when environment configuration is incomplete. #[test] fn binary_rejects_malformed_environment_before_creating_state() { diff --git a/services/recovery-supervisor/tests/fixtures/kubeconfig.yaml b/services/recovery-supervisor/tests/fixtures/kubeconfig.yaml index 23c52c6f..87e8a259 100644 --- a/services/recovery-supervisor/tests/fixtures/kubeconfig.yaml +++ b/services/recovery-supervisor/tests/fixtures/kubeconfig.yaml @@ -3,4 +3,4 @@ kind: Config clusters: [] contexts: [] users: [] -current-context: "" +current-context: absent diff --git a/services/recovery-supervisor/tests/runtime.rs b/services/recovery-supervisor/tests/runtime.rs index 1383f8c5..44dbf2f3 100644 --- a/services/recovery-supervisor/tests/runtime.rs +++ b/services/recovery-supervisor/tests/runtime.rs @@ -325,7 +325,7 @@ async fn full_evidence_queue_waits_for_capacity_without_requiring_a_resend() { assert!(error.to_string().contains("recovery evidence queue unavailable")); } -/// Exercise real kubectl context errors using an isolated fixture without contacting a cluster. +/// Exercise real kubectl errors using a fixture whose current context is undefined, so no cluster is contacted. #[tokio::test] async fn kubernetes_reports_command_errors_without_contacting_a_cluster() { let _logging = set_default(tracing_subscriber::fmt().with_test_writer().finish()); @@ -358,6 +358,8 @@ fn binary_fails_closed_on_an_unavailable_state_directory() { let output = std::process::Command::new(env!("CARGO_BIN_EXE_et-recovery-supervisor")) .envs(environment()) .env("RECOVERY_STATE_FILE", &config.state_file) + .env_remove("KUBECONFIG") + .env("HOME", directory.path()) .output() .unwrap(); assert!(!output.status.success()); diff --git a/services/recovery-supervisor/tests/shutdown.rs b/services/recovery-supervisor/tests/shutdown.rs index 24f4e2e7..928b5c7f 100644 --- a/services/recovery-supervisor/tests/shutdown.rs +++ b/services/recovery-supervisor/tests/shutdown.rs @@ -2,6 +2,7 @@ #![cfg(test)] #![cfg(unix)] +use std::path::Path; use std::time::Duration; use edge_toolkit::ws::{ClientMessage, MessageDeliveryStatus, ServerMessage}; @@ -13,20 +14,22 @@ use tokio::io::{AsyncBufReadExt as _, BufReader}; use tokio::process::{Child, Command}; use tokio_tungstenite::tungstenite::Message; -/// Launch the production executable with isolated state and a kubeconfig that cannot contact a cluster. +/// Launch the production executable with isolated state and no way to reach a cluster. +/// +/// With no kubeconfig and no kubectl on its PATH, every observation is unknown. #[expect( clippy::single_call_fn, reason = "keep subprocess setup separate from shutdown assertions" )] -fn launch(config: &Config) -> Child { - let fixture = - edge_toolkit::config::get_project_root().join("services/recovery-supervisor/tests/fixtures/kubeconfig.yaml"); +fn launch(config: &Config, empty: &Path) -> Child { Command::new(env!("CARGO_BIN_EXE_et-recovery-supervisor")) .envs(environment()) .env("RECOVERY_WS_URL", &config.ws_url) .env("RECOVERY_STATE_FILE", &config.state_file) .env("RECOVERY_POLL_SECONDS", config.poll_seconds.to_string()) - .env("KUBECONFIG", fixture) + .env_remove("KUBECONFIG") + .env("HOME", empty) + .env("PATH", empty) .env("RUST_LOG", "warn") .stdout(std::process::Stdio::piped()) .kill_on_drop(true) @@ -105,12 +108,13 @@ async fn send(socket: &mut et_ws_runner_common::RegisteredSocket, kind: &str, pr #[tokio::test] async fn sigterm_saves_the_final_report_before_a_successful_exit() { let directory = tempfile::tempdir().unwrap(); + let empty = tempfile::tempdir().unwrap(); let server = et_ws_test_server::start(); let mut config = config(); config.ws_url.clone_from(&server.ws_url); config.state_file = directory.path().join("state.json"); config.poll_seconds = 3600; - let mut child = launch(&config); + let mut child = launch(&config, empty.path()); wait_for_initial_poll(&config, &mut child).await; let (mut producer, _) = et_ws_test_server::connect_agent_as(&server.ws_url, Some("example-worker")).await; send(&mut producer, "started", 0, 1).await; diff --git a/services/ws-modules/README.md b/services/ws-modules/README.md new file mode 100644 index 00000000..e263d088 --- /dev/null +++ b/services/ws-modules/README.md @@ -0,0 +1,77 @@ +# What browser modules can sense + +The modules in this directory run in the browser, so they reach the device's sensors, and the devices plugged into or +paired with it, only through the APIs that browser exposes. Those differ by browser and platform, so each sensor and +device type below is grouped by where it works, with the API, the browser versions, and the modules here that use it. +Support data is from [MDN's browser compatibility data][bcd], read on 2026-11-05. + +## Every modern browser, on phones and computers + +- **Cameras**, front and back, and USB webcams plugged into a computer: `navigator.mediaDevices.getUserMedia`. Used by + `video1`, `face-detection`, `pyface1`, `pyeye1` and `pydemo1`. +- **Microphone**: `getUserMedia`. Used by `audio1`, `pyspeech1` and `pydemo1`. `speech-recognition` also uses the + browser's own speech recognition (`SpeechRecognition`), which Chrome and Safari have, and Firefox only on computers. +- **Location**: the Geolocation API. Used by `geolocation`. +- **Motion and orientation**: `DeviceMotionEvent` (accelerometer and gyroscope) and `DeviceOrientationEvent` (how the + device is tilted and turned). Used by `sensor1` and `har1`. Phones and tablets have the hardware; most computers do + not. +- **Compass heading**: on every phone, through a different API in each engine -- the `deviceorientationabsolute` + event in Chrome and Firefox, and `DeviceOrientationEvent.webkitCompassHeading` in Safari. No module reads it yet. +- **Touchscreen, mouse and pen**: Pointer Events everywhere, and Touch Events on phones and tablets, giving taps, + swipes, timing and drawing. No module records them yet. + +## Chrome and other Chromium browsers, on Android and computers + +Chrome, Edge, Opera and Samsung Internet share these. Safari has none of them, so neither does any iPhone or iPad +browser built on Safari's engine, and Firefox has only Web Serial. + +- **Bluetooth devices**, such as heart-rate monitors: Web Bluetooth. Chrome on Android from 56, and on computers from + 70, though off by default on Linux. Android WebView lacks it, so an Android app that embeds the page in a WebView + gets none; only a full browser does. Used by `bluetooth`, whose [README](bluetooth/README.md) covers the iPhone + workaround. +- **USB devices**, generically: WebUSB, from Chrome 61, on Android as well as computers. No module uses it yet. +- **Serial devices**, such as microcontroller boards and instruments that present a serial port over USB: Web Serial. + Chrome on computers from 89 and on Android from 148, and Firefox on computers from 151. No module uses it yet. +- **Precise motion sensors**: the Generic Sensor API's `Accelerometer`, `Gyroscope`, `LinearAccelerationSensor`, + `AbsoluteOrientationSensor` and `RelativeOrientationSensor` from Chrome 67, and `GravitySensor` from 91. Unlike the + motion events above, the page chooses how often to sample. No module uses them yet. +- **Battery**: its charge level, and whether it is charging, through the Battery Status API from Chrome 38. No module + uses it yet. + +## Chrome on Android only + +- **NFC tags**: Web NFC (`NDEFReader`), from Chrome 89. Used by `nfc`. + +## Chrome on computers only + +- **Human-interface devices** over USB or Bluetooth, such as game controllers and foot pedals: WebHID, from Chrome 89. + No module uses it yet. + +## Chrome, switched off by default + +- **Ambient light**: `AmbientLightSensor`. +- **Magnetometer**: `Magnetometer`, the raw magnetic field. + +Chrome has both from 56, behind a setting that is off by default, so a participant would have to change it. For +direction alone, the compass heading above needs no setting. + +## iPhone and iPad + +Apple's App Store rules require every browser on an iPhone or iPad to use Safari's engine, WebKit, except that in +the EU and Japan a browser may apply to use another engine. WebKit browsers all have the first group and nothing from +the Chrome groups, and two things differ from other phones: + +- **Compass heading** comes from `webkitCompassHeading`, which only Safari's engine has. +- **Motion and orientation** reach the page only after it asks permission with + `DeviceMotionEvent.requestPermission`, as `har1` does. + +The one Chrome-group device an iPhone can reach is Bluetooth, through the Bluefy browser app that the `bluetooth` +module's README describes. + +## Not available to any web page + +No browser exposes the barometer, the proximity sensor, the phone's own step counter, or heart-rate and blood-oxygen +readings from the phone or a paired watch, and none reaches Apple Health or Google Health Connect. A study that needs +them needs a native app, or an external device the browser can reach through Bluetooth, USB or a serial port. + +[bcd]: https://github.com/mdn/browser-compat-data diff --git a/services/ws-modules/bluetooth/README.md b/services/ws-modules/bluetooth/README.md new file mode 100644 index 00000000..be229251 --- /dev/null +++ b/services/ws-modules/bluetooth/README.md @@ -0,0 +1,43 @@ +# bluetooth + +The `bluetooth` module lets the participant pick a nearby Bluetooth Low Energy device through the +[Web Bluetooth API](https://developer.mozilla.org/docs/Web/API/Web_Bluetooth_API). Its `run()` calls +`navigator.bluetooth.requestDevice` with `acceptAllDevices`, so the browser shows its own device picker, then reports +the chosen device's id and name to the hub as a `device_selected` client event; it does not connect to the device. +`BluetoothAccess` also exposes `connectGatt`, which opens the device's GATT server, for a module that needs one. + +## iPhone and iPad: no standard browser can run it + +Web Bluetooth does not exist in Safari, nor in any other standard browser on an iPhone or iPad, so this module cannot +run there in Safari, Chrome, Firefox or Edge. It stops with + +```text +Web Bluetooth is not available in this browser context +``` + +Three independent sources establish it: + +- **Safari has no Web Bluetooth.** MDN's browser compatibility data marks it unsupported in Safari on macOS and iOS, + and in the iOS WebView that iPhone apps embed ([mdn/browser-compat-data `api/Bluetooth.json`][bcd]). +- **Apple does not intend to add it.** WebKit recorded a formal `oppose` position on 2 December 2025, citing + security, privacy and the web's device independence ([WebKit/standards-positions#570][webkit]). +- **Every iPhone browser runs Safari's engine.** App Store Review Guideline 2.5.6 requires every app that browses + the web to use WebKit, so Chrome, Firefox and Edge on an iPhone inherit Safari's gap ([App Store Review + Guidelines][guidelines]). Apple lets a browser apply to use another engine only in the EU and Japan, so everywhere + else, Australia included, the limit holds for every standard iPhone browser. + +## Workaround on iPhone and iPad: Bluefy + +[Bluefy -- Web BLE Browser][bluefy] is a free App Store browser that adds Web Bluetooth to the pages it opens. A +participant who installs it and opens the hub's page in Bluefy rather than Safari can run this module. Plan for it +before recruiting iPhone users: + +- Serve the page over HTTPS. Web Bluetooth is only exposed to secure contexts, and Bluefy's App Store listing says + the same, so give iPhone participants the hub's TLS address. +- Bluefy is a third-party app, not part of edge-toolkit or iOS. Test the study's page in it end to end, at the + address participants will use, and tell participants in their instructions that they need it. + +[bcd]: https://github.com/mdn/browser-compat-data/blob/main/api/Bluetooth.json +[webkit]: https://github.com/WebKit/standards-positions/issues/570 +[guidelines]: https://developer.apple.com/app-store/review/guidelines/ +[bluefy]: https://apps.apple.com/app/id1492822055 diff --git a/services/ws-server/Dockerfile b/services/ws-server/Dockerfile index f23a7a55..d1366f42 100644 --- a/services/ws-server/Dockerfile +++ b/services/ws-server/Dockerfile @@ -19,7 +19,7 @@ # docker build -t et-ws-server -f services/ws-server/Dockerfile . # docker run --rm -p 8080:8080 et-ws-server # then open http://localhost:8080 -FROM rust:1-bookworm AS builder +FROM rust:1-trixie AS builder # clang and cmake are load-bearing here rather than conveniences. # rustls resolves to aws-lc-rs, and the aws-lc-sys build script drives cmake over the vendored C sources plus @@ -98,7 +98,7 @@ cd services/ws-wasm-agent wasm-pack build . --target web EOF -FROM debian:bookworm-slim AS runtime +FROM debian:trixie-slim AS runtime LABEL org.opencontainers.image.source="https://github.com/edge-toolkit/core" diff --git a/utilities/cli/src/deployment_types/k3s_recovery.rs b/utilities/cli/src/deployment_types/k3s_recovery.rs index a53321ea..11d01089 100644 --- a/utilities/cli/src/deployment_types/k3s_recovery.rs +++ b/utilities/cli/src/deployment_types/k3s_recovery.rs @@ -1,25 +1,21 @@ -//! Kubernetes recovery deployment: explicit credentials, named-target RBAC and a durable single-owner budget. +//! Kubernetes recovery deployment: service-account credentials, named-target RBAC and a durable single-owner budget. use std::collections::BTreeMap; use et_recovery_supervisor::config::Mode; use k8s_openapi::api::apps::v1::{Deployment, DeploymentStrategy}; -use k8s_openapi::api::core::v1::{ - ConfigMap, ConfigMapVolumeSource, Container, ExecAction, Probe, ServiceAccount, Volume, -}; +use k8s_openapi::api::core::v1::{Container, ExecAction, Probe, ServiceAccount}; use k8s_openapi::api::rbac::v1::{PolicyRule, Role, RoleBinding, RoleRef, Subject}; use super::document; use super::k3s::{ IMAGE_PULL_POLICY, IMAGE_TAG, claim, deployment, env, hardened_container_context, meta, mount, plain_meta, scratch, - volume_mount, }; use crate::placement::hub_site; use crate::recovery::{Recovery, RecoverySettings}; use crate::{CliError, ClusterInput, RunnerInstance, image_prefix}; const NAME: &str = "recovery-supervisor"; -const CONFIG: &str = "recovery-credentials"; const STATE: &str = "recovery-state"; /// Inject only the bindings of allowlisted native-Python workloads. @@ -55,20 +51,13 @@ pub(super) fn documents(cluster: &ClusterInput, namespace: &str) -> Result RoleBinding { } } -/// Use the projected, rotating `ServiceAccount` token and cluster CA, never the operator's host kubeconfig. -fn kubeconfig(namespace: &str) -> Result { - let service_account = "/var/run/secrets/kubernetes.io/serviceaccount"; - Ok(serde_yaml::to_string(&serde_json::json!({ - "apiVersion": "v1", "kind": "Config", "current-context": "in-cluster", - "clusters": [{"name": "cluster", "cluster": { - "server": "https://kubernetes.default.svc", "certificate-authority": format!("{service_account}/ca.crt") - }}], - "users": [{"name": "supervisor", "user": {"tokenFile": format!("{service_account}/token")}}], - "contexts": [{"name": "in-cluster", "context": { - "cluster": "cluster", "user": "supervisor", "namespace": namespace - }}] - }))?) -} - -/// Mount persistent state separately from credentials and writable kubectl discovery caches. +/// Mount persistent state separately from writable kubectl discovery caches. fn supervisor_deployment( cluster: &ClusterInput, namespace: &str, @@ -159,7 +133,6 @@ fn supervisor_deployment( ) -> Result { let (state_mount, state_volume) = mount(STATE, "/app/state"); let (runtime_mount, runtime_volume) = scratch("recovery-runtime", "/app/runtime"); - let credentials = credentials_volume(); let (probe, startup) = health_probes(&settings.config(namespace)?)?; let container = Container { name: NAME.into(), @@ -173,18 +146,14 @@ fn supervisor_deployment( startup_probe: Some(startup), readiness_probe: Some(probe.clone()), liveness_probe: Some(probe), - volume_mounts: Some(vec![ - state_mount, - runtime_mount, - volume_mount(CONFIG, "/app/credentials"), - ]), + volume_mounts: Some(vec![state_mount, runtime_mount]), ..Container::default() }; let mut object = deployment( namespace, NAME, container, - vec![state_volume, runtime_volume, credentials], + vec![state_volume, runtime_volume], &hub_site(cluster), &settings.limits, ); @@ -197,22 +166,6 @@ fn supervisor_deployment( Ok(object) } -/// Project the generated kubeconfig independently of writable state and runtime directories. -#[expect( - clippy::single_call_fn, - reason = "the read-only credential volume is a distinct deployment resource" -)] -fn credentials_volume() -> Volume { - Volume { - name: CONFIG.into(), - config_map: Some(ConfigMapVolumeSource { - name: CONFIG.into(), - ..ConfigMapVolumeSource::default() - }), - ..Volume::default() - } -} - /// Allow registration and a full bounded cycle before liveness checks can restart a starting supervisor. #[expect( clippy::single_call_fn, diff --git a/utilities/cli/tests/recovery.rs b/utilities/cli/tests/recovery.rs index 432e837b..01faa864 100644 --- a/utilities/cli/tests/recovery.rs +++ b/utilities/cli/tests/recovery.rs @@ -93,7 +93,6 @@ fn generates_valid_supervisor_settings_and_scoped_in_cluster_credentials() { let env = environment(deployment); let config: EnvConfig = serde_env::from_iter(env).unwrap(); config.recovery.validate().unwrap(); - assert_eq!(config.recovery.context, "in-cluster"); assert_eq!(config.recovery.mode, Mode::ObserveOnly); assert_eq!(config.recovery.namespace, "et-pyo3-math1-recovery"); assert_eq!(config.recovery.workloads[0].deployment, "example-worker"); @@ -108,16 +107,29 @@ fn generates_valid_supervisor_settings_and_scoped_in_cluster_credentials() { let binding = object(&objects, "RoleBinding", "recovery-supervisor"); assert_eq!(binding["roleRef"]["kind"], "Role"); assert_eq!(binding["subjects"][0]["namespace"], "et-pyo3-math1-recovery"); - let credentials = object(&objects, "ConfigMap", "recovery-credentials")["data"]["kubeconfig"] - .as_str() - .unwrap(); - let kubeconfig: Value = serde_yaml::from_str(credentials).unwrap(); - assert_eq!(kubeconfig["current-context"], "in-cluster"); + assert!( + !objects + .iter() + .any(|object| object_name(object).starts_with("recovery-") && object["kind"] == "ConfigMap") + ); + let pod = &deployment["spec"]["template"]["spec"]; + assert_eq!(pod["automountServiceAccountToken"].as_bool(), Some(true)); + assert!( + pod["containers"][0]["env"] + .as_array() + .unwrap() + .iter() + .all(|item| item["name"] != "KUBECONFIG") + ); assert_eq!( - kubeconfig["users"][0]["user"]["tokenFile"], - "/var/run/secrets/kubernetes.io/serviceaccount/token" + pod["containers"][0]["volumeMounts"] + .as_array() + .unwrap() + .iter() + .map(|item| item["mountPath"].as_str().unwrap()) + .collect::>(), + ["/app/state", "/app/runtime"] ); - assert!(kubeconfig["users"][0]["user"]["token"].is_null()); assert!( !objects .iter() diff --git a/verification/local/output/default/Dockerfile.dockerignore b/verification/local/output/default/Dockerfile.dockerignore index a6816fee..fba7c35e 100644 --- a/verification/local/output/default/Dockerfile.dockerignore +++ b/verification/local/output/default/Dockerfile.dockerignore @@ -60,6 +60,8 @@ lcov.info # Dart pub tool dir, regenerated by `dart pub get`; one global rule for the whole workspace (root + members). **/.dart_tool/ services/ws-server/storage/ +# One-off Markdown an agent writes on request, such as an email draft or notes, which is never committed. +**/*.local.md # Lock files a generated scenario leaves behind when it is run. # `lockfile = true` makes mise write one beside whichever config it resolved tools for, so running a scenario # drops `mise.lock` into that scenario's output directory. It is a by-product of running the deployment rather diff --git a/verification/local/output/expose/Dockerfile.dockerignore b/verification/local/output/expose/Dockerfile.dockerignore index 31581dc2..3b6ec3c1 100644 --- a/verification/local/output/expose/Dockerfile.dockerignore +++ b/verification/local/output/expose/Dockerfile.dockerignore @@ -60,6 +60,8 @@ lcov.info # Dart pub tool dir, regenerated by `dart pub get`; one global rule for the whole workspace (root + members). **/.dart_tool/ services/ws-server/storage/ +# One-off Markdown an agent writes on request, such as an email draft or notes, which is never committed. +**/*.local.md # Lock files a generated scenario leaves behind when it is run. # `lockfile = true` makes mise write one beside whichever config it resolved tools for, so running a scenario # drops `mise.lock` into that scenario's output directory. It is a by-product of running the deployment rather diff --git a/verification/local/output/facility-security-scenario/Dockerfile.dockerignore b/verification/local/output/facility-security-scenario/Dockerfile.dockerignore index 86e22146..e086386a 100644 --- a/verification/local/output/facility-security-scenario/Dockerfile.dockerignore +++ b/verification/local/output/facility-security-scenario/Dockerfile.dockerignore @@ -60,6 +60,8 @@ lcov.info # Dart pub tool dir, regenerated by `dart pub get`; one global rule for the whole workspace (root + members). **/.dart_tool/ services/ws-server/storage/ +# One-off Markdown an agent writes on request, such as an email draft or notes, which is never committed. +**/*.local.md # Lock files a generated scenario leaves behind when it is run. # `lockfile = true` makes mise write one beside whichever config it resolved tools for, so running a scenario # drops `mise.lock` into that scenario's output directory. It is a by-product of running the deployment rather diff --git a/verification/local/output/math1-cron/Dockerfile.dockerignore b/verification/local/output/math1-cron/Dockerfile.dockerignore index 50524511..117571da 100644 --- a/verification/local/output/math1-cron/Dockerfile.dockerignore +++ b/verification/local/output/math1-cron/Dockerfile.dockerignore @@ -60,6 +60,8 @@ lcov.info # Dart pub tool dir, regenerated by `dart pub get`; one global rule for the whole workspace (root + members). **/.dart_tool/ services/ws-server/storage/ +# One-off Markdown an agent writes on request, such as an email draft or notes, which is never committed. +**/*.local.md # Lock files a generated scenario leaves behind when it is run. # `lockfile = true` makes mise write one beside whichever config it resolved tools for, so running a scenario # drops `mise.lock` into that scenario's output directory. It is a by-product of running the deployment rather diff --git a/verification/local/output/math1-limits/Dockerfile.dockerignore b/verification/local/output/math1-limits/Dockerfile.dockerignore index 104158aa..0afb0777 100644 --- a/verification/local/output/math1-limits/Dockerfile.dockerignore +++ b/verification/local/output/math1-limits/Dockerfile.dockerignore @@ -60,6 +60,8 @@ lcov.info # Dart pub tool dir, regenerated by `dart pub get`; one global rule for the whole workspace (root + members). **/.dart_tool/ services/ws-server/storage/ +# One-off Markdown an agent writes on request, such as an email draft or notes, which is never committed. +**/*.local.md # Lock files a generated scenario leaves behind when it is run. # `lockfile = true` makes mise write one beside whichever config it resolved tools for, so running a scenario # drops `mise.lock` into that scenario's output directory. It is a by-product of running the deployment rather diff --git a/verification/local/output/math1-multi-hub/Dockerfile.dockerignore b/verification/local/output/math1-multi-hub/Dockerfile.dockerignore index 1e6b10d4..c76560eb 100644 --- a/verification/local/output/math1-multi-hub/Dockerfile.dockerignore +++ b/verification/local/output/math1-multi-hub/Dockerfile.dockerignore @@ -60,6 +60,8 @@ lcov.info # Dart pub tool dir, regenerated by `dart pub get`; one global rule for the whole workspace (root + members). **/.dart_tool/ services/ws-server/storage/ +# One-off Markdown an agent writes on request, such as an email draft or notes, which is never committed. +**/*.local.md # Lock files a generated scenario leaves behind when it is run. # `lockfile = true` makes mise write one beside whichever config it resolved tools for, so running a scenario # drops `mise.lock` into that scenario's output directory. It is a by-product of running the deployment rather diff --git a/verification/local/output/math1-multi/Dockerfile.dockerignore b/verification/local/output/math1-multi/Dockerfile.dockerignore index 2a0d2eda..09b79819 100644 --- a/verification/local/output/math1-multi/Dockerfile.dockerignore +++ b/verification/local/output/math1-multi/Dockerfile.dockerignore @@ -60,6 +60,8 @@ lcov.info # Dart pub tool dir, regenerated by `dart pub get`; one global rule for the whole workspace (root + members). **/.dart_tool/ services/ws-server/storage/ +# One-off Markdown an agent writes on request, such as an email draft or notes, which is never committed. +**/*.local.md # Lock files a generated scenario leaves behind when it is run. # `lockfile = true` makes mise write one beside whichever config it resolved tools for, so running a scenario # drops `mise.lock` into that scenario's output directory. It is a by-product of running the deployment rather diff --git a/verification/local/output/math1/Dockerfile.dockerignore b/verification/local/output/math1/Dockerfile.dockerignore index 7c8d89d8..874b2af4 100644 --- a/verification/local/output/math1/Dockerfile.dockerignore +++ b/verification/local/output/math1/Dockerfile.dockerignore @@ -60,6 +60,8 @@ lcov.info # Dart pub tool dir, regenerated by `dart pub get`; one global rule for the whole workspace (root + members). **/.dart_tool/ services/ws-server/storage/ +# One-off Markdown an agent writes on request, such as an email draft or notes, which is never committed. +**/*.local.md # Lock files a generated scenario leaves behind when it is run. # `lockfile = true` makes mise write one beside whichever config it resolved tools for, so running a scenario # drops `mise.lock` into that scenario's output directory. It is a by-product of running the deployment rather diff --git a/verification/local/output/pyo3-math1-recovery/Dockerfile.dockerignore b/verification/local/output/pyo3-math1-recovery/Dockerfile.dockerignore index 3d30949a..9cdba7a2 100644 --- a/verification/local/output/pyo3-math1-recovery/Dockerfile.dockerignore +++ b/verification/local/output/pyo3-math1-recovery/Dockerfile.dockerignore @@ -60,6 +60,8 @@ lcov.info # Dart pub tool dir, regenerated by `dart pub get`; one global rule for the whole workspace (root + members). **/.dart_tool/ services/ws-server/storage/ +# One-off Markdown an agent writes on request, such as an email draft or notes, which is never committed. +**/*.local.md # Lock files a generated scenario leaves behind when it is run. # `lockfile = true` makes mise write one beside whichever config it resolved tools for, so running a scenario # drops `mise.lock` into that scenario's output directory. It is a by-product of running the deployment rather diff --git a/verification/local/output/pyo3-math1-recovery/k3s.yaml b/verification/local/output/pyo3-math1-recovery/k3s.yaml index 6bc5c414..5f978ed2 100644 --- a/verification/local/output/pyo3-math1-recovery/k3s.yaml +++ b/verification/local/output/pyo3-math1-recovery/k3s.yaml @@ -320,32 +320,6 @@ subjects: namespace: et-pyo3-math1-recovery --- apiVersion: v1 -kind: ConfigMap -data: - kubeconfig: | - apiVersion: v1 - clusters: - - cluster: - certificate-authority: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt - server: https://kubernetes.default.svc - name: cluster - contexts: - - context: - cluster: cluster - namespace: et-pyo3-math1-recovery - user: supervisor - name: in-cluster - current-context: in-cluster - kind: Config - users: - - name: supervisor - user: - tokenFile: /var/run/secrets/kubernetes.io/serviceaccount/token -metadata: - name: recovery-credentials - namespace: et-pyo3-math1-recovery ---- -apiVersion: v1 kind: PersistentVolumeClaim metadata: name: recovery-state @@ -379,14 +353,10 @@ spec: automountServiceAccountToken: true containers: - env: - - name: KUBECONFIG - value: /app/credentials/kubeconfig - name: RECOVERY_AGENT_ID value: recovery-supervisor - name: RECOVERY_COMMAND_TIMEOUT_SECONDS value: "10" - - name: RECOVERY_CONTEXT - value: in-cluster - name: RECOVERY_JOB_ID value: example-job - name: RECOVERY_MODE @@ -458,8 +428,6 @@ spec: name: recovery-state - mountPath: /app/runtime name: recovery-runtime - - mountPath: /app/credentials - name: recovery-credentials securityContext: fsGroup: 10001 runAsGroup: 10001 @@ -474,6 +442,3 @@ spec: claimName: recovery-state - emptyDir: {} name: recovery-runtime - - configMap: - name: recovery-credentials - name: recovery-credentials diff --git a/verification/local/output/pyo3-math1/Dockerfile.dockerignore b/verification/local/output/pyo3-math1/Dockerfile.dockerignore index c5730b57..6b73225f 100644 --- a/verification/local/output/pyo3-math1/Dockerfile.dockerignore +++ b/verification/local/output/pyo3-math1/Dockerfile.dockerignore @@ -60,6 +60,8 @@ lcov.info # Dart pub tool dir, regenerated by `dart pub get`; one global rule for the whole workspace (root + members). **/.dart_tool/ services/ws-server/storage/ +# One-off Markdown an agent writes on request, such as an email draft or notes, which is never committed. +**/*.local.md # Lock files a generated scenario leaves behind when it is run. # `lockfile = true` makes mise write one beside whichever config it resolved tools for, so running a scenario # drops `mise.lock` into that scenario's output directory. It is a by-product of running the deployment rather diff --git a/verification/local/output/wasi-math1/Dockerfile.dockerignore b/verification/local/output/wasi-math1/Dockerfile.dockerignore index 956b698d..3cd412df 100644 --- a/verification/local/output/wasi-math1/Dockerfile.dockerignore +++ b/verification/local/output/wasi-math1/Dockerfile.dockerignore @@ -60,6 +60,8 @@ lcov.info # Dart pub tool dir, regenerated by `dart pub get`; one global rule for the whole workspace (root + members). **/.dart_tool/ services/ws-server/storage/ +# One-off Markdown an agent writes on request, such as an email draft or notes, which is never committed. +**/*.local.md # Lock files a generated scenario leaves behind when it is run. # `lockfile = true` makes mise write one beside whichever config it resolved tools for, so running a scenario # drops `mise.lock` into that scenario's output directory. It is a by-product of running the deployment rather