Use in-cluster ServiceAccount credentials

.dockerignore

@@ -53,6 +53,8 @@ lcov.info
 # Dart pub tool dir, regenerated by `dart pub get`; one global rule for the whole workspace (root + members).
 **/.dart_tool/
 services/ws-server/storage/
+# One-off Markdown an agent writes on request, such as an email draft or notes, which is never committed.
+**/*.local.md
 # Lock files a generated scenario leaves behind when it is run.
 # `lockfile = true` makes mise write one beside whichever config it resolved tools for, so running a scenario
 # drops `mise.lock` into that scenario's output directory. It is a by-product of running the deployment rather

.github/actions/k3s-scenario/action.yaml

@@ -0,0 +1,33 @@
+---
+name: Verify a k3s scenario
+description: >-
+  Installs the mise tools, builds every image a local scenario's k3s manifests name, and applies them to a
+  throwaway k3d cluster with `k3s-verify`. One job per scenario calls it after `actions/checkout`, which a local
+  action cannot do itself: GitHub reads this file from the checked-out tree. The calling job's `timeout-minutes`
+  bounds the whole of it, since a composite action's steps take no timeout of their own.
+
+inputs:
+  scenario:
+    description: >-
+      The scenario under verification/local/output to build and apply. Its module packages must already be built
+      when they are wasm-pack artifacts; committed module sources need nothing.
+    required: true
+
+runs:
+  using: composite
+  steps:
+    - name: Install mise + tools
+      uses: $/.github/actions/install-mise-tools
+
+    - name: Build the scenario's images
+      shell: bash --noprofile --norc -euo pipefail {0}
+      env:
+        SCENARIO: ${{ inputs.scenario }}
+      run: mise run k3s-images --scenario "$SCENARIO"
+
+    - name: Apply the manifests and verify the deployment
+      shell: bash --noprofile --norc -euo pipefail {0}
+      env:
+        CARGO_NET_RETRY: "5"
+        SCENARIO: ${{ inputs.scenario }}
+      run: mise run k3s-verify --scenario "$SCENARIO"

.github/workflows/coverage.yaml

@@ -42,12 +42,12 @@ jobs:
           - os: ubuntu-latest
             os_flag: linux
             timeout: 150
-          # Longer than the Linux lane, despite doing less.
+          # As long as the Linux lane, despite doing less.
           # The macOS runners are slower per-core, and this one still pays the full instrumented rebuild even
           # though it skips the wasm/browser steps.
           - os: macos-latest
             os_flag: macos
-            timeout: 180
+            timeout: 150
     env:
       MISE_ENV: dart,dotnet,java,js,kotlin,python,r,rust,zig,coverage
       ET_TEST_COVERAGE: "true"

.github/workflows/k3s.yaml

@@ -17,6 +17,7 @@ name: k3s
       - .github/actions/add-swap/**
       - .github/actions/free-disk-space/**
       - .github/actions/install-mise-tools/**
+      - .github/actions/k3s-scenario/**
       - .github/workflows/k3s.yaml
       # The `k3s-verify` task itself lives here.
       # A change to it must re-trigger this job rather than letting an edit to the verification steps ship
@@ -52,6 +53,13 @@ name: k3s
       - verification/published/input/math1-multi.yaml
       - verification/published/output/math1-multi/Dockerfile
       - verification/published/output/math1-multi/k3s.yaml
+      # What the `recovery` job builds and applies: the supervisor and how it reaches the API, its runner target.
+      - services/recovery-supervisor/**
+      - services/ws-pyo3-runner/Dockerfile
+      - utilities/cli/src/recovery.rs
+      - verification/local/input/pyo3-math1-recovery.yaml
+      - verification/local/output/pyo3-math1-recovery/Dockerfile
+      - verification/local/output/pyo3-math1-recovery/k3s.yaml
   workflow_dispatch:
 
 permissions:
@@ -176,6 +184,29 @@ jobs:
           CARGO_NET_RETRY: "5"
         run: mise run compose-verify --scenario math1-limits
 
+  # The recovery supervisor reaching the Kubernetes API through its pod's service account, with no kubeconfig.
+  # Only a live pod has that service account, so no unit test can show it. Its own job, so its three release image
+  # builds -- the hub, the native-Python runner and the supervisor -- run beside `verify` rather than after it.
+  # The scenario's one module is committed Python source, so nothing is built before the images.
+  recovery:
+    runs-on: ubuntu-latest
+    # Read-only pull of the maintainer-published ghcr.io mise-tools store, as in `verify`.
+    permissions:
+      contents: read
+      packages: read
+    timeout-minutes: 150
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v7
+        with:
+          fetch-depth: 1
+          persist-credentials: false
+
+      - name: Verify the recovery scenario
+        uses: $/.github/actions/k3s-scenario
+        with:
+          scenario: pyo3-math1-recovery
+
   # Whether what is on crates.io and GitHub Packages actually runs, which no other job asks.
   # Every sibling check reads this working tree; this one installs the released hub, runner and module packages
   # and runs the math1 exchange on them, so it is the only signal that a release is usable.

.github/workflows/test.yaml

@@ -96,7 +96,7 @@ jobs:
           '{"os":"ubuntu-26.04","timeout":40}',
           '{"os":"ubuntu-24.04-arm","timeout":50}',
           '{"os":"macos-latest","timeout":75}',
-          '{"os":"macos-26-intel","timeout":225,"install_timeout":120}',
+          '{"os":"macos-26-intel","timeout":150,"install_timeout":120}',
           '{"os":"windows-latest","timeout":120}',
           '{"os":"windows-11-arm","timeout":60}'
           )
@@ -105,7 +105,7 @@ jobs:
           || github.event.inputs.os == 'ubuntu-24.04-arm' && '[{"os":"ubuntu-24.04-arm","timeout":50}]'
           || github.event.inputs.os == 'macos-latest' && '[{"os":"macos-latest","timeout":75}]'
           || github.event.inputs.os == 'macos-26-intel'
-          && '[{"os":"macos-26-intel","timeout":225,"install_timeout":120}]'
+          && '[{"os":"macos-26-intel","timeout":150,"install_timeout":120}]'
           || github.event.inputs.os == 'windows-latest' && '[{"os":"windows-latest","timeout":120}]'
           || github.event.inputs.os == 'windows-11-arm' && '[{"os":"windows-11-arm","timeout":60}]'
           )
@@ -199,11 +199,11 @@ jobs:
       packages: read
       id-token: write
     runs-on: windows-latest
-    # Covers the step ceilings this job declares, which now sum past the old 120.
-    # Install 60 + prefetch 20 + build-modules 25 + tests 55 is 160, so at 120 a slow setup could have the
-    # job cancelled while the test step still had budget left -- the test step's own limit would never apply.
-    # In practice msvc runs ~68m end to end; this only binds when something is already pathological.
-    timeout-minutes: 165
+    # Covers every step ceiling this job declares.
+    # Install 60 + prefetch 15 + build-modules 20 + tests 55 is 150, so a slow setup cannot have the job cancelled
+    # while the test step still has budget left. In practice msvc runs ~68m end to end; this only binds when
+    # something is already pathological.
+    timeout-minutes: 150
     strategy:
       fail-fast: false
       matrix:
@@ -245,7 +245,7 @@ jobs:
           key: portable-msvc-${{ hashFiles('.mise/config.msvc.toml') }}
 
       - name: Prefetch dependencies
-        timeout-minutes: 20
+        timeout-minutes: 15
         env:
           GITHUB_TOKEN: ${{ github.token }}
         run: mise run prefetch-ci
@@ -255,7 +255,7 @@ jobs:
         run: echo "MISE_JOBS=1" >> "$GITHUB_ENV"
 
       - name: Build WASM modules
-        timeout-minutes: 25
+        timeout-minutes: 20
         env:
           GITHUB_TOKEN: ${{ github.token }}
         run: mise run build-modules

.gitignore

@@ -49,6 +49,8 @@ coverage-python.xml
 # Dart pub tool dir, regenerated by `dart pub get`; one global rule for the whole workspace (root + members).
 **/.dart_tool/
 services/ws-server/storage/
+# One-off Markdown an agent writes on request, such as an email draft or notes, which is never committed.
+*.local.md
 # Lock files a generated scenario leaves behind when it is run.
 # `lockfile = true` makes mise write one beside whichever config it resolved tools for, so running a scenario
 # drops `mise.lock` into that scenario's output directory. It is a by-product of running the deployment rather

.mise/config.toml

@@ -804,6 +804,8 @@ toml_files = "git ls-files -z --cached --others --exclude-standard '*.toml'"
 task_shell_trace = 'bash -xeuo pipefail "{{ config_root }}/.mise/task-shell.sh"'
 # Unguarded variant, for a body that checks its own exit codes and must not abort on the first failure.
 task_shell_plain = 'bash "{{ config_root }}/.mise/task-shell.sh"'
+# A yq program printing every image a k3s manifest names without a registry: the ones this repository builds.
+k3s_local_images = '.. | select(tag == "!!map" and has("image")) | .image | select(test("/") | not)'
 # OpenObserve docker image for the openobserve/o2 dev-observability task.
 o2_image = "openobserve/openobserve:v1.1.2"
 # Port publish spec for that task, bound to loopback rather than every interface.
@@ -1906,17 +1908,22 @@ shell = "{{ vars.task_shell }}"
 # stands up a throwaway k3s cluster (k3d runs one in Docker, so no root and no systemd), applies the scenario,
 # and asserts the twin stored the same model the mise and compose deployments are held to.
 #
-# The images are the caller's to build: they come from this repository rather than a registry, and building
-# them dwarfs the run itself, so a missing one fails immediately rather than starting a deployment whose pods
-# can only sit in ErrImagePull. Deliberately not part of the `check` aggregate -- it needs Docker and those
-# images, which no other check does.
+# The images are the caller's to build, with `k3s-images`: they come from this repository rather than a
+# registry, and building them dwarfs the run itself, so a missing one fails immediately rather than starting a
+# deployment whose pods can only sit in ErrImagePull. Deliberately not part of the `check` aggregate -- it needs
+# Docker and those images, which no other check does.
+#
+# A scenario with a `recovery:` block has no math1 exchange, so what it is held to instead is the supervisor's
+# own access to the cluster. Its rollout completes only once a supervision cycle has, and every cycle asks the
+# API about each workload through the pod's service account. A refused or unreachable request does not stop the
+# cycle; it is counted in the report's `unknown_observations`, so that count staying at zero is the assertion.
 #
 # `--scenario` picks another math1-shaped scenario. One that names machines gets a k3d node per machine, the
 # first as the server and the rest as agents, each carrying the node label the manifests pin its pods to, and
 # the run then checks every pod landed on its own machine's node. k3d's nodes are full k3s nodes joined to the
 # server by token, so the scheduling and the cross-node traffic are real even though one host runs them all.
 [tasks.k3s-verify]
-description = "Apply a generated math1 scenario's k3s manifests to a throwaway cluster and verify the stored model"
+description = "Apply a generated scenario's k3s manifests to a throwaway cluster and verify the deployment works"
 run = """
 scenario="${usage_scenario:-math1}"
 source="${usage_source:-local}"
@@ -1925,6 +1932,7 @@ cluster="et-$scenario-verify"
 out="verification/$source/output/$scenario"
 input="verification/$source/input/$scenario.yaml"
 hub_image="et-ws-server-$scenario:latest"
+recovery=$(yq '.recovery != null' "$input")
 manifest_wait=300s
 machine_label=edge-toolkit/machine
 # One name per line; each is an RFC 1123 label, so the unquoted word-splitting loops over it below are safe.
@@ -1950,13 +1958,16 @@ if [ "$source" = "published" ]; then
   # on https://github.com/edge-toolkit/core/commit/bf8e71a0565e58c8fabb4d53640970eb181c5726 at
   # https://github.com/edge-toolkit/core/actions/runs/41288288633/job/124794004602, the first ubuntu-26.04 runners.
   printf 'FROM %s\\n' "$runner_image" | docker build --provenance=false -t "$runner_image" -
+  images="$hub_image $runner_image"
 else
-  runner_image="et-ws-web-runner:latest"
+  # The hub, each runner kind and a recovery supervisor are whichever the manifest names, so none is assumed.
+  images=$(yq -N '{{ vars.k3s_local_images }}' "$out/k3s.yaml" | coreutils sort -u)
 fi
 
-for image in "$hub_image" "$runner_image"; do
+# Word-split on purpose: an image reference holds no whitespace.
+for image in $images; do
   if ! docker image inspect "$image" >/dev/null 2>&1; then
-    echo "k3s-verify: $image is not built; $out/README.md has the build and import steps" >&2
+    echo "k3s-verify: $image is not built; $out/README.md has the steps, or for a local scenario, k3s-images" >&2
     exit 1
   fi
 done
@@ -1995,7 +2006,7 @@ trap 'status=$?;
   k3d cluster delete "$cluster" >/dev/null 2>&1 || true
   coreutils rm -rf "$secrets_dir"' EXIT
 
-cargo run -q -p et-cli -- generate-deployment --input-file "$input" --output-dir "$secrets_dir"
+cargo run -q -p et-cli -- generate-deployment --input-file "$input" --output-dir "$secrets_dir" --output-type k3s
 
 # The first machine is the server node and every other one an agent.
 # Each is labelled as the README's join commands label them. A scenario naming no machines gets no flags, which
@@ -2042,13 +2053,15 @@ for _ in $(coreutils seq 1 60); do
 done
 kubectl wait --for=condition=Ready node --all --timeout="$manifest_wait"
 # `k3d image import` loads every node, so a pod finds its image whichever machine it is pinned to.
-k3d image import -c "$cluster" "$hub_image" "$runner_image"
-# Every node is checked for both, since the import reports success even when a node's containerd drops an image.
+# $images is a word-split list of references by design; do not quote it.
+# shellcheck disable=SC2086
+k3d image import -c "$cluster" $images
+# Every node is checked for each, since the import reports success even when a node's containerd drops an image.
 # k3d names each node's container as Kubernetes names the node, and a bare image name is docker.io's library one.
 for node in $(kubectl get nodes -o name); do
   node=${node#node/}
   held=$(docker exec "$node" ctr -n k8s.io images ls -q)
-  for image in "$hub_image" "$runner_image"; do
+  for image in $images; do
     case "$image" in
       */*) ref=$image ;;
       *) ref="docker.io/library/$image" ;;
@@ -2071,10 +2084,27 @@ if [ -n "$other_hubs" ]; then
     deployments="$deployments ws-server-$machine"
   done
 fi
+if [ "$recovery" = true ]; then
+  # The supervisor's report counts failed API requests, not unready workloads, so each workload is rolled out too.
+  deployments="$deployments recovery-supervisor $(yq '.recovery.workloads[].deployment' "$input")"
+fi
 for deployment in $deployments; do
   kubectl rollout status "deploy/$deployment" -n "$ns" --timeout="$manifest_wait"
 done
 
+if [ "$recovery" = true ]; then
+  # The supervisor image's own `cat` reads its state file; no mise-managed coreutils exists there.
+  # nosemgrep: config.semgrep.mise-task-coreutils-prefix
+  report=$(kubectl exec -n "$ns" deploy/recovery-supervisor -- cat /app/state/recovery.json)
+  unknown=$(printf '%s' "$report" | jaq '.unknown_observations')
+  if [ "$unknown" != 0 ]; then
+    echo "k3s-verify: $unknown supervisor requests to the Kubernetes API failed" >&2
+    kubectl logs -n "$ns" deploy/recovery-supervisor --tail=40 >&2 || true
+    exit 1
+  fi
+  exit 0
+fi
+
 # The runners restart until the hub answers, so the model is what says the exchange ran -- not pod readiness.
 #
 # Only the first bucket is read. Both agents register and each gets its own storage bucket, so the glob matches
@@ -2153,7 +2183,34 @@ done
 shell = "{{ vars.task_shell }}"
 usage = """
 arg "[source]" help="Which verification tree to apply: local (default) or published"
-flag "--scenario <scenario>" help="Which math1-shaped scenario to apply" default="math1"
+flag "--scenario <scenario>" help="Which math1-shaped or recovery scenario to apply" default="math1"
+"""
+
+# Build every image a local scenario's k3s manifest names from this repository, so `k3s-verify` can apply it.
+# The hub image comes first because the scenario image takes it as the `hub` named build context. Every other
+# image is `et-<service>`, built from `services/<service>/Dockerfile`. The scenario image stages its modules'
+# built `pkg/` directories, so a module that needs building must be built beforehand.
+[tasks.k3s-images]
+description = "Build the images a local scenario's k3s manifests name"
+# docker is a host daemon rather than a mise tool, and these images install nothing through mise.
+run = """
+scenario="${usage_scenario:-math1}"
+out="verification/local/output/$scenario"
+hub=et-ws-server-hub:latest
+scenario_image="et-ws-server-$scenario:latest"
+docker build -t "$hub" -f services/ws-server/Dockerfile .
+for image in $(yq -N '{{ vars.k3s_local_images }}' "$out/k3s.yaml" | coreutils sort -u); do
+  if [ "$image" = "$scenario_image" ]; then
+    docker build --build-context "hub=docker-image://$hub" -t "$image" -f "$out/Dockerfile" .
+  else
+    name=${image%:*}
+    docker build -t "$image" -f "services/${name#et-}/Dockerfile" .
+  fi
+done
+"""
+shell = "{{ vars.task_shell }}"
+usage = """
+flag "--scenario <scenario>" help="Which local scenario's images to build" default="math1"
 """
 
 # End-to-end check that a generated compose deployment runs the math1 exchange, one machine at a time.

AGENTS.md

@@ -83,6 +83,23 @@ outside the tree on purpose), then simply never point at it from inside. Creatin
 repo is fine when the operator wants that -- this rule is **only** about never referring to out-of-repo files
 from within the repo.
 
+## Misc Markdown files are not committed
+
+Never create a `docs/` directory. When asked for a miscellaneous `.md` file related to this repo -- an email draft,
+notes, a write-up -- write it at the top level of the repo as `<topic>.local.md`. `.gitignore` covers that suffix, so
+it is never committed and no linter reads it. It needs no formatting either, and the line-length limit does not
+apply. Write it for its reader instead: text meant to be pasted elsewhere, such as an email, keeps each paragraph
+on one line so it pastes without hard breaks. Once it is written, offer to open it in a Markdown editor; on macOS,
+`open` on the file launches the default app for `.md`.
+
+## Finding a Curtin staff member's details
+
+A Curtin staff member's email address is `firstname.lastname@curtin.edu.au`, so build it from their name rather than
+searching for it. Their honorific, position and school are public in Curtin Profiles, which takes a surname at
+`https://profiles.curtin.edu.au/search?by=text&type=user&v=<surname>`; it renders client-side, so read it in the
+browser pane rather than fetching it. Web search finds neither, so do not spend searches on a name. The older
+`https://staffportal.curtin.edu.au/staff/profile/view/` profile URLs redirect to Curtin Profiles.
+
 ## On macOS: use homebrew bash for ad-hoc agent commands
 
 On macOS, every ad-hoc command the agent runs that is **not** a `mise run <task>` invocation -- investigations,
@@ -137,6 +154,14 @@ back to the host binary -- surface it. Recommend to the user that the tool be ad
 (prebuilt-binary backend, cross-platform, per the "Tools must work on every OS" rule), and let them decide. A tool
 worth using in this repo is worth pinning.
 
+## Only free tools with a GitHub repository
+
+When evaluating a tool -- for the repo, or to install on the operator's machine -- consider only free, open-source
+tools, and confirm each has a GitHub repository before recommending or installing it. Never install a paid or
+trial-licensed tool, even where it is the best of its kind; a free trial is still a non-free tool. Read the licence
+in the repository rather than inferring it from a package listing: a Homebrew cask or an app-store page says nothing
+about whether the tool is free -- the Marked Markdown viewer was installed that way and turned out to be paid.
+
 ## Kill any command past an hour unless it is visibly progressing
 
 A command that has run for an hour is presumed hung and must be killed, unless there is positive evidence it is