Extend recovery supervision

.mise/config.js.toml

@@ -119,13 +119,13 @@ dir = "services/ws-modules/har1"
 # Chain with `&&` on one line rather than a multi-line `bash` block.
 # wasm-pack shells out to `cargo metadata`, which can't find `cargo` under Git Bash on Windows; the default
 # shell (cmd there) resolves it. web_pack_cov folds in the coverage wrapper prefix so this line stays under 120.
-run = "{{ vars.web_pack_cov }} {{ vars.no_opt }}{{ vars.web_cov_feat }} && {{ vars.et_cli }} module-package-json"
+run = "{{ vars.web_pack_cov }} {{ vars.no_opt }}{{ vars.web_cargo_args }} && {{ vars.et_cli }} module-package-json"
 
 [tasks.build-ws-face-detection-module]
 depends = ["build-et-cli", "build-wasm-cov-wrapper"]
 description = "Build the face detection workflow WASM module"
 dir = "services/ws-modules/face-detection"
-run = "{{ vars.web_pack_cov }} {{ vars.no_opt }}{{ vars.web_cov_feat }} && {{ vars.et_cli }} module-package-json"
+run = "{{ vars.web_pack_cov }} {{ vars.no_opt }}{{ vars.web_cargo_args }} && {{ vars.et_cli }} module-package-json"
 
 [tasks.vendor-llm1-ort]
 description = "Stage transformers.js's pinned ORT wasm runtime into llm1's pkg/ort/"
@@ -177,7 +177,7 @@ shell = "{{ vars.task_shell }}"
 depends = ["build-et-cli", "build-wasm-cov-wrapper", "vendor-llm1-ort"]
 description = "Build the llm1 chat WASM module"
 dir = "services/ws-modules/llm1"
-run = "{{ vars.web_pack_cov }} {{ vars.no_opt }}{{ vars.web_cov_feat }} && {{ vars.et_cli }} module-package-json"
+run = "{{ vars.web_pack_cov }} {{ vars.no_opt }}{{ vars.web_cargo_args }} && {{ vars.et_cli }} module-package-json"
 
 [tasks.build-ws-js-data1-module]
 description = "Bundle the js-data1 module (AWS SDK for JS v3) into pkg/ with esbuild"

.mise/config.rust.toml

@@ -87,7 +87,7 @@ shell = "{{ vars.task_shell }}"
 # Browser wasm-pack builds are coverage-instrumented when ET_TEST_COVERAGE is set.
 # The web_cov_wrapper RUSTC_WORKSPACE_WRAPPER (built by build-wasm-cov-wrapper) instruments only workspace
 # crates -- never dependency cdylibs such as wasm-streams that don't link minicov's
-# __llvm_profile_runtime -- while web_cov_feat enables et-web/coverage to supply minicov's runtime. et-web's
+# __llvm_profile_runtime -- while web_cargo_args enables et-web/coverage to supply minicov's runtime. et-web's
 # __et_capture_coverage export (folded by wasm-bindgen into every module's glue) lets the web-runner pull each
 # module's .profraw for wasm-cov.
 [tasks.build-wasm-cov-wrapper]
@@ -98,85 +98,85 @@ run = "cargo build -p int-wasm-cov-wrapper"
 depends = ["build-wasm-cov-wrapper"]
 description = "Build the data1 workflow WASM module"
 dir = "services/ws-modules/data1"
-run = "{{ vars.web_pack_cov }}{{ vars.no_opt }}{{ vars.web_cov_feat }}"
+run = "{{ vars.web_pack_cov }}{{ vars.no_opt }}{{ vars.web_cargo_args }}"
 
 [tasks.build-ws-math1-module]
 depends = ["build-wasm-cov-wrapper"]
 description = "Build the math1 FedAvg WASM module"
 dir = "services/ws-modules/math1"
-run = "{{ vars.web_pack_cov }}{{ vars.no_opt }}{{ vars.web_cov_feat }}"
+run = "{{ vars.web_pack_cov }}{{ vars.no_opt }}{{ vars.web_cargo_args }}"
 
 [tasks.build-ws-math1-sender-module]
 depends = ["build-wasm-cov-wrapper"]
 description = "Build the math1-sender trigger WASM module"
 dir = "services/ws-modules/math1-sender"
-run = "{{ vars.web_pack_cov }}{{ vars.no_opt }}{{ vars.web_cov_feat }}"
+run = "{{ vars.web_pack_cov }}{{ vars.no_opt }}{{ vars.web_cargo_args }}"
 
 [tasks.build-ws-comm1-module]
 depends = ["build-wasm-cov-wrapper"]
 description = "Build the comm1 workflow WASM module"
 dir = "services/ws-modules/comm1"
-run = "{{ vars.web_pack_cov }}{{ vars.no_opt }}{{ vars.web_cov_feat }}"
+run = "{{ vars.web_pack_cov }}{{ vars.no_opt }}{{ vars.web_cargo_args }}"
 
 [tasks.build-ws-pic-viewer-module]
 depends = ["build-wasm-cov-wrapper"]
 description = "Build the pic-viewer broadcast picture viewer WASM module"
 dir = "services/ws-modules/pic-viewer"
-run = "{{ vars.web_pack_cov }}{{ vars.no_opt }}{{ vars.web_cov_feat }}"
+run = "{{ vars.web_pack_cov }}{{ vars.no_opt }}{{ vars.web_cargo_args }}"
 
 [tasks.build-ws-except1-module]
 depends = ["build-wasm-cov-wrapper"]
 description = "Build the except1 exception-handling demo WASM module"
 dir = "services/ws-modules/except1"
-run = "{{ vars.web_pack_cov }}{{ vars.no_opt }}{{ vars.web_cov_feat }}"
+run = "{{ vars.web_pack_cov }}{{ vars.no_opt }}{{ vars.web_cargo_args }}"
 
 [tasks.build-ws-sensor1-module]
 depends = ["build-wasm-cov-wrapper"]
 description = "Build the sensor1 workflow WASM module"
 dir = "services/ws-modules/sensor1"
-run = "{{ vars.web_pack_cov }}{{ vars.no_opt }}{{ vars.web_cov_feat }}"
+run = "{{ vars.web_pack_cov }}{{ vars.no_opt }}{{ vars.web_cargo_args }}"
 
 [tasks.build-ws-audio1-module]
 depends = ["build-wasm-cov-wrapper"]
 description = "Build the audio1 workflow WASM module"
 dir = "services/ws-modules/audio1"
-run = "{{ vars.web_pack_cov }}{{ vars.no_opt }}{{ vars.web_cov_feat }}"
+run = "{{ vars.web_pack_cov }}{{ vars.no_opt }}{{ vars.web_cargo_args }}"
 
 [tasks.build-ws-video1-module]
 depends = ["build-wasm-cov-wrapper"]
 description = "Build the video1 workflow WASM module"
 dir = "services/ws-modules/video1"
-run = "{{ vars.web_pack_cov }}{{ vars.no_opt }}{{ vars.web_cov_feat }}"
+run = "{{ vars.web_pack_cov }}{{ vars.no_opt }}{{ vars.web_cargo_args }}"
 
 [tasks.build-ws-bluetooth-module]
 depends = ["build-wasm-cov-wrapper"]
 description = "Build the bluetooth workflow WASM module"
 dir = "services/ws-modules/bluetooth"
-run = "{{ vars.web_pack_cov }}{{ vars.no_opt }}{{ vars.web_cov_feat }}"
+run = "{{ vars.web_pack_cov }}{{ vars.no_opt }}{{ vars.web_cargo_args }}"
 
 [tasks.build-ws-geolocation-module]
 depends = ["build-wasm-cov-wrapper"]
 description = "Build the geolocation workflow WASM module"
 dir = "services/ws-modules/geolocation"
-run = "{{ vars.web_pack_cov }}{{ vars.no_opt }}{{ vars.web_cov_feat }}"
+run = "{{ vars.web_pack_cov }}{{ vars.no_opt }}{{ vars.web_cargo_args }}"
 
 [tasks.build-ws-graphics-info-module]
 depends = ["build-wasm-cov-wrapper"]
 description = "Build the graphics info workflow WASM module"
 dir = "services/ws-modules/graphics-info"
-run = "{{ vars.web_pack_cov }}{{ vars.no_opt }}{{ vars.web_cov_feat }}"
+run = "{{ vars.web_pack_cov }}{{ vars.no_opt }}{{ vars.web_cargo_args }}"
 
 [tasks.build-ws-speech-recognition-module]
 depends = ["build-wasm-cov-wrapper"]
 description = "Build the speech recognition workflow WASM module"
 dir = "services/ws-modules/speech-recognition"
-run = "{{ vars.web_pack_cov }}{{ vars.no_opt }}{{ vars.web_cov_feat }}"
+run = "{{ vars.web_pack_cov }}{{ vars.no_opt }}{{ vars.web_cargo_args }}"
 
 [tasks.build-ws-nfc-module]
 depends = ["build-wasm-cov-wrapper"]
 description = "Build the nfc workflow WASM module"
 dir = "services/ws-modules/nfc"
-run = "{{ vars.web_pack_cov }}{{ vars.no_opt }}{{ vars.web_cov_feat }}"
+run = "{{ vars.web_pack_cov }}{{ vars.no_opt }}{{ vars.web_cargo_args }}"
 
 [tasks.build-ws-wasi-data1-module]
 depends = ["build-et-cli"]

.mise/config.toml

@@ -648,13 +648,18 @@ a_wasm_cov_instr = "-Cinstrument-coverage -Zcoverage-options=branch"
 # RUSTFLAGS for the wasm coverage builds: instrument + emit LLVM IR, no profiler runtime (minicov provides it).
 # Single codegen unit + LTO off so each crate emits one .ll the wasm-cov task can gut and feed to llc/llvm-cov.
 wasm_cov_rustflags = "--emit=llvm-ir {{ vars.a_wasm_cov_instr }} -Ccodegen-units=1 -Clto=off -Zno-profiler-runtime"
-# Coverage fragments the wasm module build commands append unconditionally (empty unless ET_TEST_COVERAGE is set).
+# WASI coverage arguments are empty unless ET_TEST_COVERAGE is set.
 # wasm_cov is an inline RUSTFLAGS env prefix; the nightly toolchain + conda clang come from the coverage [env].
-# The *_feat vars turn on minicov: cargo `--features` for the WASI cdylibs, wasm-pack `-- --features` for browsers
-# (comm1 pulls et-web only under this feature).
+# Browser builds explicitly select Rust's wasm target, independently of Windows' native CARGO_BUILD_TARGET.
+# wasm-pack 0.15.1 otherwise builds native output and reports:
+# "Expected exactly one .wasm file in the compiler artifact, but found []".
+# Observed on https://github.com/edge-toolkit/core/actions/runs/41204094871/job/124545540158.
+# Coverage adds minicov through et-web to the same Cargo argument list.
 wasi_cov_feat = '{% if env?.ET_TEST_COVERAGE == "true" %} --features coverage{% endif %}'
 wasm_cov = '{% if env?.ET_TEST_COVERAGE == "true" %}RUSTFLAGS="{{ vars.wasm_cov_rustflags }}"{% endif %}'
-web_cov_feat = '{% if env?.ET_TEST_COVERAGE == "true" %} -- --features et-web/coverage{% endif %}'
+web_cargo_args = """ -- --target wasm32-unknown-unknown
+{%- if env?.ET_TEST_COVERAGE == "true" %} --features et-web/coverage{% endif -%}
+"""
 # Command prefix routing a browser module's wasm-pack build through the workspace-only coverage wrapper.
 # Instruments our crates, never dependency cdylibs. Empty unless ET_TEST_COVERAGE is set, so normal builds get
 # no wrapper (no fingerprint churn). The wrapper binary is built by the build-wasm-cov-wrapper task dependency.
@@ -675,7 +680,7 @@ wasm_scope = "--scope edge-toolkit "
 hub_root_module = "@edge-toolkit/et-ws-server-static"
 # Override-free head of a browser module's wasm-pack build: coverage wrapper prefix plus fixed build args.
 # Used by the two modules that also run et-cli module-package-json (har1, face-detection). It excludes no_opt /
-# web_cov_feat -- those are appended inline at the call site because no_opt carries a per-platform override that a
+# web_cargo_args -- those are appended inline at the call site because no_opt carries a per-platform override that a
 # nested var would freeze at this file's value.
 web_pack_cov = '{{ vars.web_cov_wrapper }}wasm-pack build . --target web {{ vars.wasm_scope }}'
 # Extra flag for the wasm-pack module builds; empty so wasm-opt runs.
@@ -2681,7 +2686,7 @@ dir = "services/ws-wasm-agent"
 # so other globally-loaded tasks -- prefetch-ci's transitive deps via `prefetch:node` -- can reference it
 # without forcing `rust` into MISE_ENV. A bash `shell` for the same base-config PATH-dropout reason
 # `prefetch:rust` carries one.
-run = "wasm-pack build . --target web {{ vars.wasm_scope }}{{ vars.no_opt }}"
+run = "wasm-pack build . --target web {{ vars.wasm_scope }}{{ vars.no_opt }} -- --target wasm32-unknown-unknown"
 shell = "{{ vars.task_shell }}"
 
 [tasks."prefetch:node"]

CHANGELOG.md

@@ -45,6 +45,12 @@ Changelog 1.1.0.
 - `et-ws-wasi-runner` builds as `no_std` + `alloc` without its new default `std` feature, and its `board` module
   runs a guest on a microcontroller: compiled ahead of time with `engine::precompile` for the board's Pulley target,
   and hosted through the firmware's `board::Platform` and `board::Transport`.
+- Native REST requests retry transient HTTP 408/429/500/502/503/504 responses as well as transport failures, using a
+  bounded backoff policy for replayable idempotent operations. Permission errors, non-idempotent methods and streaming
+  request bodies are not retried. Storage-call timings include the retry/backoff interval; retry attempts are logged.
+- Recovery supervision reconnects after hub interruptions and restores incident context with explicit observation gaps.
+  Optional dependency checks, fleet action limits, bounded backoff with jitter and sustained-recovery assessment extend
+  the existing restart policy. Reports distinguish observed failure conditions from reasons for withholding actions.
 - An opt-in `et-recovery-supervisor` service observes application progress and coordinates bounded Kubernetes recovery.
 - The Python SDK provides recovery lifecycle messages and integrity-checked checkpoint storage helpers.
 - A scenario can list `machines:` to span several hosts. The first is the server, running the hub and the collector,
@@ -112,6 +118,9 @@ Changelog 1.1.0.
   name a shell would need quoted or that some filesystem could not hold, such as `my mod`, `mod$(id)` or `con`; rename
   the directory. Generated deployments no longer escape these paths. The check is the new `et-sanitise` crate, which
   accepts a name only when both sanitize-filename, with its Windows rules, and shlex leave it unchanged.
+- **Breaking:** Recovery state is bound to its complete configuration. Changing configuration requires a new state
+  location or an explicitly provisioned new job. Legacy reports retain their restart budget and are archived in the
+  new context.
 - The workspace requires actix-web 4.17.
 - **Breaking:** the minimum supported Rust version is 1.99.0, up from 1.91.1, for every published crate.
 - **Breaking:** an agent's IP address is always known. `AgentSummary::last_known_ip` and

generated/rust-rest/Cargo.toml

@@ -1,12 +1,17 @@
 [package]
 name = "et-rest-client"
 publish = true
-description = "Rust client for ws-server's REST API. src/lib.rs regenerated; this Cargo.toml is hand-maintained."
+description = "Rust REST client for ws-server. src/lib.rs is regenerated; retry.rs and Cargo.toml are hand-maintained."
 version = "0.2.0"
 edition.workspace = true
 license.workspace = true
 repository.workspace = true
 
+# Private retry tests need log capture and temporary files.
+# The compiler loads tests/unit/retry.rs through src/retry.rs's #[path] module, but cargo-shear misses its uses.
+[package.metadata.cargo-shear]
+ignored = ["tempfile", "tracing-subscriber"]
+
 [lib]
 # src/lib.rs is generated by progenitor; its /** */ doc comments contain
 # markdown backtick prose that Rust's doctest runner tries to compile as code.
@@ -30,8 +35,7 @@ tracing-opentelemetry = { workspace = true, optional = true }
 # `rustls` and `stream` are native-only -- WASM reqwest dispatches via the
 # browser's `fetch()` and has no notion of a TLS stack.
 #
-# `retry-policies` + `tokio` back the generator-injected `ClientHooks::exec`
-# backoff retry (see `utilities/int-gen/src/openapi.rs::inject_retry_exec`).
+# `retry-policies` + `tokio` back the hand-maintained native request retry policy invoked by the generated hook.
 # Native-only: that hook is `#[cfg(not(wasm32))]`, and `tokio::time::sleep` /
 # `SystemTime` don't work under `wasm32-unknown-unknown`.
 [target.'cfg(not(target_arch = "wasm32"))'.dependencies]
@@ -47,6 +51,11 @@ reqwest = { workspace = true, features = ["json"] }
 # Deno runtime's stubbed `globalThis.location`).
 web-sys = { workspace = true, features = ["Location", "Window"] }
 
+[target.'cfg(not(target_arch = "wasm32"))'.dev-dependencies]
+tempfile.workspace = true
+tokio = { workspace = true, features = ["fs", "io-util", "macros", "net", "rt", "test-util"] }
+tracing-subscriber.workspace = true
+
 # Every rustc warning here is an error, as it is in every hand-written crate.
 # This was the one workspace member with no [lints] table at all, so its warnings stayed warnings -- which is how
 # six unreachable calls sat in `cargo-doc-check` output instead of failing it. The full `[workspace.lints]` set is

generated/rust-rest/src/lib.rs

@@ -116,40 +116,12 @@ impl ClientInfo<()> for Client {
         &()
     }
 }
+#[cfg(not(target_arch = "wasm32"))]
+mod retry;
 impl ClientHooks<()> for &Client {
-    // Injected by `utilities/int-gen` (inject_retry_exec): retry request execution with exponential backoff. reqwest's
-    // native retry has no backoff yet -- remove this and use `ClientBuilder::retries` once it does.
     #[cfg(not(target_arch = "wasm32"))]
     async fn exec(&self, request: ::reqwest::Request, _info: &OperationInfo) -> ::reqwest::Result<::reqwest::Response> {
-        use ::retry_policies::policies::ExponentialBackoff;
-        use ::retry_policies::{RetryDecision, RetryPolicy as _};
-        let policy = ExponentialBackoff::builder()
-            .retry_bounds(
-                ::core::time::Duration::from_millis(250),
-                ::core::time::Duration::from_secs(5),
-            )
-            .build_with_total_retry_duration(::core::time::Duration::from_secs(30));
-        let started = ::std::time::SystemTime::now();
-        let mut n_past_retries: u32 = 0;
-        loop {
-            // Retry only when the request can be replayed (no streaming body).
-            let Some(attempt) = request.try_clone() else {
-                return self.client().execute(request).await;
-            };
-            match self.client().execute(attempt).await {
-                Ok(response) => return Ok(response),
-                Err(err) => match policy.should_retry(started, n_past_retries) {
-                    RetryDecision::Retry { execute_after } => {
-                        let wait = execute_after
-                            .duration_since(::std::time::SystemTime::now())
-                            .unwrap_or_default();
-                        ::tokio::time::sleep(wait).await;
-                        n_past_retries = n_past_retries.saturating_add(1);
-                    }
-                    RetryDecision::DoNotRetry => return Err(err),
-                },
-            }
-        }
+        crate::retry::execute(self.client(), request).await
     }
 }
 #[allow(clippy::all)]