Multi-hub generation

.mise/config.toml

@@ -1799,6 +1799,9 @@ machine_label=edge-toolkit/machine
 # One name per line; each is an RFC 1123 label, so the unquoted word-splitting loops over it below are safe.
 machines=$(yq '(.machines // [])[].name' "$input")
 server_machine=$(yq '.machines[0].name // ""' "$input")
+# The machines running a hub besides the server, each of which runs `ws-server-<machine>`.
+# Any at all means the hubs share NATS and an object store, which run on the server.
+other_hubs=$(server="$server_machine" yq '(.services.hub.machines // [])[] | select(. != strenv(server))' "$input")
 
 # The runner is the one image that differs between the two sources.
 # A published deployment names the released one and so has it pulled here rather than built; the scenario
@@ -1906,8 +1909,16 @@ k3d image import -c "$cluster" "$hub_image" "$runner_image"
 kubectl create namespace "$ns" --save-config
 kubectl create secret generic "$ns-secrets" --from-env-file="$secrets_dir/secrets.env" -n "$ns"
 kubectl apply -f "$out/k3s.yaml"
-kubectl rollout status deploy/openobserve -n "$ns" --timeout="$manifest_wait"
-kubectl rollout status deploy/ws-server -n "$ns" --timeout="$manifest_wait"
+deployments="openobserve ws-server"
+if [ -n "$other_hubs" ]; then
+  deployments="$deployments nats rustfs"
+  for machine in $other_hubs; do
+    deployments="$deployments ws-server-$machine"
+  done
+fi
+for deployment in $deployments; do
+  kubectl rollout status "deploy/$deployment" -n "$ns" --timeout="$manifest_wait"
+done
 
 # The runners restart until the hub answers, so the model is what says the exchange ran -- not pod readiness.
 #
@@ -1921,6 +1932,22 @@ stored=""
 # shellcheck disable=SC2016
 # nosemgrep: config.semgrep.mise-task-coreutils-prefix
 read_model='set -- /app/storage/*/math1-output.json; cat "$1"'
+if [ -n "$other_hubs" ]; then
+  # Hubs that share storage keep it in the object store, so the server's hub reads the model back through S3.
+  # It uses the credentials it was given, listing the bucket for the first key that holds one.
+  # The twin stored it through the other hub, so reading it here is what shows the store is shared.
+  # rustfs refuses a request without `x-amz-content-sha256`, which the image's curl does not send on its own:
+  #   <Error><Code>InvalidRequest</Code><Message>missing header: x-amz-content-sha256</Message></Error>
+  # shellcheck disable=SC2016
+  # nosemgrep: config.semgrep.mise-task-coreutils-prefix
+  read_model='s3() {
+    login="$AWS_ACCESS_KEY_ID:$AWS_SECRET_ACCESS_KEY"
+    unsigned="x-amz-content-sha256: UNSIGNED-PAYLOAD"
+    curl -fsS --aws-sigv4 "aws:amz:$AWS_DEFAULT_REGION:s3" --user "$login" -H "$unsigned" "$AWS_ENDPOINT/et-storage/$1"
+  }
+  key=$(s3 "?list-type=2" | grep -o "[^>]*/math1-output[.]json" | head -n 1)
+  [ -n "$key" ] && s3 "$key"'
+fi
 for _ in $(coreutils seq 1 60); do
   stored=$(kubectl exec -n "$ns" deploy/ws-server -- sh -c "$read_model" 2>/dev/null || true)
   if [ -n "$stored" ]; then
@@ -1930,7 +1957,7 @@ for _ in $(coreutils seq 1 60); do
 done
 
 if [ -z "$stored" ]; then
-  echo "k3s-verify: no math1-output.json appeared under /app/storage" >&2
+  echo "k3s-verify: no math1-output.json appeared in the hub's storage" >&2
   exit 1
 fi
 
@@ -1941,14 +1968,23 @@ if [ -f "$template" ]; then
 fi
 
 # Every pod has to be on the node labelled as its machine.
-# The collector and the hub belong on the server, and each listed agent's runner on the machine listing it.
+# The collector, the server's hub and what several hubs share belong on the server, any other machine's hub on
+# that machine, and each listed agent's runner on the machine listing it.
 # Checked after the exchange, when every pod is scheduled.
 listed='.machines[] | select(.name == strenv(machine)) | .agents[]'
 for machine in $machines; do
   apps=$(machine="$machine" yq "$listed" "$input")
   if [ "$machine" = "$server_machine" ]; then
     apps="openobserve ws-server $apps"
+    if [ -n "$other_hubs" ]; then
+      apps="nats rustfs $apps"
+    fi
   fi
+  for hub_machine in $other_hubs; do
+    if [ "$machine" = "$hub_machine" ]; then
+      apps="ws-server-$machine $apps"
+    fi
+  done
   expected=$(kubectl get nodes -l "$machine_label=$machine" -o jsonpath='{.items[0].metadata.name}')
   for app in $apps; do
     placed=$(kubectl get pods -n "$ns" -l "app=$app" -o jsonpath='{.items[0].spec.nodeName}')
@@ -2242,7 +2278,7 @@ shell = "{{ vars.task_shell }}"
 
 [tasks.ryl-check]
 description = "Lint YAML with ryl (a yamllint-compatible Rust linter)"
-run = "ryl -c config/ryl.yaml .github config"
+run = "ryl check -c config/ryl.toml .github config"
 
 [tasks._setup_all]
 description = "Private shared cross-platform preinstall"

CHANGELOG.md

@@ -37,6 +37,12 @@ Changelog 1.1.0.
   uploads to storage through any hub, and can reconnect to a different hub without losing what was queued for it.
   The default, `local`, behaves as before. In NATS mode the `WS_AGENT_REGISTRY` file is neither read nor written. The
   new `et-hub-backplane` crate holds the NATS side, and `et-test-nats` starts a throwaway broker for tests.
+- A scenario can run a hub on more than one machine: `services.hub.machines` lists them, the server among them. Each
+  runner joins the hub on its own machine, or the server's when its machine runs none. Every format then also runs a
+  NATS server and a rustfs object store on the server, which the hubs share as their backplane and their agent
+  storage, so an upload through one hub reads back through any other. `services.backplane` and `services.storage`
+  take `limits:` for the two. `secrets.env` carries the store's secret as `AWS_SECRET_ACCESS_KEY` and
+  `RUSTFS_SECRET_KEY`. A scenario without `services.hub.machines` renders as before.
 
 ### Changed
 

CHECKS.md

@@ -296,7 +296,8 @@ Rules, from `config/ast-grep/rules`:
   so an `args` entry also silently discards the arguments the image was built to start with.
 - `write-ending-newline` -- A `write!` whose format string ends in `\n` is a `writeln!` written the long way: use
   `writeln!` and drop the final `\n`. This mirrors `DeepSource RS-C1005`. clippy's `write_with_newline` covers only a
-  single trailing newline and deliberately skips a string ending in `\n\n`, which is the case this rule exists to catch.
+  single trailing newline in a plain literal: it skips a string ending in `\n\n`, and one assembled with `concat!`,
+  which are the cases this rule exists to catch.
 
 ## `cargo check`
 
@@ -772,7 +773,7 @@ Rules, from `config/conftest/policy`:
 ## `ryl`
 
 - `ryl-check` (`default` env; run by `check:rust`) -- Lint YAML with ryl (a yamllint-compatible Rust linter). Config:
-  `config/ryl.yaml`.
+  `config/ryl.toml`.
 
 ## `semgrep`
 

README.md

@@ -397,15 +397,6 @@ docker compose up --build
 The generated scenario config only selects which prebuilt modules `ws-server` serves. Module builds are expected
 to be handled separately from the repository root.
 
-To regenerate all checked-in verification outputs from `verification/*/input`, writing each scenario to the
-matching `verification/*/output/<input-file-stem>` folder. This renders each scenario for every supported deployment
-tool -- mise, dekit, process-compose, Docker Compose and k3s -- and each scenario's own `README.md` names the files
-it holds and how to run them:
-
-```bash
-mise run regen-verification
-```
-
 ### k3s
 
 The third output type writes Kubernetes manifests. Pass `--output-type k3s`, or set `deployment_type: k3s` in the
@@ -425,12 +416,9 @@ every image has to exist on the node before `kubectl apply`. And the scenario's
 the scenario also writes a `README.md` beside the manifests carrying that scenario's own build, import and
 secret-loading commands.
 
-Applying them to a live cluster is Linux-only, because k3s is a Linux Kubernetes distribution. `mise run k3s-verify`
-does the whole round trip for the `math1` scenario against a disposable k3d cluster -- create, import, apply, wait
-for both rollouts, then read back the model the two runners exchanged -- and deletes the cluster on the way out. It
-expects the images to have been built already and says so if they have not. Generating and checking the manifests
-is not Linux-only: `verification-check` fails on drift, `kubeconform-check` validates them against the Kubernetes
-schemas, and `trivy-check` applies its Kubernetes security rules, all on every platform.
+Applying them to a live cluster is Linux-only, because k3s is a Linux Kubernetes distribution. Generating and
+checking the manifests is not: `kubeconform-check` validates them against the Kubernetes schemas, and `trivy-check`
+applies its Kubernetes security rules, both on every platform.
 
 The containers run hardened: a read-only root filesystem, no privilege escalation, every capability dropped, and a
 non-root user, with `emptyDir` mounts for the paths that still have to be writable. Traffic between them is
@@ -453,11 +441,31 @@ machines:
     agents: ["math1-twin"]
 ```
 
-`verification/local/input/math1-multi.yaml` is that split with both machines at loopback, so one host can play every
-machine; its `verification/published/` twin renders the same split against the released artifacts, which need
-nothing new to run it. `mise run k3s-verify --scenario math1-multi` applies it to a k3d cluster with one node per
-machine and checks each pod landed on its own machine's node. `mise run compose-verify` brings it up one machine
-profile at a time.
+### Several hubs
+
+`services.hub.machines:` runs a hub on each machine it lists, the server among them. The hubs join into one through
+a NATS server, the backplane carrying the agent directory, the mailbox and delivery between hubs, and point their
+agent storage at one rustfs object store, so an upload through one hub reads back through any other. Both run on the
+server beside the collector, and `services.backplane.limits:` and `services.storage.limits:` set their limits. Each
+runner joins the hub on its own machine, or the server's when its machine runs none:
+
+```yaml
+machines:
+  - name: "server"
+    address: "10.0.0.1"
+    agents: ["math1-trigger"]
+  - name: "edge"
+    address: "10.0.0.2"
+    agents: ["math1-twin"]
+services:
+  hub:
+    machines: ["server", "edge"]
+```
+
+NATS takes no credential, the same trust the hubs' plaintext WebSocket listeners already place in the network between
+the machines; the object store's secret is in `secrets.env`. In every format but k3s a hub off the server exports no
+telemetry, because the collector listens on the server's loopback only. Every hub listens on the same ports, so only
+k3s, where each pod has a network of its own, runs several on one host.
 
 ### Resource limits
 
@@ -487,9 +495,7 @@ Compose and k3s apply every field, and each lands in the same cgroup value under
 takes a default both share. k3s carries what the Pod API cannot through two upstream container runtime plugins,
 containerd's `ulimit-adjuster` and nri-plugins' `nri-memory-qos`, and refuses a limited pod on a node without them, so
 the generated `README.md` has every machine prepared first. The mise, dekit and process-compose deployments run each
-process directly on the host and apply none. `verification/local/input/math1-limits.yaml` sets a sample, which
-`mise run k3s-verify local --scenario math1-limits` and `mise run compose-verify --scenario math1-limits` check in the
-running containers.
+process directly on the host and apply none.
 
 ### Scheduled agents
 
@@ -512,8 +518,7 @@ agents:
 k3s runs it as a `CronJob`. Compose adds a `scheduler` service, [Ofelia](https://github.com/netresearch/ofelia), which
 starts the runner's container again on each tick and mounts the Docker socket to do so. process-compose, whose
 `process-compose.yaml` runs the `mise.toml` tasks as `dekit.yaml` does, schedules the task itself. mise and dekit have
-no scheduler, so generating either for a scheduled scenario is an error, and `regen-verification` leaves both out of
-such a scenario. `verification/local/input/math1-cron.yaml` runs the trigger every minute.
+no scheduler, so generating either for a scheduled scenario is an error.
 
 ## Grant
 

config/ast-grep/rules/write-ending-newline.yaml

@@ -4,7 +4,8 @@ severity: error
 message: |
   A `write!` whose format string ends in `\n` is a `writeln!` written the long way: use `writeln!` and drop the
   final `\n`. This mirrors `DeepSource RS-C1005`. clippy's `write_with_newline` covers only a single trailing
-  newline and deliberately skips a string ending in `\n\n`, which is the case this rule exists to catch.
+  newline in a plain literal: it skips a string ending in `\n\n`, and one assembled with `concat!`, which are the
+  cases this rule exists to catch.
 # There is no autofix.
 # The rewrite renames the macro and edits the string literal inside its token tree at once, which a single `fix:`
 # template over the whole invocation cannot express without re-spelling every argument.
@@ -13,7 +14,12 @@ rule:
   has:
     field: macro
     regex: ^write$
-  regex: '(?s)^write!\s*\((?:[^"()]|\([^()]*\))*,\s*"(?:[^"\\]|\\.)*\\n"'
+  # The format string is either a literal ending in `\n`, or a `concat!` whose last literal does.
+  # Verbose mode, so the pattern can span lines: its whitespace is all `\s`, which verbose mode leaves alone.
+  regex: |-
+    (?sx)^write!\s*\((?:[^"()]|\([^()]*\))*,\s*
+    (?:"(?:[^"\\]|\\.)*\\n"
+    |concat!\s*\((?:\s*"(?:[^"\\]|\\.)*"\s*,)*\s*"(?:[^"\\]|\\.)*\\n"\s*,?\s*\))
 ignores:
   - generated/**
   # Generated; refer to int-gen bindings.rs for details.

config/ast-grep/tests/write-ending-newline.yaml

@@ -0,0 +1,50 @@
+id: write-ending-newline
+valid:
+  - |
+    fn run(out: &mut String) {
+        let _result = writeln!(out, "done");
+    }
+  - |
+    fn run(out: &mut String) {
+        let _result = write!(out, "no newline");
+    }
+  - |
+    fn run(out: &mut String) {
+        let _result = write!(out, "a newline\nin the middle");
+    }
+  - |
+    fn run(out: &mut String) {
+        let _result = writeln!(out, concat!("first ", "second"));
+    }
+  - |
+    fn run(out: &mut String) {
+        let _result = write!(out, concat!("first line\n", "then no newline"));
+    }
+invalid:
+  - |
+    fn run(out: &mut String) {
+        let _result = write!(out, "done\n");
+    }
+  - |
+    fn run(out: &mut String) {
+        let _result = write!(out, "a blank line after\n\n");
+    }
+  - |
+    fn run(out: &mut String, name: &str) {
+        let _result = write!(out, "{name}\n", name = name);
+    }
+  - |
+    fn run(out: &mut String) {
+        let _result = write!(out, concat!("first ", "second\n"));
+    }
+  - |
+    fn run(out: &mut String, name: &str) {
+        let _result = write!(
+            out,
+            concat!(
+                "first\n",
+                "{name}\n",
+            ),
+            name = name,
+        );
+    }

config/conftest/policy/jscpd/jscpd.rego

@@ -24,7 +24,7 @@ package jscpd
 
 # Ceiling on the clone fingerprints recorded in config/jscpd-baseline.json.
 # May only ever be lowered. See the header for what raising it implies.
-declared_clones := 341
+declared_clones := 347
 
 # How far under the ceiling the baseline may sit before the ceiling has to be banked downward.
 max_slack := 20

config/deny.toml

@@ -113,6 +113,15 @@ ignore = [
   # decryption. Not reachable unless guest JS explicitly imports crypto.subtle and performs RSA-OAEP decrypt
   # with a private key. Our modules don't.
   { id = "RUSTSEC-2023-0071", reason = "expires 2026-10-31" },
+  # crypto-bigint 0.5.5 secret-dependent branches in operations meant to be constant-time (timing side channel).
+  # The arithmetic under elliptic-curve 0.13, which deno_crypto and deno_node_crypto use for P-256/384/521 and
+  # secp256k1 in WebCrypto and node:crypto. The fix is only in 0.7.6 and later, and deno_crypto requires
+  # elliptic-curve ^0.13 through its newest release, 0.274.0, so no bump reaches it. Live code, but reached only when
+  # guest JS in the web runner signs or derives with an EC private key, and leaking it takes an attacker able to time
+  # those operations on the same host. Our modules do neither; the hub's TLS uses rustls and aws-lc, not this. Drop
+  # when deno_crypto moves to elliptic-curve 0.14. cargo-deny ignores by id only, so this also mutes the 0.7 copy
+  # elliptic-curve 0.14 pulls -- Cargo.lock pins the patched 0.7.8 there, so re-check that copy when touching this.
+  { id = "RUSTSEC-2026-0387", reason = "expires 2026-10-31" },
   # h2 0.3.27 unbounded empty DATA frames (low-severity HTTP/2 DoS: undrained streams grow memory).
   # Accepted for the 0.3 copy only. hyperium patched the 0.4 line (>= 0.4.16, which Cargo.lock now pins for the
   # hyper/reqwest path) and shipped no 0.3 backport, so 0.3.27 is the last release of a dead line. It arrives