diff --git a/.mise/config.toml b/.mise/config.toml index 7d629dde..e4884a39 100644 --- a/.mise/config.toml +++ b/.mise/config.toml @@ -1799,6 +1799,9 @@ machine_label=edge-toolkit/machine # One name per line; each is an RFC 1123 label, so the unquoted word-splitting loops over it below are safe. machines=$(yq '(.machines // [])[].name' "$input") server_machine=$(yq '.machines[0].name // ""' "$input") +# The machines running a hub besides the server, each of which runs `ws-server-`. +# Any at all means the hubs share NATS and an object store, which run on the server. +other_hubs=$(server="$server_machine" yq '(.services.hub.machines // [])[] | select(. != strenv(server))' "$input") # The runner is the one image that differs between the two sources. # A published deployment names the released one and so has it pulled here rather than built; the scenario @@ -1906,8 +1909,16 @@ k3d image import -c "$cluster" "$hub_image" "$runner_image" kubectl create namespace "$ns" --save-config kubectl create secret generic "$ns-secrets" --from-env-file="$secrets_dir/secrets.env" -n "$ns" kubectl apply -f "$out/k3s.yaml" -kubectl rollout status deploy/openobserve -n "$ns" --timeout="$manifest_wait" -kubectl rollout status deploy/ws-server -n "$ns" --timeout="$manifest_wait" +deployments="openobserve ws-server" +if [ -n "$other_hubs" ]; then + deployments="$deployments nats rustfs" + for machine in $other_hubs; do + deployments="$deployments ws-server-$machine" + done +fi +for deployment in $deployments; do + kubectl rollout status "deploy/$deployment" -n "$ns" --timeout="$manifest_wait" +done # The runners restart until the hub answers, so the model is what says the exchange ran -- not pod readiness. # @@ -1921,6 +1932,22 @@ stored="" # shellcheck disable=SC2016 # nosemgrep: config.semgrep.mise-task-coreutils-prefix read_model='set -- /app/storage/*/math1-output.json; cat "$1"' +if [ -n "$other_hubs" ]; then + # Hubs that share storage keep it in the object store, so the server's hub reads the model back through S3. + # It uses the credentials it was given, listing the bucket for the first key that holds one. + # The twin stored it through the other hub, so reading it here is what shows the store is shared. + # rustfs refuses a request without `x-amz-content-sha256`, which the image's curl does not send on its own: + # InvalidRequestmissing header: x-amz-content-sha256 + # shellcheck disable=SC2016 + # nosemgrep: config.semgrep.mise-task-coreutils-prefix + read_model='s3() { + login="$AWS_ACCESS_KEY_ID:$AWS_SECRET_ACCESS_KEY" + unsigned="x-amz-content-sha256: UNSIGNED-PAYLOAD" + curl -fsS --aws-sigv4 "aws:amz:$AWS_DEFAULT_REGION:s3" --user "$login" -H "$unsigned" "$AWS_ENDPOINT/et-storage/$1" + } + key=$(s3 "?list-type=2" | grep -o "[^>]*/math1-output[.]json" | head -n 1) + [ -n "$key" ] && s3 "$key"' +fi for _ in $(coreutils seq 1 60); do stored=$(kubectl exec -n "$ns" deploy/ws-server -- sh -c "$read_model" 2>/dev/null || true) if [ -n "$stored" ]; then @@ -1930,7 +1957,7 @@ for _ in $(coreutils seq 1 60); do done if [ -z "$stored" ]; then - echo "k3s-verify: no math1-output.json appeared under /app/storage" >&2 + echo "k3s-verify: no math1-output.json appeared in the hub's storage" >&2 exit 1 fi @@ -1941,14 +1968,23 @@ if [ -f "$template" ]; then fi # Every pod has to be on the node labelled as its machine. -# The collector and the hub belong on the server, and each listed agent's runner on the machine listing it. +# The collector, the server's hub and what several hubs share belong on the server, any other machine's hub on +# that machine, and each listed agent's runner on the machine listing it. # Checked after the exchange, when every pod is scheduled. listed='.machines[] | select(.name == strenv(machine)) | .agents[]' for machine in $machines; do apps=$(machine="$machine" yq "$listed" "$input") if [ "$machine" = "$server_machine" ]; then apps="openobserve ws-server $apps" + if [ -n "$other_hubs" ]; then + apps="nats rustfs $apps" + fi fi + for hub_machine in $other_hubs; do + if [ "$machine" = "$hub_machine" ]; then + apps="ws-server-$machine $apps" + fi + done expected=$(kubectl get nodes -l "$machine_label=$machine" -o jsonpath='{.items[0].metadata.name}') for app in $apps; do placed=$(kubectl get pods -n "$ns" -l "app=$app" -o jsonpath='{.items[0].spec.nodeName}') @@ -2242,7 +2278,7 @@ shell = "{{ vars.task_shell }}" [tasks.ryl-check] description = "Lint YAML with ryl (a yamllint-compatible Rust linter)" -run = "ryl -c config/ryl.yaml .github config" +run = "ryl check -c config/ryl.toml .github config" [tasks._setup_all] description = "Private shared cross-platform preinstall" diff --git a/CHANGELOG.md b/CHANGELOG.md index 0c201e04..2114330d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -37,6 +37,12 @@ Changelog 1.1.0. uploads to storage through any hub, and can reconnect to a different hub without losing what was queued for it. The default, `local`, behaves as before. In NATS mode the `WS_AGENT_REGISTRY` file is neither read nor written. The new `et-hub-backplane` crate holds the NATS side, and `et-test-nats` starts a throwaway broker for tests. +- A scenario can run a hub on more than one machine: `services.hub.machines` lists them, the server among them. Each + runner joins the hub on its own machine, or the server's when its machine runs none. Every format then also runs a + NATS server and a rustfs object store on the server, which the hubs share as their backplane and their agent + storage, so an upload through one hub reads back through any other. `services.backplane` and `services.storage` + take `limits:` for the two. `secrets.env` carries the store's secret as `AWS_SECRET_ACCESS_KEY` and + `RUSTFS_SECRET_KEY`. A scenario without `services.hub.machines` renders as before. ### Changed diff --git a/CHECKS.md b/CHECKS.md index 0e5d31d7..7320d962 100644 --- a/CHECKS.md +++ b/CHECKS.md @@ -296,7 +296,8 @@ Rules, from `config/ast-grep/rules`: so an `args` entry also silently discards the arguments the image was built to start with. - `write-ending-newline` -- A `write!` whose format string ends in `\n` is a `writeln!` written the long way: use `writeln!` and drop the final `\n`. This mirrors `DeepSource RS-C1005`. clippy's `write_with_newline` covers only a - single trailing newline and deliberately skips a string ending in `\n\n`, which is the case this rule exists to catch. + single trailing newline in a plain literal: it skips a string ending in `\n\n`, and one assembled with `concat!`, + which are the cases this rule exists to catch. ## `cargo check` @@ -772,7 +773,7 @@ Rules, from `config/conftest/policy`: ## `ryl` - `ryl-check` (`default` env; run by `check:rust`) -- Lint YAML with ryl (a yamllint-compatible Rust linter). Config: - `config/ryl.yaml`. + `config/ryl.toml`. ## `semgrep` diff --git a/Cargo.lock b/Cargo.lock index 4b5b07a9..8db4b08a 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1471,9 +1471,9 @@ checksum = "1c133bc6a41be0d194c306b5506d15e6feeea7b1d6604bd3f8310dfb2ca96486" [[package]] name = "clipboard-win" -version = "5.4.1" +version = "5.4.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bde03770d3df201d4fb868f2c9c59e66a3e4e2bd06692a0fe701e7103c7e84d4" +checksum = "052af7878466feccda12ba7a6a67d019d712f296809559b47c12c8cc196d3a9a" dependencies = [ "error-code", ] @@ -2155,9 +2155,9 @@ dependencies = [ [[package]] name = "crypto-bigint" -version = "0.7.5" +version = "0.7.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1a52aa3fcda4e6302a9f48734f234d35d4721b96f8fe07d073f07ce9df4f0271" +checksum = "d726facc8b27e604391b232b970cdc19b5468c69f08b466b9734c759216c7415" dependencies = [ "cpubits", "ctutils", @@ -2225,9 +2225,9 @@ dependencies = [ [[package]] name = "ctutils" -version = "0.4.3" +version = "0.4.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "03bb0e1cc970d482d121d9a1744999169b69a07470b3d644a7894e53fcaf4574" +checksum = "38d28fb6aa7dd680b3ffd1a3e64e115c177bdba63c0ae7f9cd48129ff05c4fa0" dependencies = [ "cmov", "subtle", @@ -4338,7 +4338,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9d65aa39b3a5c1c9c1b745c9a019234bb7a21b77abcb4f4d266d706e2d577d65" dependencies = [ "base16ct 1.0.0", - "crypto-bigint 0.7.5", + "crypto-bigint 0.7.8", "crypto-common 0.2.2", "ff 0.14.0", "group 0.14.0", @@ -11459,7 +11459,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "32497e9a4c7b38532efcdebeef879707aa9f794296a4f0244f6f69e9bc8574bd" dependencies = [ "fastrand", - "getrandom 0.4.3", + "getrandom 0.3.4", "once_cell", "rustix 1.1.5", "windows-sys 0.61.2", diff --git a/README.md b/README.md index a03d0853..74c98da0 100644 --- a/README.md +++ b/README.md @@ -397,15 +397,6 @@ docker compose up --build The generated scenario config only selects which prebuilt modules `ws-server` serves. Module builds are expected to be handled separately from the repository root. -To regenerate all checked-in verification outputs from `verification/*/input`, writing each scenario to the -matching `verification/*/output/` folder. This renders each scenario for every supported deployment -tool -- mise, dekit, process-compose, Docker Compose and k3s -- and each scenario's own `README.md` names the files -it holds and how to run them: - -```bash -mise run regen-verification -``` - ### k3s The third output type writes Kubernetes manifests. Pass `--output-type k3s`, or set `deployment_type: k3s` in the @@ -425,12 +416,9 @@ every image has to exist on the node before `kubectl apply`. And the scenario's the scenario also writes a `README.md` beside the manifests carrying that scenario's own build, import and secret-loading commands. -Applying them to a live cluster is Linux-only, because k3s is a Linux Kubernetes distribution. `mise run k3s-verify` -does the whole round trip for the `math1` scenario against a disposable k3d cluster -- create, import, apply, wait -for both rollouts, then read back the model the two runners exchanged -- and deletes the cluster on the way out. It -expects the images to have been built already and says so if they have not. Generating and checking the manifests -is not Linux-only: `verification-check` fails on drift, `kubeconform-check` validates them against the Kubernetes -schemas, and `trivy-check` applies its Kubernetes security rules, all on every platform. +Applying them to a live cluster is Linux-only, because k3s is a Linux Kubernetes distribution. Generating and +checking the manifests is not: `kubeconform-check` validates them against the Kubernetes schemas, and `trivy-check` +applies its Kubernetes security rules, both on every platform. The containers run hardened: a read-only root filesystem, no privilege escalation, every capability dropped, and a non-root user, with `emptyDir` mounts for the paths that still have to be writable. Traffic between them is @@ -453,11 +441,31 @@ machines: agents: ["math1-twin"] ``` -`verification/local/input/math1-multi.yaml` is that split with both machines at loopback, so one host can play every -machine; its `verification/published/` twin renders the same split against the released artifacts, which need -nothing new to run it. `mise run k3s-verify --scenario math1-multi` applies it to a k3d cluster with one node per -machine and checks each pod landed on its own machine's node. `mise run compose-verify` brings it up one machine -profile at a time. +### Several hubs + +`services.hub.machines:` runs a hub on each machine it lists, the server among them. The hubs join into one through +a NATS server, the backplane carrying the agent directory, the mailbox and delivery between hubs, and point their +agent storage at one rustfs object store, so an upload through one hub reads back through any other. Both run on the +server beside the collector, and `services.backplane.limits:` and `services.storage.limits:` set their limits. Each +runner joins the hub on its own machine, or the server's when its machine runs none: + +```yaml +machines: + - name: "server" + address: "10.0.0.1" + agents: ["math1-trigger"] + - name: "edge" + address: "10.0.0.2" + agents: ["math1-twin"] +services: + hub: + machines: ["server", "edge"] +``` + +NATS takes no credential, the same trust the hubs' plaintext WebSocket listeners already place in the network between +the machines; the object store's secret is in `secrets.env`. In every format but k3s a hub off the server exports no +telemetry, because the collector listens on the server's loopback only. Every hub listens on the same ports, so only +k3s, where each pod has a network of its own, runs several on one host. ### Resource limits @@ -487,9 +495,7 @@ Compose and k3s apply every field, and each lands in the same cgroup value under takes a default both share. k3s carries what the Pod API cannot through two upstream container runtime plugins, containerd's `ulimit-adjuster` and nri-plugins' `nri-memory-qos`, and refuses a limited pod on a node without them, so the generated `README.md` has every machine prepared first. The mise, dekit and process-compose deployments run each -process directly on the host and apply none. `verification/local/input/math1-limits.yaml` sets a sample, which -`mise run k3s-verify local --scenario math1-limits` and `mise run compose-verify --scenario math1-limits` check in the -running containers. +process directly on the host and apply none. ### Scheduled agents @@ -512,8 +518,7 @@ agents: k3s runs it as a `CronJob`. Compose adds a `scheduler` service, [Ofelia](https://github.com/netresearch/ofelia), which starts the runner's container again on each tick and mounts the Docker socket to do so. process-compose, whose `process-compose.yaml` runs the `mise.toml` tasks as `dekit.yaml` does, schedules the task itself. mise and dekit have -no scheduler, so generating either for a scheduled scenario is an error, and `regen-verification` leaves both out of -such a scenario. `verification/local/input/math1-cron.yaml` runs the trigger every minute. +no scheduler, so generating either for a scheduled scenario is an error. ## Grant diff --git a/config/ast-grep/rules/write-ending-newline.yaml b/config/ast-grep/rules/write-ending-newline.yaml index 60a5f964..72c70349 100644 --- a/config/ast-grep/rules/write-ending-newline.yaml +++ b/config/ast-grep/rules/write-ending-newline.yaml @@ -4,7 +4,8 @@ severity: error message: | A `write!` whose format string ends in `\n` is a `writeln!` written the long way: use `writeln!` and drop the final `\n`. This mirrors `DeepSource RS-C1005`. clippy's `write_with_newline` covers only a single trailing - newline and deliberately skips a string ending in `\n\n`, which is the case this rule exists to catch. + newline in a plain literal: it skips a string ending in `\n\n`, and one assembled with `concat!`, which are the + cases this rule exists to catch. # There is no autofix. # The rewrite renames the macro and edits the string literal inside its token tree at once, which a single `fix:` # template over the whole invocation cannot express without re-spelling every argument. @@ -13,7 +14,12 @@ rule: has: field: macro regex: ^write$ - regex: '(?s)^write!\s*\((?:[^"()]|\([^()]*\))*,\s*"(?:[^"\\]|\\.)*\\n"' + # The format string is either a literal ending in `\n`, or a `concat!` whose last literal does. + # Verbose mode, so the pattern can span lines: its whitespace is all `\s`, which verbose mode leaves alone. + regex: |- + (?sx)^write!\s*\((?:[^"()]|\([^()]*\))*,\s* + (?:"(?:[^"\\]|\\.)*\\n" + |concat!\s*\((?:\s*"(?:[^"\\]|\\.)*"\s*,)*\s*"(?:[^"\\]|\\.)*\\n"\s*,?\s*\)) ignores: - generated/** # Generated; refer to int-gen bindings.rs for details. diff --git a/config/ast-grep/tests/write-ending-newline.yaml b/config/ast-grep/tests/write-ending-newline.yaml new file mode 100644 index 00000000..78486001 --- /dev/null +++ b/config/ast-grep/tests/write-ending-newline.yaml @@ -0,0 +1,50 @@ +id: write-ending-newline +valid: + - | + fn run(out: &mut String) { + let _result = writeln!(out, "done"); + } + - | + fn run(out: &mut String) { + let _result = write!(out, "no newline"); + } + - | + fn run(out: &mut String) { + let _result = write!(out, "a newline\nin the middle"); + } + - | + fn run(out: &mut String) { + let _result = writeln!(out, concat!("first ", "second")); + } + - | + fn run(out: &mut String) { + let _result = write!(out, concat!("first line\n", "then no newline")); + } +invalid: + - | + fn run(out: &mut String) { + let _result = write!(out, "done\n"); + } + - | + fn run(out: &mut String) { + let _result = write!(out, "a blank line after\n\n"); + } + - | + fn run(out: &mut String, name: &str) { + let _result = write!(out, "{name}\n", name = name); + } + - | + fn run(out: &mut String) { + let _result = write!(out, concat!("first ", "second\n")); + } + - | + fn run(out: &mut String, name: &str) { + let _result = write!( + out, + concat!( + "first\n", + "{name}\n", + ), + name = name, + ); + } diff --git a/config/conftest/policy/jscpd/jscpd.rego b/config/conftest/policy/jscpd/jscpd.rego index 2d611c13..3ad4bfd3 100644 --- a/config/conftest/policy/jscpd/jscpd.rego +++ b/config/conftest/policy/jscpd/jscpd.rego @@ -24,7 +24,7 @@ package jscpd # Ceiling on the clone fingerprints recorded in config/jscpd-baseline.json. # May only ever be lowered. See the header for what raising it implies. -declared_clones := 341 +declared_clones := 347 # How far under the ceiling the baseline may sit before the ceiling has to be banked downward. max_slack := 20 diff --git a/config/deny.toml b/config/deny.toml index c1e87c6c..4fc21cf7 100644 --- a/config/deny.toml +++ b/config/deny.toml @@ -113,6 +113,15 @@ ignore = [ # decryption. Not reachable unless guest JS explicitly imports crypto.subtle and performs RSA-OAEP decrypt # with a private key. Our modules don't. { id = "RUSTSEC-2023-0071", reason = "expires 2026-10-31" }, + # crypto-bigint 0.5.5 secret-dependent branches in operations meant to be constant-time (timing side channel). + # The arithmetic under elliptic-curve 0.13, which deno_crypto and deno_node_crypto use for P-256/384/521 and + # secp256k1 in WebCrypto and node:crypto. The fix is only in 0.7.6 and later, and deno_crypto requires + # elliptic-curve ^0.13 through its newest release, 0.274.0, so no bump reaches it. Live code, but reached only when + # guest JS in the web runner signs or derives with an EC private key, and leaking it takes an attacker able to time + # those operations on the same host. Our modules do neither; the hub's TLS uses rustls and aws-lc, not this. Drop + # when deno_crypto moves to elliptic-curve 0.14. cargo-deny ignores by id only, so this also mutes the 0.7 copy + # elliptic-curve 0.14 pulls -- Cargo.lock pins the patched 0.7.8 there, so re-check that copy when touching this. + { id = "RUSTSEC-2026-0387", reason = "expires 2026-10-31" }, # h2 0.3.27 unbounded empty DATA frames (low-severity HTTP/2 DoS: undrained streams grow memory). # Accepted for the 0.3 copy only. hyperium patched the 0.4 line (>= 0.4.16, which Cargo.lock now pins for the # hyper/reqwest path) and shipped no 0.3 backport, so 0.3.27 is the last release of a dead line. It arrives diff --git a/config/jscpd-baseline.json b/config/jscpd-baseline.json index 7671ae99..16e9b83c 100644 --- a/config/jscpd-baseline.json +++ b/config/jscpd-baseline.json @@ -39,6 +39,7 @@ "1a07ddf368451c0a": 1, "1a1867cca4e1d4e8": 1, "1a1e1611e4db4656": 1, + "1a97e5c72a30152e": 1, "1bd6f5989d1961e5": 1, "1c64414236b8476c": 1, "1c8cf13e10b67513": 1, @@ -52,6 +53,7 @@ "249241d8c264bc63": 1, "25f2fdb4e93d5683": 1, "264e155c8fdaf095": 1, + "265b3a5011a42fc4": 1, "266cd9babed7be0c": 1, "26bec4de311d99cf": 1, "2c47cf795be8c0d9": 1, @@ -74,6 +76,7 @@ "380c183a78daef06": 1, "389aea3e8662fb9b": 1, "39df8b0c7c0dccc1": 1, + "3a3c6b4e89834dc8": 1, "3aec07afda461f58": 1, "3afe9805243f94d2": 1, "3b30dfa6b363682d": 2, @@ -225,6 +228,7 @@ "a5f0c7895f14c806": 2, "a68200ccaced4620": 1, "a68703c304b8c5fa": 1, + "a73a923e61e87ae8": 1, "a743e181c2f6424b": 1, "a896f65e46666bd0": 1, "a9f793157ec6fced": 1, @@ -240,6 +244,7 @@ "ae590b605758bec9": 1, "afc243bb6427e6d9": 1, "b07cc7ec51bc1602": 1, + "b0d913f1fde4e8e3": 1, "b29077918336343b": 1, "b32ec6344a29f094": 1, "b3b79293ad6e0885": 1, @@ -301,6 +306,7 @@ "d8a0ad2f6e337e91": 1, "da7b27ae8c89cb73": 1, "da812f4e54fefbf1": 1, + "dd162080de996936": 1, "dd41701c3663b9f3": 1, "ddca47a4caded0fd": 1, "e152738c60f43b0e": 1, diff --git a/config/osv-scanner.toml b/config/osv-scanner.toml index 98fa975c..ae29aae4 100644 --- a/config/osv-scanner.toml +++ b/config/osv-scanner.toml @@ -19,4 +19,5 @@ IgnoredVulns = [ { id = "RUSTSEC-2026-0349", ignoreUntil = 2026-10-31 }, { id = "RUSTSEC-2026-0381", ignoreUntil = 2026-10-31 }, { id = "RUSTSEC-2026-0383", ignoreUntil = 2026-10-31 }, + { id = "RUSTSEC-2026-0387", ignoreUntil = 2026-10-31 }, ] diff --git a/config/ryl.toml b/config/ryl.toml new file mode 100644 index 00000000..8666450f --- /dev/null +++ b/config/ryl.toml @@ -0,0 +1,46 @@ +# Starts from yamllint's standard ruleset, every rule of it stated. +# Then aligns the cosmetic rules with the repo's other tools so they never disagree. +[files] +yaml = ["*.yaml", "*.yml", ".yamllint"] + +[lint.rules] +anchors = "enable" +braces = "enable" +brackets = "enable" +colons = "enable" +commas = "enable" +document-end = "disable" +# The repo's workflows and configs don't use a `---` document-start marker. +document-start = "disable" +empty-lines = "enable" +empty-values = "disable" +float-values = "disable" +hyphens = "enable" +indentation = "enable" +key-duplicates = "enable" +key-ordering = "disable" +new-line-at-end-of-file = "enable" +new-lines = "enable" +octal-values = "disable" +quoted-strings = "disable" +trailing-spaces = "enable" + +# dprint (the YAML formatter) writes one space before an inline `#`. +# yamllint defaults to two. Defer to dprint so the two can't fight. +[lint.rules.comments] +level = "warning" +min-spaces-from-content = 1 + +# A misindented comment reads as attached to the wrong block, so treat it as an error, not a warning. +[lint.rules.comments-indentation] +level = "error" + +# Match the editorconfig line length (yamllint defaults to 80). +[lint.rules.line-length] +max = 120 + +[lint.rules.truthy] +level = "warning" + +[format] +quote-style = "preserve" diff --git a/config/ryl.yaml b/config/ryl.yaml deleted file mode 100644 index 8aa543c8..00000000 --- a/config/ryl.yaml +++ /dev/null @@ -1,17 +0,0 @@ -# Starts from yamllint's standard ruleset. -# Then aligns the cosmetic rules with the repo's other tools so they never disagree. -extends: default - -rules: - # A misindented comment reads as attached to the wrong block, so treat it as an error, not a warning. - comments-indentation: - level: error - # dprint (the YAML formatter) writes one space before an inline `#`. - # yamllint defaults to two. Defer to dprint so the two can't fight. - comments: - min-spaces-from-content: 1 - # Match the editorconfig line length (yamllint defaults to 80). - line-length: - max: 120 - # The repo's workflows and configs don't use a `---` document-start marker. - document-start: disable diff --git a/config/semgrep/no-check-tool-mentions.yaml b/config/semgrep/no-check-tool-mentions.yaml index 5e297e15..69af9143 100644 --- a/config/semgrep/no-check-tool-mentions.yaml +++ b/config/semgrep/no-check-tool-mentions.yaml @@ -258,7 +258,7 @@ rules: - "/Cargo.lock" # Carries its plugin crates as dependency names. - "/config/dprint.jsonc" # Its own config, including the settings held to agree with the JS formatter. - "/config/oxfmtrc.jsonc" # The other half of that agreement. - - "/config/ryl.yaml" # Defers to it on inline-comment spacing so the two cannot fight. + - "/config/ryl.toml" # Defers to it on inline-comment spacing so the two cannot fight. - "/config/semgrep/no-line-length-in-comment.yaml" # Names the configs that hold the line-length value. - "/config/semgrep/prefer-yaml-toml.yaml" # Allowlists its config, which can only be JSONC. - "/config/ast-grep/rules/no-current-dir.yaml" # Names the root-marker file, not the tool. @@ -584,7 +584,7 @@ rules: exclude: - "/.mise/**" # Tool pin, lockfile entry, and the task that runs it. - "/AGENTS.md" # Documents which checks an agent runs for which file types. - - "/config/ryl.yaml" # Its own config. + - "/config/ryl.toml" # Its own config. - "/config/conftest/policy/mise_lock/mise_lock.rego" # Allowlists it as a tool locking a glibc build. - "/config/conftest/policy/mise/mise.rego" # Allowlists it as a tool with no prebuilt at some triple. - "/config/taplo/mise-cargo-backend-allowlist.schema.json" # The same allowlist, as a TOML schema enum. diff --git a/libs/edge-toolkit/src/ports.rs b/libs/edge-toolkit/src/ports.rs index e5fd5846..c4754531 100644 --- a/libs/edge-toolkit/src/ports.rs +++ b/libs/edge-toolkit/src/ports.rs @@ -1,6 +1,8 @@ //! Port allocation. -use Services::{InsecureWebSocketServer, OtlpCollector, SecureWebSocketServer}; +use Services::{ + Backplane, BackplaneMonitor, InsecureWebSocketServer, OtlpCollector, SecureWebSocketServer, SharedStorage, +}; /// Define each services port. #[derive(Clone, Copy, Debug, Eq, Hash, PartialEq)] @@ -13,6 +15,13 @@ pub enum Services { InsecureWebSocketServer, /// Secure WebSocket server. SecureWebSocketServer, + + /// NATS client port of the backplane several hubs share. + Backplane, + /// NATS monitoring port of the backplane, which serves its health check. + BackplaneMonitor, + /// S3 endpoint of the object store several hubs share. + SharedStorage, } impl Services { @@ -25,6 +34,11 @@ impl Services { InsecureWebSocketServer => 8080, SecureWebSocketServer => 8443, + + // NATS's and rustfs's own defaults + Backplane => 4222, + BackplaneMonitor => 8222, + SharedStorage => 9000, } } } diff --git a/libs/edge-toolkit/tests/ports.rs b/libs/edge-toolkit/tests/ports.rs new file mode 100644 index 00000000..6a358426 --- /dev/null +++ b/libs/edge-toolkit/tests/ports.rs @@ -0,0 +1,14 @@ +#![cfg(test)] + +use edge_toolkit::ports::Services; + +#[rstest::rstest] +#[case::otlp_collector(Services::OtlpCollector, 5080)] +#[case::insecure_web_socket_server(Services::InsecureWebSocketServer, 8080)] +#[case::secure_web_socket_server(Services::SecureWebSocketServer, 8443)] +#[case::backplane(Services::Backplane, 4222)] +#[case::backplane_monitor(Services::BackplaneMonitor, 8222)] +#[case::shared_storage(Services::SharedStorage, 9000)] +fn port(#[case] service: Services, #[case] expected: u16) { + assert_eq!(service.port(), expected); +} diff --git a/services/hub-backplane/Cargo.toml b/services/hub-backplane/Cargo.toml index bbf618a4..e8a7df5f 100644 --- a/services/hub-backplane/Cargo.toml +++ b/services/hub-backplane/Cargo.toml @@ -19,6 +19,7 @@ futures-util.workspace = true serde.workspace = true serde_json.workspace = true thiserror.workspace = true +tokio = { workspace = true, features = ["time"] } tracing.workspace = true uuid.workspace = true diff --git a/services/hub-backplane/src/error.rs b/services/hub-backplane/src/error.rs index e88ddf2a..b8e00849 100644 --- a/services/hub-backplane/src/error.rs +++ b/services/hub-backplane/src/error.rs @@ -44,6 +44,9 @@ pub enum BackplaneError { #[error("no pending message {0} to acknowledge")] NoPendingMessage(String), + + #[error("the NATS server did not confirm this hub's subscriptions within {0:?}")] + SubscriptionsUnconfirmed(std::time::Duration), } /// Convert any NATS client error into [`BackplaneError::Nats`], naming what was being attempted. diff --git a/services/hub-backplane/src/lib.rs b/services/hub-backplane/src/lib.rs index b4934342..361ff6f2 100644 --- a/services/hub-backplane/src/lib.rs +++ b/services/hub-backplane/src/lib.rs @@ -16,6 +16,7 @@ use std::collections::BTreeMap; use std::future::ready; use std::net::IpAddr; +use std::time::Duration; use async_nats::jetstream::kv::{self, CreateErrorKind, Operation, Store, UpdateErrorKind}; use async_nats::{Client, HeaderMap, Message}; @@ -42,6 +43,9 @@ const FROM_HEADER: &str = "Et-From"; const ORIGIN_HEADER: &str = "Et-Origin-Hub"; const KIND_HEADER: &str = "Et-Kind"; +/// How long `subscribe` waits for the server to confirm its subscriptions; a healthy server takes one round trip. +const CONFIRM_TIMEOUT: Duration = Duration::from_secs(10); + /// The backplane a hub routes through: nothing beyond this process, or a shared NATS server. #[derive(Clone, Debug)] #[expect( @@ -357,6 +361,38 @@ impl NatsBackplane { .map_nats_err("broadcast to the other hubs") } + /// Wait until the server has registered every subscription this connection has sent. + /// + /// The server registers a subscription only once it reads the SUB, and a frame another hub publishes before then + /// is dropped, since core NATS keeps nothing for a subject with no subscriber. A flush does not wait for that: it + /// resolves once the bytes leave this process. So a message is sent to a fresh inbox this connection subscribes + /// to, and awaited: the server handles a connection's commands in order, so the echo arriving means every SUB + /// before it has been read. + /// + /// The wait is bounded, because an echo can be lost rather than late: a reconnect between the publish and its + /// delivery drops it, and an account not allowed to publish to its inboxes never sends it. Either way the caller + /// gets an error to act on, rather than a hub that never finishes starting. + async fn confirm_subscriptions(&self) -> Result<(), BackplaneError> { + let inbox = self.client.new_inbox(); + let mut echo = self + .client + .subscribe(inbox.clone()) + .await + .map_nats_err("subscribe to the confirmation inbox")?; + self.client + .publish(inbox, Bytes::new()) + .await + .map_nats_err("publish to the confirmation inbox")?; + let echoed = tokio::time::timeout(CONFIRM_TIMEOUT, echo.next()).await; + let _echoed: Message = echoed + .ok() + .flatten() + .ok_or(BackplaneError::SubscriptionsUnconfirmed(CONFIRM_TIMEOUT))?; + echo.unsubscribe() + .await + .map_nats_err("unsubscribe from the confirmation inbox") + } + /// The frames other hubs send this hub's agents, from the moment this returns. /// /// This hub's own broadcasts are left out, since it has already fanned them out locally. A message that does not @@ -372,9 +408,7 @@ impl NatsBackplane { .subscribe(BROADCAST_SUBJECT) .await .map_nats_err("subscribe to broadcasts")?; - // The server registers a subscription only once it reads the SUB; flushing waits for that, so nothing - // published after this returns can miss it. - self.client.flush().await.map_nats_err("flush subscriptions")?; + self.confirm_subscriptions().await?; let own_hub = self.hub_id.clone(); let direct = direct.map(|message| Inbound::decode_direct(&message)); diff --git a/services/hub-backplane/tests/backplane.rs b/services/hub-backplane/tests/backplane.rs index fde12d4c..e616e65a 100644 --- a/services/hub-backplane/tests/backplane.rs +++ b/services/hub-backplane/tests/backplane.rs @@ -261,19 +261,14 @@ async fn concurrent_senders_all_land_in_one_mailbox() { assert_eq!(hubs[0].pending_for("busy").await.unwrap().len(), 40); } -// Known intermittent failure, not yet fixed: on slow runners (the `override (mingw)` lane of test.yaml) this times -// out in `next_inbound` with +// Fixed in `NatsBackplane::subscribe`, which now returns only once the server has registered its subscriptions. +// Before that, on slow runners (the `override (mingw)` lane of test.yaml) this timed out in `next_inbound` with // // panicked at services\hub-backplane\tests\backplane.rs:30:10: // called `Result::unwrap()` on an `Err` value: Elapsed(()) // // on commit https://github.com/edge-toolkit/core/commit/c91d332c72d0c94f9fd4aa915c1d485fee251fd0 at -// https://github.com/edge-toolkit/core/actions/runs/39857810014/job/120100468951. The cause is in -// `NatsBackplane::subscribe`, not here: its `flush()` is meant to wait until the server has registered the -// subscriptions, but async-nats 0.50's `Client::flush` resolves once the write buffer reaches the socket, with no -// server round trip. A frame the other hub publishes on its own connection can then reach the server before the -// SUB does, and core NATS drops it. The fix is a real round trip in `subscribe` (say, publish to a fresh inbox and -// await its echo), not a longer timeout here. +// https://github.com/edge-toolkit/core/actions/runs/39857810014/job/120100468951. #[tokio::test(flavor = "multi_thread", worker_threads = 2)] async fn frames_reach_the_named_hub_and_broadcasts_reach_the_others() { let server = NatsServer::start(); diff --git a/utilities/cli/src/deployment_types/dekit.rs b/utilities/cli/src/deployment_types/dekit.rs index fef764cc..873ac735 100644 --- a/utilities/cli/src/deployment_types/dekit.rs +++ b/utilities/cli/src/deployment_types/dekit.rs @@ -1,23 +1,23 @@ use std::path::Path; -use edge_toolkit::ports::Services; use fs_err as fs; use serde::Serialize; use serde_yaml::{Mapping, Value}; use super::document; -use super::mise::{COLLECTOR_TASK, HUB_TASK, OPENER_TASK, collector_container}; +use super::hub_sharing::server_containers; +use super::mise::{COLLECTOR_TASK, OPENER_TASK, container}; use super::yaml::{insert, strings}; use crate::error::CliError; use crate::input::ClusterInput; -use crate::placement::{Site, hub_site}; +use crate::placement::{Site, hub_site, hubs, shares_hubs}; use crate::{OutputType, hub_http_base, scenario_runners}; -/// How long the collector gets to exit once stopped. +/// How long the collector, or any other container the deployment runs, gets to exit once stopped. /// /// `docker stop` itself waits up to ten seconds before it kills the container, which is dekit's whole default, so the /// default would kill the task while Docker was still stopping it. -const COLLECTOR_STOP_TIMEOUT: &str = "30s"; +const CONTAINER_STOP_TIMEOUT: &str = "30s"; /// Write the `dekit.yaml` that runs the scenario with `dekit up`, one task per process. /// @@ -27,12 +27,11 @@ const COLLECTOR_STOP_TIMEOUT: &str = "30s"; /// /// What it adds is order. mise starts the hub, the collector and the runners together, so a runner can ask the hub for /// its module before the hub is listening. Here each waits on what it talks to being ready, the same edges the compose -/// stack gates on: the hub on the collector's health endpoint, every runner on the hub's. The task names need no path -/// of their own, because the file is its own project root: dekit takes the nearest `dekit.yaml` as the root, and -/// starts every task there, which is where mise finds the `mise.toml`. +/// stack gates on: the server's hub on the collector's health endpoint and on those of what the hubs share, every +/// runner on its hub's. The task names need no path of their own, because the file is its own project root: dekit +/// takes the nearest `dekit.yaml` as the root, and starts every task there, which is where mise finds the `mise.toml`. pub fn generate_dekit_deployment(cluster: &ClusterInput, output_dir: &Path) -> Result<(), CliError> { let runners = scenario_runners(cluster)?; - let collector_url = format!("http://localhost:{}", Services::OtlpCollector.port()); // A multi-machine cluster tags each task with its machine and starts none on `dekit up`, since every machine runs // the same file and `up` would otherwise start the whole cluster on each of them; `dekit start +` starts // one machine's share instead. @@ -40,26 +39,32 @@ pub fn generate_dekit_deployment(cluster: &ClusterInput, output_dir: &Path) -> R let server = hub_site(cluster); let mut tasks = Mapping::new(); - let mut collector = task(COLLECTOR_TASK, &[], autostart, &server); - insert(&mut collector, "ready", http_ready(&format!("{collector_url}/healthz"))); - // `docker run` is stopped through Docker rather than by signalling the client: a client that dies before it has - // passed the signal on leaves the container running and holding the collector's port. - let mut stop = Mapping::new(); - let container = collector_container(&cluster.cluster_name); - insert(&mut stop, "cmd", strings(&["docker", "stop", &container])); - insert(&mut stop, "timeout", Value::from(COLLECTOR_STOP_TIMEOUT)); - insert(&mut collector, "stop", Value::Mapping(stop)); - insert(&mut tasks, COLLECTOR_TASK, Value::Mapping(collector)); - - let mut hub = task(HUB_TASK, &[COLLECTOR_TASK], autostart, &server); - insert(&mut hub, "ready", http_ready(&format!("{}/health", hub_http_base()))); - insert(&mut tasks, HUB_TASK, Value::Mapping(hub)); + let containers = server_containers(shares_hubs(cluster)); + let server_hub_deps: Vec<&str> = containers.iter().map(|(name, ..)| *name).collect(); + for (name, port, path) in containers { + let mut service = task(name, &[], autostart, &server); + insert( + &mut service, + "ready", + http_ready(&format!("http://localhost:{port}{path}")), + ); + // `docker run` is stopped through Docker rather than by signalling the client: a client that dies before it + // has passed the signal on leaves the container running and holding the service's port. + let mut stop = Mapping::new(); + let container = container(name, &cluster.cluster_name); + insert(&mut stop, "cmd", strings(&["docker", "stop", &container])); + insert(&mut stop, "timeout", Value::from(CONTAINER_STOP_TIMEOUT)); + insert(&mut service, "stop", Value::Mapping(stop)); + insert(&mut tasks, name, Value::Mapping(service)); + } + insert_hub_tasks(&mut tasks, cluster, &server_hub_deps, autostart); for runner in runners { - // A runner on another machine cannot wait on a hub task this runner does not hold, so it restarts until the - // hub on the server answers instead. + // A runner on a machine without a hub cannot wait on a hub task this machine does not hold, so it restarts + // until the hub on the server answers instead. let remote = runner.site.is_remote(); - let deps: &[&str] = if remote { &[] } else { &[HUB_TASK] }; + let hub = runner.site.hub_name(cluster); + let deps: &[&str] = if remote { &[] } else { &[&hub] }; let mut runner_task = task(&runner.name, deps, autostart, &runner.site); if remote { insert(&mut runner_task, "autorestart", Value::from("on-failure")); @@ -87,6 +92,26 @@ pub fn generate_dekit_deployment(cluster: &ClusterInput, output_dir: &Path) -> R Ok(()) } +/// Add a task per hub, the server's waiting on `server_deps`. +/// +/// A hub off the server cannot wait on the services it shares there, so it restarts until they answer instead. +#[expect(clippy::single_call_fn, reason = "the per-hub step of generate_dekit_deployment")] +fn insert_hub_tasks(tasks: &mut Mapping, cluster: &ClusterInput, server_deps: &[&str], autostart: bool) { + for hub in hubs(cluster) { + let deps: &[&str] = if hub.on_server() { server_deps } else { &[] }; + let mut hub_task = task(&hub.name, deps, autostart, &hub.site); + insert( + &mut hub_task, + "ready", + http_ready(&format!("{}/health", hub_http_base())), + ); + if !hub.on_server() { + insert(&mut hub_task, "autorestart", Value::from("on-failure")); + } + insert(tasks, &hub.name, Value::Mapping(hub_task)); + } +} + /// The task that runs the mise task `name`, after `deps`, started by `dekit up` when `autostart` is set. /// /// A task left out of `autostart` still starts when another that depends on it does, so only what nothing else pulls in diff --git a/utilities/cli/src/deployment_types/docker_compose.rs b/utilities/cli/src/deployment_types/docker_compose.rs index dbd5ace2..5b5a03c6 100644 --- a/utilities/cli/src/deployment_types/docker_compose.rs +++ b/utilities/cli/src/deployment_types/docker_compose.rs @@ -1,13 +1,18 @@ use std::path::{Path, PathBuf}; +use edge_toolkit::ports::Services; use et_path::{absolute_from, relative_path_from}; use fs_err as fs; -use super::{REGISTRY_TOKEN_ENV, REGISTRY_TOKEN_SECRET, scenario_image_needs}; +use super::hub_sharing::{ + BACKPLANE_HEALTH_PATH, BACKPLANE_IMAGE, BACKPLANE_TASK, DATA_DIR, SHARED_STORAGE_HEALTH_PATH, SHARED_STORAGE_IMAGE, + SHARED_STORAGE_TASK, backplane_args, shared_hub_env, shared_storage_env, shared_storage_start, +}; +use super::{COLLECTOR_TASK, REGISTRY_TOKEN_ENV, REGISTRY_TOKEN_SECRET, scenario_image_needs}; use crate::error::CliError; use crate::input::{ArtifactSource, ClusterInput}; use crate::limits::{CpuCap, ResolvedLimits, compose_bytes}; -use crate::placement::hub_site; +use crate::placement::{Hub, hub_site, hubs, shares_hubs}; use crate::{ COLLECTOR_SETTINGS, COLLECTOR_USERNAME, IMAGE_REGISTRY, OutputType, RunnerInstance, SECRETS_ENV_FILE, cluster_module_names, hub_http_base, module_path_contexts, module_registry, resolve_cluster_runners, @@ -42,6 +47,14 @@ const YAML_INDICATORS: [char; 18] = [ '@', '`', '-', '?', ':', ',', '[', ']', '{', '}', '#', '&', '*', '!', '|', '>', '\'', '"', ]; +/// Whether YAML reads `value`, written bare, as a boolean, a null or a number rather than the string it is. +/// +/// The YAML 1.1 spellings are included, since compose's parser still honours `yes`, `no`, `on` and `off`. +fn reads_as_non_string(value: &str) -> bool { + const RESERVED: [&str; 10] = ["true", "false", "yes", "no", "on", "off", "y", "n", "null", "~"]; + RESERVED.iter().any(|reserved| value.eq_ignore_ascii_case(reserved)) || value.parse::().is_ok() +} + /// The scheduler service for the machine `machines` names, or for the single host when it names none. /// /// One per machine with a scheduled runner, since Ofelia can only start a container on the Docker daemon it runs on. @@ -98,13 +111,18 @@ fn healthy(service: &str) -> (String, ComposeDependsOnCondition) { reason = "the healthcheck step of generate_docker_compose_deployment" )] fn hub_healthcheck() -> ComposeHealthcheck { + curl_healthcheck(&format!("{}/health", hub_http_base())) +} + +/// A healthcheck that passes once `url` answers with a success status, asked with the image's own `curl`. +fn curl_healthcheck(url: &str) -> ComposeHealthcheck { + healthcheck(&["CMD", "curl", "-fsS", url]) +} + +/// A healthcheck running `test`, on the cadence every service here shares. +fn healthcheck(test: &[&str]) -> ComposeHealthcheck { ComposeHealthcheck { - test: vec![ - "CMD".to_string(), - "curl".to_string(), - "-fsS".to_string(), - format!("{}/health", hub_http_base()), - ], + test: test.iter().map(|word| (*word).to_string()).collect(), interval: "5s".to_string(), timeout: "3s".to_string(), retries: 20, @@ -124,7 +142,7 @@ fn scenario_hub_service( cluster: &ClusterInput, output_abs: &Path, hub_contexts: Vec<(String, String)>, - profiles: Vec, + hub: &Hub, ) -> Result<(ComposeService, Vec), CliError> { let workspace_root = edge_toolkit::config::get_project_root(); let serves_a_page = super::serves_a_page(cluster); @@ -145,8 +163,27 @@ fn scenario_hub_service( } else { Vec::new() }; + let shared = shares_hubs(cluster); + let mut environment = hub_environment(module_paths, &workspace_root, serves_a_page, shared); + let mut depends_on = vec![healthy(COLLECTOR_TASK)]; + if shared { + environment.extend( + shared_hub_env(&hub.server_host, &hub.server_host) + .into_iter() + .map(|(name, value)| (name.to_string(), ComposeValue::Plain(value))), + ); + environment.sort_by(|(left, _), (right, _)| left.cmp(right)); + depends_on.extend([healthy(BACKPLANE_TASK), healthy(SHARED_STORAGE_TASK)]); + } + // A hub off the server cannot be gated on services its own compose project does not start, so the edges are kept + // but waived, and the hub is restarted until they answer instead, as a runner off the server is. + if !hub.on_server() { + for (_, condition) in &mut depends_on { + condition.required = false; + } + } let service = ComposeService { - profiles, + profiles: hub.site.machine_names(), build: Some(ComposeBuild { source, // Relative to this file, which is what compose resolves a context path against. @@ -156,13 +193,18 @@ fn scenario_hub_service( .collect(), secrets: secrets.clone(), }), + restart: (!hub.on_server()).then(|| "on-failure".to_string()), network_mode: Some("host".to_string()), // Carries `OTLP_AUTH_PASSWORD`: the server authenticates its OTLP exports against the same root credential the // collector was started with. The account it authenticates as is in the environment. env_file: vec![SECRETS_ENV_FILE.to_string()], - environment: hub_environment(module_paths, &workspace_root, serves_a_page), - volumes: vec!["ws-server-storage:/app/storage".to_string()], - depends_on: vec![healthy("openobserve")], + environment, + volumes: if shared { + Vec::new() + } else { + vec!["ws-server-storage:/app/storage".to_string()] + }, + depends_on, healthcheck: Some(hub_healthcheck()), limits: cluster.services.hub.limits.resolve(), ..ComposeService::default() @@ -170,6 +212,111 @@ fn scenario_hub_service( Ok((service, secrets)) } +/// The NATS and rustfs services the hubs share, reachable at `localhost` on the server and at its address elsewhere. +/// +/// rustfs takes host networking, as the hub does. NATS publishes its ports instead, as the mise task does: its client +/// port to every interface for the other machines' hubs, and its monitoring port to loopback alone, since only the +/// healthcheck reads it and NATS has no setting that binds that listener apart from the client one. +#[expect( + clippy::single_call_fn, + reason = "the shared-services step of generate_docker_compose_deployment" +)] +fn shared_services(cluster: &ClusterInput, profiles: &[String]) -> [(String, ComposeService); 2] { + let backplane = ComposeService { + profiles: profiles.to_vec(), + image: Some(BACKPLANE_IMAGE.to_string()), + command: backplane_args(), + healthcheck: Some(backplane_healthcheck()), + ports: vec![ + format!("{client}:{client}", client = Services::Backplane.port()), + format!( + "127.0.0.1:{monitor}:{monitor}", + monitor = Services::BackplaneMonitor.port() + ), + ], + volumes: vec![format!("{BACKPLANE_TASK}-data:{DATA_DIR}")], + limits: cluster.services.backplane.limits.resolve(), + ..ComposeService::default() + }; + let storage = ComposeService { + profiles: profiles.to_vec(), + image: Some(SHARED_STORAGE_IMAGE.to_string()), + entrypoint: vec!["sh".to_string(), "-c".to_string()], + command: vec![shared_storage_start()], + healthcheck: Some(curl_healthcheck(&format!( + "http://localhost:{}{SHARED_STORAGE_HEALTH_PATH}", + Services::SharedStorage.port() + ))), + network_mode: Some("host".to_string()), + // Carries `RUSTFS_SECRET_KEY`, the secret the hubs present as `AWS_SECRET_ACCESS_KEY`. + env_file: vec![SECRETS_ENV_FILE.to_string()], + environment: shared_storage_env() + .into_iter() + .map(|(name, value)| (name.to_string(), ComposeValue::Plain(value))) + .collect(), + volumes: vec![format!("{SHARED_STORAGE_TASK}-data:{DATA_DIR}")], + limits: cluster.services.storage.limits.resolve(), + ..ComposeService::default() + }; + [ + (BACKPLANE_TASK.to_string(), backplane), + (SHARED_STORAGE_TASK.to_string(), storage), + ] +} + +/// The NATS server's healthcheck, which passes once its health endpoint answers `200` with `JetStream` up. +/// +/// The image ships no `curl`, so the request is written by hand and sent with its `nc`, and the status line read back. +#[expect( + clippy::single_call_fn, + reason = "the NATS step of shared_services, separate to keep the request out of the literal" +)] +fn backplane_healthcheck() -> ComposeHealthcheck { + let probe = format!( + "printf 'GET {BACKPLANE_HEALTH_PATH} HTTP/1.0\\r\\n\\r\\n' | nc localhost {} | head -1 | grep -q ' 200 '", + Services::BackplaneMonitor.port() + ); + healthcheck(&["CMD-SHELL", &probe]) +} + +/// One service per hub, and the top-level secrets their builds read. +/// +/// A local scenario also gets the service building the hub image it layers onto, on every machine with a hub; a +/// published scenario's Dockerfile names the released image itself. +#[expect( + clippy::single_call_fn, + reason = "the hub step of generate_docker_compose_deployment" +)] +fn hub_services( + cluster: &ClusterInput, + output_abs: &Path, + workspace_rel: String, + published: bool, +) -> Result<(Vec, Vec), CliError> { + let hubs = hubs(cluster); + let mut services = Vec::new(); + let hub_contexts = if published { + Vec::new() + } else { + services.push(( + HUB_IMAGE_BUILD.to_string(), + ComposeService { + limits: cluster.services.hub.limits.resolve(), + profiles: hubs.iter().flat_map(|hub| hub.site.machine_names()).collect(), + ..hub_build_service(workspace_rel) + }, + )); + vec![("hub".to_string(), format!("service:{HUB_IMAGE_BUILD}"))] + }; + let mut secrets = Vec::new(); + for hub in &hubs { + let (service, hub_secrets) = scenario_hub_service(cluster, output_abs, hub_contexts.clone(), hub)?; + services.push((hub.name.clone(), service)); + secrets = hub_secrets; + } + Ok((services, secrets)) +} + pub fn generate_docker_compose_deployment(cluster: &ClusterInput, output_dir: &Path) -> Result<(), CliError> { let output_path = output_dir.join(OutputType::DockerCompose.output_file_name()); let workspace_root = edge_toolkit::config::get_project_root(); @@ -177,32 +324,22 @@ pub fn generate_docker_compose_deployment(cluster: &ClusterInput, output_dir: &P let workspace_rel = relative_path_from(&output_abs, &workspace_root); let artifacts = cluster.artifact_source; let published = matches!(artifacts, ArtifactSource::Published); - // Everything but the runners runs on the server machine. + // Everything but the runners and the other machines' hubs runs on the server machine. let server_profiles = hub_site(cluster).machine_names(); + let shared = shares_hubs(cluster); let mut services = vec![( - "openobserve".to_string(), + COLLECTOR_TASK.to_string(), ComposeService { profiles: server_profiles.clone(), limits: cluster.services.collector.limits.resolve(), ..openobserve_service() }, )]; - // A local scenario builds the hub it layers onto; a published one's Dockerfile names the released image itself. - let hub_contexts = if published { - Vec::new() - } else { - services.push(( - HUB_IMAGE_BUILD.to_string(), - ComposeService { - limits: cluster.services.hub.limits.resolve(), - profiles: server_profiles.clone(), - ..hub_build_service(workspace_rel) - }, - )); - vec![("hub".to_string(), format!("service:{HUB_IMAGE_BUILD}"))] - }; - let (hub, secrets) = scenario_hub_service(cluster, &output_abs, hub_contexts, server_profiles)?; - services.push(("ws-server".to_string(), hub)); + if shared { + services.extend(shared_services(cluster, &server_profiles)); + } + let (hub_services, secrets) = hub_services(cluster, &output_abs, workspace_rel, published)?; + services.extend(hub_services); let runners = resolve_cluster_runners( &module_registry( &workspace_root, @@ -212,18 +349,24 @@ pub fn generate_docker_compose_deployment(cluster: &ClusterInput, output_dir: &P cluster, )?; services.extend(runner_services( + cluster, &runners, - &cluster.cluster_name, &relative_path_from(&output_abs, &workspace_root), artifacts, )); services.extend(scheduler_services(&runners, &cluster.cluster_name)); + let volumes = if shared { + vec![ + format!("{BACKPLANE_TASK}-data"), + "openobserve-data".to_string(), + format!("{SHARED_STORAGE_TASK}-data"), + ] + } else { + vec!["openobserve-data".to_string(), "ws-server-storage".to_string()] + }; let compose = ComposeFile { services, - volumes: vec![ - ("openobserve-data".to_string(), ComposeVolume), - ("ws-server-storage".to_string(), ComposeVolume), - ], + volumes: volumes.into_iter().map(|name| (name, ComposeVolume)).collect(), secrets, }; let content = render_compose_yaml(&compose); @@ -249,21 +392,22 @@ pub fn generate_docker_compose_deployment(cluster: &ClusterInput, output_dir: &P /// its container is named, labelled with the scenario, and never restarted by compose, since its next tick is its /// retry. fn runner_services( + cluster: &ClusterInput, runners: &[RunnerInstance], - cluster_name: &str, context: &str, artifacts: ArtifactSource, ) -> Vec<(String, ComposeService)> { let published = matches!(artifacts, ArtifactSource::Published); + let cluster_name = cluster.cluster_name.as_str(); runners .iter() .map(|runner| { let kind = &runner.runner; - // A runner on another machine cannot be gated on a hub its own compose project does not start, so the edge - // is kept but waived, and the runner is restarted until the hub answers instead. + // A runner on a machine without a hub cannot be gated on a hub its own compose project does not start, so + // the edge is kept but waived, and the runner is restarted until the hub answers instead. let remote = runner.site.is_remote(); let scheduled = !runner.schedule.is_unset(); - let (name, mut hub) = healthy("ws-server"); + let (name, mut hub) = healthy(&runner.site.hub_name(cluster)); hub.required = !remote; let service = ComposeService { container_name: scheduled.then(|| scheduled_container(cluster_name, &runner.name)), @@ -397,18 +541,7 @@ fn collector_environment() -> Vec<(String, ComposeValue)> { fn openobserve_service() -> ComposeService { ComposeService { image: Some("openobserve/openobserve:v1.1.1".to_string()), - healthcheck: Some(ComposeHealthcheck { - test: vec![ - "CMD".to_string(), - "/openobserve".to_string(), - "node".to_string(), - "status".to_string(), - ], - interval: "5s".to_string(), - timeout: "3s".to_string(), - retries: 20, - start_period: "10s".to_string(), - }), + healthcheck: Some(healthcheck(&["CMD", "/openobserve", "node", "status"])), // Bound to loopback, not every interface. A collector published on 0.0.0.0 would hand its root login to anyone // who can reach the host and read the scenario's generated env file. Nothing outside the developer's machine // needs to talk to it: the ws-server exports to 127.0.0.1:5080 and the UI is opened locally. @@ -436,6 +569,7 @@ fn hub_environment( module_paths: Vec, workspace_root: &Path, serves_a_page: bool, + shared: bool, ) -> Vec<(String, ComposeValue)> { let root_module = if serves_a_page { super::hub_root_module(&workspace_root.join("services/ws-server")) @@ -458,11 +592,14 @@ fn hub_environment( "OTLP_COLLECTOR_URL".to_string(), ComposeValue::Plain("http://127.0.0.1:5080/api/default/v1".to_string()), ), - ( + ]); + // Hubs that share storage are pointed at it by the caller instead. + if !shared { + environment.push(( "STORAGE_URL".to_string(), ComposeValue::Plain("file:///app/storage".to_string()), - ), - ]); + )); + } environment } @@ -483,9 +620,12 @@ pub fn docker_image_module_paths( resolve_module_paths(®istry, &names, |entry| entry.docker_path.clone()) } +/// One service of the file, under the name it is written as. +type NamedService = (String, ComposeService); + #[derive(Debug, Default)] struct ComposeFile { - services: Vec<(String, ComposeService)>, + services: Vec, volumes: Vec<(String, ComposeVolume)>, /// Build secrets, by name; each is the registry token, read from the host's environment. secrets: Vec, @@ -499,6 +639,8 @@ struct ComposeService { image: Option, /// A fixed container name, for a container something outside compose has to find by name. container_name: Option, + /// The command replacing the image's `ENTRYPOINT`. + entrypoint: Vec, /// Arguments replacing the image's `CMD`. command: Vec, /// Container labels, in the order they are written; every value is quoted. @@ -596,13 +738,20 @@ impl ComposeRenderer { } /// Write `key:` and then each of `items` as a `- item` line one level deeper, or nothing when there are none. + /// + /// An item YAML would read as something other than a string is quoted, since compose wants strings here. fn push_list(&mut self, indent: usize, key: &str, items: &[String]) { if items.is_empty() { return; } self.push_line(indent, &format!("{key}:")); for item in items { - self.push_line(indent.saturating_add(1), &format!("- {item}")); + let rendered = if reads_as_non_string(item) { + format!("\"{item}\"") + } else { + item.clone() + }; + self.push_line(indent.saturating_add(1), &format!("- {rendered}")); } } @@ -639,6 +788,7 @@ impl ComposeRenderer { if let Some(container_name) = &service.container_name { self.push_line(2, &format!("container_name: {container_name}")); } + self.push_list(2, "entrypoint", &service.entrypoint); self.push_list(2, "command", &service.command); if !service.labels.is_empty() { self.push_line(2, "labels:"); @@ -747,8 +897,10 @@ impl ComposeRenderer { // entry as "take this from the host environment" rather than as a value. The variable would then be absent // from the container and the image's own default would stand -- the opposite of what stating it empty is // for. + // + // So is one YAML would read as a boolean or a number, which compose rejects as an environment value. ComposeValue::Plain(value) => { - let rendered = if value.is_empty() || value.starts_with(YAML_INDICATORS) { + let rendered = if value.is_empty() || value.starts_with(YAML_INDICATORS) || reads_as_non_string(value) { format!("\"{value}\"") } else { value.clone() diff --git a/utilities/cli/src/deployment_types/hub_sharing.rs b/utilities/cli/src/deployment_types/hub_sharing.rs new file mode 100644 index 00000000..a689c6e0 --- /dev/null +++ b/utilities/cli/src/deployment_types/hub_sharing.rs @@ -0,0 +1,129 @@ +//! What several hubs share, which every deployment format runs on the server once a cluster has more than one hub. +//! +//! NATS carries the agent directory, the mailbox and delivery between hubs, and rustfs is the S3 object store every +//! hub's agent storage points at. Both run from their upstream images in every format, mise included: rustfs's pinned +//! release cannot initialise its storage on Windows, and a container is the one way it runs the same everywhere. + +use edge_toolkit::ports::Services; + +use super::mise::COLLECTOR_TASK; + +/// The task, service and `Deployment` that runs NATS. +pub(crate) const BACKPLANE_TASK: &str = "nats"; + +/// The task, service and `Deployment` that runs rustfs. +pub(crate) const SHARED_STORAGE_TASK: &str = "rustfs"; + +/// The NATS image, at the release the repository's own `nats-server` tool is pinned to. +pub(crate) const BACKPLANE_IMAGE: &str = "nats:2.15.1-alpine"; + +/// The rustfs image, at the release the repository's own `rustfs` tool is pinned to. +pub(crate) const SHARED_STORAGE_IMAGE: &str = "rustfs/rustfs:1.0.0-beta.12"; + +/// Where each of the two keeps its data inside its container, which is what its volume is mounted over. +pub(crate) const DATA_DIR: &str = "/data"; + +/// The bucket every hub's agent storage is kept in, keyed `/` as the hub's local store is. +const BUCKET: &str = "et-storage"; + +/// The account the hubs present to rustfs: not a secret, so stated wherever it is needed, like the collector's. +pub(crate) const STORAGE_ACCOUNT: &str = "et-storage"; + +/// The region the S3 client signs requests for; rustfs accepts any, and the client requires one. +const REGION: &str = "us-east-1"; + +/// The variables the scenario's env file carries the storage secret in: the hubs' S3 client's, then rustfs's. +pub(crate) const STORAGE_SECRET_ENVS: [&str; 2] = ["AWS_SECRET_ACCESS_KEY", "RUSTFS_SECRET_KEY"]; + +/// The path NATS answers healthy on once `JetStream` is up, which is what a hub needs from it. +pub(crate) const BACKPLANE_HEALTH_PATH: &str = "/healthz?js-enabled-only=true"; + +/// The path rustfs answers healthy on. +pub(crate) const SHARED_STORAGE_HEALTH_PATH: &str = "/health"; + +/// Where the NATS image's own `CMD` reads its configuration file from. +pub(crate) const BACKPLANE_CONFIG_DIR: &str = "/etc/nats"; + +/// The NATS configuration file's name, under [`BACKPLANE_CONFIG_DIR`]. +pub(crate) const BACKPLANE_CONFIG_FILE: &str = "nats-server.conf"; + +/// Each container the server runs as a task, with the port and path its readiness check asks. +/// +/// The collector always, and what the hubs share when `shared`: the tasks the server's hub waits on. +pub(crate) fn server_containers(shared: bool) -> Vec<(&'static str, u16, &'static str)> { + let mut containers = vec![(COLLECTOR_TASK, Services::OtlpCollector.port(), "/healthz")]; + if shared { + containers.extend([ + (BACKPLANE_TASK, Services::BackplaneMonitor.port(), BACKPLANE_HEALTH_PATH), + ( + SHARED_STORAGE_TASK, + Services::SharedStorage.port(), + SHARED_STORAGE_HEALTH_PATH, + ), + ]); + } + containers +} + +/// The NATS server's arguments, for the formats that start it from the command line. +/// +/// `JetStream`, which the backplane needs, kept in the data volume, and the monitoring port its health check is served +/// on. Every one is a flag, so the image's entrypoint runs them as `nats-server` arguments rather than its default +/// config file. They say what [`backplane_config`] does. +pub(crate) fn backplane_args() -> Vec { + vec![ + "--jetstream".to_string(), + "--store_dir".to_string(), + DATA_DIR.to_string(), + "--http_port".to_string(), + Services::BackplaneMonitor.port().to_string(), + ] +} + +/// The NATS server's configuration file, for k3s, which runs the image's own `CMD` over it rather than arguments. +/// +/// It says what [`backplane_args`] does: the client port, the monitoring port and `JetStream` kept in the data volume. +pub(crate) fn backplane_config() -> String { + format!( + "port: {client}\nmonitor_port: {monitor}\njetstream {{\n store_dir: {DATA_DIR}\n}}\n", + client = Services::Backplane.port(), + monitor = Services::BackplaneMonitor.port(), + ) +} + +/// The shell command rustfs is started with: create the bucket, then run the image's own entrypoint. +/// +/// rustfs creates no bucket on a write to one it does not have, but adopts a directory already in its volume as a +/// bucket. Making it here, before the server starts, is what lets the hubs write without anything else running first. +pub(crate) fn shared_storage_start() -> String { + format!("mkdir -p {DATA_DIR}/{BUCKET} && exec /entrypoint.sh rustfs") +} + +/// rustfs's own settings, besides the secret the env file carries. +pub(crate) fn shared_storage_env() -> Vec<(&'static str, String)> { + vec![ + ("RUSTFS_ACCESS_KEY", STORAGE_ACCOUNT.to_string()), + ("RUSTFS_VOLUMES", DATA_DIR.to_string()), + ] +} + +/// The environment that joins a hub to the others: NATS at `backplane_host`, rustfs at `storage_host`. +/// +/// The storage secret is not among them; it arrives from the env file, beside the collector credential. +/// `AWS_ALLOW_HTTP` because rustfs is reached in plaintext, as everything between these machines is. +pub(crate) fn shared_hub_env(backplane_host: &str, storage_host: &str) -> Vec<(&'static str, String)> { + vec![ + ("AWS_ACCESS_KEY_ID", STORAGE_ACCOUNT.to_string()), + ("AWS_ALLOW_HTTP", "true".to_string()), + ("AWS_DEFAULT_REGION", REGION.to_string()), + ( + "AWS_ENDPOINT", + format!("http://{storage_host}:{}", Services::SharedStorage.port()), + ), + ("STORAGE_URL", format!("s3://{BUCKET}")), + ( + "WS_BACKPLANE", + format!("nats://{backplane_host}:{}", Services::Backplane.port()), + ), + ] +} diff --git a/utilities/cli/src/deployment_types/k3s.rs b/utilities/cli/src/deployment_types/k3s.rs index eb3240af..54607891 100644 --- a/utilities/cli/src/deployment_types/k3s.rs +++ b/utilities/cli/src/deployment_types/k3s.rs @@ -42,15 +42,20 @@ use k8s_openapi::apimachinery::pkg::apis::meta::v1::{LabelSelector, ObjectMeta}; use k8s_openapi::apimachinery::pkg::util::intstr::IntOrString; use super::document; +use super::hub_sharing::{ + BACKPLANE_CONFIG_DIR, BACKPLANE_CONFIG_FILE, BACKPLANE_HEALTH_PATH, BACKPLANE_IMAGE, BACKPLANE_TASK, DATA_DIR, + SHARED_STORAGE_HEALTH_PATH, SHARED_STORAGE_IMAGE, SHARED_STORAGE_TASK, backplane_config, shared_hub_env, + shared_storage_env, shared_storage_start, +}; use super::k3s_recovery; use crate::error::CliError; use crate::input::{ArtifactSource, ClusterInput}; use crate::limits::{CpuCap, Limits, MAX_CPU_SHARES, ResolvedLimits, k8s_bytes, k8s_cpus}; -use crate::placement::{MACHINE_LABEL, Site, hub_site}; +use crate::placement::{Hub, MACHINE_LABEL, Site, hub_site, hubs, shares_hubs}; use crate::schedule::Schedule; use crate::{ - COLLECTOR_SETTINGS, COLLECTOR_USERNAME, HUB_SERVICE, OutputType, RunnerInstance, cluster_module_names, - docker_image_module_paths, hub_service_ws_url, image_prefix, module_registry, resolve_cluster_runners, + COLLECTOR_SETTINGS, COLLECTOR_USERNAME, OutputType, RunnerInstance, cluster_module_names, + docker_image_module_paths, hub_service_ws_url_named, image_prefix, module_registry, resolve_cluster_runners, }; /// Pull policy that lets an image built from this repository and imported into the node satisfy a manifest. @@ -76,6 +81,9 @@ const OPENOBSERVE_IMAGE: &str = "openobserve/openobserve:v1.1.1"; /// it should be able to reach. The hub's equivalent lives beside the URL builder that needs it. const COLLECTOR_SERVICE: &str = "openobserve"; +/// The `ConfigMap` holding the NATS server's configuration file, and the volume it is mounted from. +const BACKPLANE_CONFIG_MAP: &str = "nats-config"; + /// How long a probe waits, and how many misses it tolerates, mirroring `compose.yaml`'s healthchecks. const PROBE_PERIOD_SECONDS: i32 = 5; const PROBE_FAILURE_THRESHOLD: i32 = 20; @@ -217,9 +225,11 @@ pub fn generate_k3s_deployment(cluster: &ClusterInput, output_dir: &Path) -> Res let mut docs = fixed_documents(&namespace, cluster, &module_paths, &root_module, &hub_site(cluster))?; for mut runner in runners { let supervised = k3s_recovery::configure_runner(cluster, &mut runner); + let hub = runner.site.hub_name(cluster); docs.push(runner_document( &namespace, &runner, + &hub, cluster.artifact_source, supervised, )?); @@ -242,8 +252,10 @@ pub fn generate_k3s_deployment(cluster: &ClusterInput, output_dir: &Path) -> Res /// and eight of them in one function put it past the repo's cyclomatic-complexity ceiling without saying anything about /// how the deployment is shaped. /// -/// `site` places both the collector and the hub. Each mounts a claim, and k3s provisions a claim on the node that first -/// mounts it, so the two stay on the server machine rather than going wherever the scheduler finds room. +/// `site` places the collector, the one hub of a cluster that has one, and what several hubs share. Each mounts a +/// claim, and k3s provisions a claim on the node that first mounts it, so they stay on the server machine rather than +/// going wherever the scheduler finds room. Hubs that share storage mount no claim of their own, so each runs on its +/// machine. fn fixed_documents( namespace: &str, cluster: &ClusterInput, @@ -251,7 +263,8 @@ fn fixed_documents( root_module: &str, site: &Site, ) -> Result, CliError> { - Ok(vec![ + let shared = shares_hubs(cluster); + let mut docs = vec![ document(&namespace_object(namespace))?, document(&collector_config(namespace))?, document(&claim(namespace, "openobserve-data"))?, @@ -261,19 +274,160 @@ fn fixed_documents( &cluster.services.collector.limits, ))?, document(&collector_service(namespace))?, - document(&claim(namespace, "ws-server-storage"))?, - document(&hub_deployment( + ]; + if shared { + docs.extend(shared_documents(namespace, cluster, site)?); + } else { + docs.push(document(&claim(namespace, "ws-server-storage"))?); + } + let hub_image = HubImage { + cluster_name: &cluster.cluster_name, + module_paths, + root_module, + shared, + }; + for hub in hubs(cluster) { + docs.push(document(&hub_deployment( namespace, - &cluster.cluster_name, - module_paths, - root_module, - site, + &hub_image, + &hub, &cluster.services.hub.limits, + ))?); + docs.push(document(&hub_service(namespace, &hub.name))?); + } + Ok(docs) +} + +/// The NATS and rustfs workloads several hubs share, each with its claim and its `Service`, at `site`. +#[expect(clippy::single_call_fn, reason = "the shared-services step of fixed_documents")] +fn shared_documents(namespace: &str, cluster: &ClusterInput, site: &Site) -> Result, CliError> { + let backplane_data = format!("{BACKPLANE_TASK}-data"); + let storage_data = format!("{SHARED_STORAGE_TASK}-data"); + Ok(vec![ + document(&ConfigMap { + data: Some(BTreeMap::from([( + BACKPLANE_CONFIG_FILE.to_string(), + backplane_config(), + )])), + metadata: plain_meta(namespace, BACKPLANE_CONFIG_MAP), + ..ConfigMap::default() + })?, + document(&claim(namespace, &backplane_data))?, + document(&backplane_deployment( + namespace, + &backplane_data, + site, + &cluster.services.backplane.limits, + ))?, + document(&service( + namespace, + BACKPLANE_TASK, + vec![ + port("client", Services::Backplane.port()), + port("monitor", Services::BackplaneMonitor.port()), + ], + ))?, + document(&claim(namespace, &storage_data))?, + document(&shared_storage_deployment( + namespace, + &storage_data, + site, + &cluster.services.storage.limits, + ))?, + document(&service( + namespace, + SHARED_STORAGE_TASK, + vec![port("s3", Services::SharedStorage.port())], ))?, - document(&hub_service(namespace))?, ]) } +/// A readiness probe asking `path` on `port` over HTTP. +fn http_probe(port: u16, path: &str) -> Probe { + probe( + None, + Some(HTTPGetAction { + path: Some(path.to_string()), + port: IntOrString::Int(i32::from(port)), + ..HTTPGetAction::default() + }), + ) +} + +/// The NATS server the hubs share, keeping its `JetStream` store on the claim `data`. +/// +/// It runs the image's own entrypoint and `CMD`, which read the configuration file the `ConfigMap` puts where the +/// image looks for it, since NATS reads nothing it needs from its environment. +fn backplane_deployment(namespace: &str, data: &str, site: &Site, limits: &Limits) -> Deployment { + let (data_mount, data_volume) = mount(data, DATA_DIR); + let config_volume = Volume { + config_map: Some(ConfigMapVolumeSource { + name: BACKPLANE_CONFIG_MAP.to_string(), + ..ConfigMapVolumeSource::default() + }), + name: BACKPLANE_CONFIG_MAP.to_string(), + ..Volume::default() + }; + let config_mount = volume_mount(BACKPLANE_CONFIG_MAP, BACKPLANE_CONFIG_DIR); + let container = Container { + image: Some(BACKPLANE_IMAGE.to_string()), + image_pull_policy: Some(IMAGE_PULL_POLICY.to_string()), + name: BACKPLANE_TASK.to_string(), + ports: Some(vec![ + container_port(Services::Backplane.port()), + container_port(Services::BackplaneMonitor.port()), + ]), + readiness_probe: Some(http_probe(Services::BackplaneMonitor.port(), BACKPLANE_HEALTH_PATH)), + security_context: Some(hardened_container_context()), + volume_mounts: Some(vec![data_mount, config_mount]), + ..Container::default() + }; + deployment( + namespace, + BACKPLANE_TASK, + container, + vec![data_volume, config_volume], + site, + limits, + ) +} + +/// The object store the hubs share, keeping its objects on the claim `data` and reading its secret from the `Secret`. +/// +/// It also writes logs and temporary files, which a read-only root would refuse, so both get a scratch directory. +fn shared_storage_deployment(namespace: &str, data: &str, site: &Site, limits: &Limits) -> Deployment { + let (data_mount, data_volume) = mount(data, DATA_DIR); + let (logs_mount, logs_volume) = scratch("rustfs-logs", "/logs"); + // skipcq: RS-S1003 + let (tmp_mount, tmp_volume) = scratch("rustfs-tmp", "/tmp"); + let container = Container { + command: Some(vec!["sh".to_string(), "-c".to_string(), shared_storage_start()]), + env: Some( + shared_storage_env() + .into_iter() + .map(|(name, value)| env(name, value)) + .collect(), + ), + env_from: Some(vec![from_secret(namespace)]), + image: Some(SHARED_STORAGE_IMAGE.to_string()), + image_pull_policy: Some(IMAGE_PULL_POLICY.to_string()), + name: SHARED_STORAGE_TASK.to_string(), + ports: Some(vec![container_port(Services::SharedStorage.port())]), + readiness_probe: Some(http_probe(Services::SharedStorage.port(), SHARED_STORAGE_HEALTH_PATH)), + security_context: Some(hardened_container_context()), + volume_mounts: Some(vec![data_mount, logs_mount, tmp_mount]), + ..Container::default() + }; + deployment( + namespace, + SHARED_STORAGE_TASK, + container, + vec![data_volume, logs_volume, tmp_volume], + site, + limits, + ) +} + /// Namespace a scenario's objects live in, so two scenarios can be applied to one cluster at once. fn namespace_name(cluster_name: &str) -> String { format!("et-{cluster_name}") @@ -574,12 +728,17 @@ fn cluster_plugins(cluster: &ClusterInput) -> Vec<&'static RuntimePlugin> { crate::recovery::Recovery::Configured(settings) => vec![&settings.limits], crate::recovery::Recovery::Disabled => Vec::new(), }; - let mut needed: Vec<&RuntimePlugin> = [&services.collector.limits, &services.hub.limits] - .into_iter() - .chain(agents) - .chain(recovery_limits) - .flat_map(|limits| required_plugins(&limits.resolve())) - .collect(); + let mut needed: Vec<&RuntimePlugin> = [ + &services.backplane.limits, + &services.collector.limits, + &services.hub.limits, + &services.storage.limits, + ] + .into_iter() + .chain(agents) + .chain(recovery_limits) + .flat_map(|limits| required_plugins(&limits.resolve())) + .collect(); needed.sort_unstable_by_key(|plugin| plugin.name); needed.dedup_by_key(|plugin| plugin.name); needed @@ -964,11 +1123,30 @@ fn collector_service(namespace: &str) -> Service { ) } +/// What every hub of a scenario runs alike: its image and module set, and whether it shares storage with others. +struct HubImage<'scenario> { + /// The scenario, which names the image every hub runs. + cluster_name: &'scenario str, + /// The modules the image serves, as `MODULES_PATHS` lists them. + module_paths: &'scenario [String], + /// The module served at `/`, empty for a headless cluster. + root_module: &'scenario str, + /// Whether the hubs share NATS and the object store, which replaces each one's own storage claim. + shared: bool, +} + /// The hub's environment, everything it needs that is not the credential the `Secret` carries. /// /// The paths under the runtime directory are the writable ones: a read-only root filesystem cannot take the self-signed -/// certificate the hub generates on first start, so both halves are redirected to the scratch mount. -fn hub_env(module_paths: &[String], root_module: &str) -> Vec { +/// certificate the hub generates on first start, so both halves are redirected to the scratch mount. Hubs that share +/// storage reach NATS and the object store by their `Service` names, instead of keeping a store of their own. +fn hub_env(image: &HubImage<'_>) -> Vec { + let HubImage { + module_paths, + root_module, + shared, + .. + } = *image; // `MODULES_ROOT` is stated even when it is empty, which is what a headless cluster needs rather than the entry // being left out. The hub image names the page it bundles, and a container inherits an image's environment for // every variable the manifest does not set -- so omitting it would serve a page this cluster was not given the @@ -988,63 +1166,59 @@ fn hub_env(module_paths: &[String], root_module: &str) -> Vec { Services::OtlpCollector.port() ), ), - env("STORAGE_URL", "file:///app/storage".to_string()), env("TLS_CERT_FILE", format!("{HUB_RUNTIME_DIR}/cert.pem")), env("TLS_KEY_FILE", format!("{HUB_RUNTIME_DIR}/key.pem")), env("WS_AGENT_REGISTRY", format!("{HUB_RUNTIME_DIR}/registry.yaml")), ]); + if shared { + vars.extend( + shared_hub_env(BACKPLANE_TASK, SHARED_STORAGE_TASK) + .into_iter() + .map(|(name, value)| env(name, value)), + ); + } else { + vars.push(env("STORAGE_URL", "file:///app/storage".to_string())); + } + vars.sort_by(|left, right| left.name.cmp(&right.name)); vars } -/// The hub, running this scenario's image because that is what carries its module set. -fn hub_deployment( - namespace: &str, - cluster_name: &str, - module_paths: &[String], - root_module: &str, - site: &Site, - limits: &Limits, -) -> Deployment { +/// One hub, running this scenario's image because that is what carries its module set. +fn hub_deployment(namespace: &str, image: &HubImage<'_>, hub: &Hub, limits: &Limits) -> Deployment { let insecure = Services::InsecureWebSocketServer.port(); - let (storage_mount, storage_volume) = mount("ws-server-storage", "/app/storage"); let (runtime_mount, runtime_volume) = scratch("ws-server-runtime", HUB_RUNTIME_DIR); + let mut mounts = Vec::new(); + let mut volumes = Vec::new(); + if !image.shared { + let (storage_mount, storage_volume) = mount("ws-server-storage", "/app/storage"); + mounts.push(storage_mount); + volumes.push(storage_volume); + } + mounts.push(runtime_mount); + volumes.push(runtime_volume); let container = Container { - env: Some(hub_env(module_paths, root_module)), + env: Some(hub_env(image)), env_from: Some(vec![from_secret(namespace)]), - image: Some(format!("et-ws-server-{cluster_name}:{IMAGE_TAG}")), + image: Some(format!("et-ws-server-{}:{IMAGE_TAG}", image.cluster_name)), image_pull_policy: Some(IMAGE_PULL_POLICY.to_string()), - name: HUB_SERVICE.to_string(), + name: hub.name.clone(), ports: Some(vec![ container_port(insecure), container_port(Services::SecureWebSocketServer.port()), ]), - readiness_probe: Some(probe( - None, - Some(HTTPGetAction { - path: Some("/health".to_string()), - port: IntOrString::Int(i32::from(insecure)), - ..HTTPGetAction::default() - }), - )), + readiness_probe: Some(http_probe(insecure, "/health")), security_context: Some(hardened_container_context()), - volume_mounts: Some(vec![storage_mount, runtime_mount]), + volume_mounts: Some(mounts), ..Container::default() }; - deployment( - namespace, - HUB_SERVICE, - container, - vec![storage_volume, runtime_volume], - site, - limits, - ) + deployment(namespace, &hub.name, container, volumes, &hub.site, limits) } -/// Hub service, carrying both the insecure and secure ports the server listens on. -fn hub_service(namespace: &str) -> Service { +/// The service fronting the hub `name`, carrying both the insecure and secure ports the server listens on. +fn hub_service(namespace: &str, name: &str) -> Service { service( namespace, - HUB_SERVICE, + name, vec![ port("ws", Services::InsecureWebSocketServer.port()), port("wss", Services::SecureWebSocketServer.port()), @@ -1052,13 +1226,13 @@ fn hub_service(namespace: &str) -> Service { ) } -/// What wires a runner to its module and its hub, plus whatever the scenario declared for that agent. +/// What wires a runner to its module and to `hub`, plus whatever the scenario declared for that agent. /// /// The scenario's entries cannot collide with the derived two: the resolver rejects an `env:` naming either. -fn runner_env(runner: &RunnerInstance) -> Vec { +fn runner_env(runner: &RunnerInstance, hub: &str) -> Vec { let mut vars = vec![ env("RUNNER_MODULE", runner.module.clone()), - env("WS_SERVER_URL", hub_service_ws_url()), + env("WS_SERVER_URL", hub_service_ws_url_named(hub)), ]; vars.extend(runner.env.iter().map(|(name, value)| env(name, value.clone()))); vars @@ -1070,6 +1244,7 @@ fn runner_env(runner: &RunnerInstance) -> Vec { fn runner_document( namespace: &str, runner: &RunnerInstance, + hub: &str, images: ArtifactSource, supervised: bool, ) -> Result { @@ -1077,7 +1252,7 @@ fn runner_document( // root needs somewhere writable even though nothing is meant to persist. skipcq: RS-S1003 let (runtime_mount, runtime_volume) = scratch("runner-tmp", "/tmp"); let container = Container { - env: Some(runner_env(runner)), + env: Some(runner_env(runner, hub)), image: Some(format!( "{}et-ws-{}-runner:{IMAGE_TAG}", image_prefix(images), diff --git a/utilities/cli/src/deployment_types/mise.rs b/utilities/cli/src/deployment_types/mise.rs index e0cde8a9..6125bf76 100644 --- a/utilities/cli/src/deployment_types/mise.rs +++ b/utilities/cli/src/deployment_types/mise.rs @@ -2,13 +2,19 @@ use std::collections::BTreeMap; use std::fmt::Write as _; use std::path::{Path, PathBuf}; +use edge_toolkit::ports::Services; use et_path::{absolute_from, relative_path_from}; use fs_err as fs; use toml::{Table, Value}; +use super::hub_sharing::{ + BACKPLANE_IMAGE, BACKPLANE_TASK, DATA_DIR, SHARED_STORAGE_IMAGE, SHARED_STORAGE_TASK, backplane_args, + shared_hub_env, shared_storage_env, shared_storage_start, +}; use super::process_compose::PROCESS_COMPOSE_TOOL; use crate::error::CliError; -use crate::input::{ArtifactSource, ClusterInput, Machine}; +use crate::input::{ArtifactSource, ClusterInput}; +use crate::placement::{hubs, shares_hubs}; use crate::{ COLLECTOR_SETTINGS, COLLECTOR_USERNAME, MODULE_SCOPE, ModuleRegistryEntry, ModuleSource, RunnerInstance, cluster_module_names, escape_for_double_quotes, module_registry, resolve_cluster_modules, resolve_module_paths, @@ -65,7 +71,21 @@ pub(crate) fn machine_task(machine: &str) -> String { /// Scoped to the scenario so a stop can only ever reach this scenario's collector: a bare `openobserve` is also the /// name this repository's own `o2` task runs its collector under, and any other scenario's on the same Docker daemon. pub(crate) fn collector_container(cluster_name: &str) -> String { - format!("openobserve-{cluster_name}") + container(COLLECTOR_TASK, cluster_name) +} + +/// Name the task `task` gives the container it runs, scoped to the scenario as [`collector_container`] explains. +pub(crate) fn container(task: &str, cluster_name: &str) -> String { + format!("{task}-{cluster_name}") +} + +/// The tasks the server runs besides its runners: the collector, its hub, and what the hubs share when there are more. +fn server_tasks(cluster: &ClusterInput) -> Vec { + let mut tasks = vec![COLLECTOR_TASK.to_string(), HUB_TASK.to_string()]; + if shares_hubs(cluster) { + tasks.extend([BACKPLANE_TASK.to_string(), SHARED_STORAGE_TASK.to_string()]); + } + tasks } /// The task that runs one runner agent, from the checkout root for a local scenario. @@ -82,11 +102,14 @@ fn runner_task(runner: &RunnerInstance, workspace_rel: &str, artifacts: Artifact } /// Add the tasks a person runs: `generated-scenario`, which starts every role together, and the UI opener. +/// +/// `generated-scenario` starts the server's hub alone, never another machine's: every hub listens on the same ports, so +/// on one host a second could not start, and each runner reaches the one there is at `localhost` anyway. #[expect(clippy::single_call_fn, reason = "the entry-point step of generate_mise_deployment")] -fn insert_entry_tasks(tasks: &mut Table, cluster_name: &str, machines: &[Machine], runners: &[RunnerInstance]) { - let mut scenario_depends = vec![COLLECTOR_TASK.to_string(), HUB_TASK.to_string()]; +fn insert_entry_tasks(tasks: &mut Table, cluster: &ClusterInput, runners: &[RunnerInstance]) { + let mut scenario_depends = server_tasks(cluster); scenario_depends.extend(runners.iter().map(|runner| runner.name.clone())); - let description = format!("Run generated scenario for {cluster_name}"); + let description = format!("Run generated scenario for {}", cluster.cluster_name); let scenario = mise_task( None, Some(&description), @@ -96,7 +119,7 @@ fn insert_entry_tasks(tasks: &mut Table, cluster_name: &str, machines: &[Machine None, ); let _previous: Option = tasks.insert("generated-scenario".to_string(), Value::Table(scenario)); - insert_machine_tasks(tasks, machines, runners); + insert_machine_tasks(tasks, cluster, runners); let opener = mise_task( None, Some("Open the OpenObserve UI"), @@ -110,15 +133,20 @@ fn insert_entry_tasks(tasks: &mut Table, cluster_name: &str, machines: &[Machine /// Add one `machine-` task per machine, each starting what that machine runs and nothing else. /// -/// The server's also starts the collector and the hub. `generated-scenario` stays alongside them, still starting -/// everything, because the whole cluster on one host remains the quickest way to run it. +/// The server's also starts the collector and the hub, and any other machine running a hub starts its own. +/// `generated-scenario` stays alongside them, still starting everything, because the whole cluster on one host remains +/// the quickest way to run it. #[expect(clippy::single_call_fn, reason = "the per-machine step of insert_entry_tasks")] -fn insert_machine_tasks(tasks: &mut Table, machines: &[Machine], runners: &[RunnerInstance]) { - for (index, machine) in machines.iter().enumerate() { +fn insert_machine_tasks(tasks: &mut Table, cluster: &ClusterInput, runners: &[RunnerInstance]) { + let hubs = hubs(cluster); + for (index, machine) in cluster.machines.iter().enumerate() { let mut depends = if index == 0 { - vec![COLLECTOR_TASK.to_string(), HUB_TASK.to_string()] + server_tasks(cluster) } else { - Vec::new() + hubs.iter() + .filter(|hub| hub.site.machine_names().contains(&machine.name)) + .map(|hub| hub.name.clone()) + .collect() }; depends.extend( runners @@ -168,7 +196,7 @@ pub fn generate_mise_deployment( let _previous: Option = tasks.insert(runner.name.clone(), Value::Table(task)); } - insert_entry_tasks(&mut tasks, &cluster.cluster_name, &cluster.machines, &runners); + insert_entry_tasks(&mut tasks, cluster, &runners); let _previous: Option = root.insert("env".to_string(), Value::Table(mise_env(artifacts))); let _previous: Option = root.insert("tasks".to_string(), Value::Table(tasks)); @@ -207,20 +235,95 @@ fn service_tasks( ); // A published hub is handed absolute paths that `mise where` resolves, so it needs no directory of its own; a local // one is given paths relative to the server's source dir and has to start there. - let _previous: Option = tasks.insert( - HUB_TASK.to_string(), - Value::Table(mise_task( + let shared = shares_hubs(cluster); + for hub in hubs(cluster) { + let description = match hub.site.machine_names().as_slice() { + [machine] if !hub.on_server() => format!("Run the WebSocket server on {machine}"), + _ => "Run the WebSocket server".to_string(), + }; + let mut env = ws_server_env(scenario, artifacts, shared); + if shared { + for (name, value) in shared_hub_env(&hub.server_host, &hub.server_host) { + let _previous: Option = env.insert(name.to_string(), Value::String(value)); + } + } + let task = mise_task( None, - Some("Run the WebSocket server"), + Some(&description), workspace_dir(&ws_server_rel, artifacts), Some(&ws_server_run), None, - Some(ws_server_env(scenario, artifacts)), - )), - ); + Some(env), + ); + let _previous: Option = tasks.insert(hub.name, Value::Table(task)); + } + if shared { + let backplane = backplane_run_body(&cluster.cluster_name); + let storage = shared_storage_run_body(&cluster.cluster_name); + for (name, description, run) in [ + (BACKPLANE_TASK, "Run the NATS server the hubs share", backplane), + (SHARED_STORAGE_TASK, "Run the object store the hubs share", storage), + ] { + let task = mise_task(None, Some(description), None, Some(&run), None, None); + let _previous: Option = tasks.insert(name.to_string(), Value::Table(task)); + } + } Ok(tasks) } +/// The body of the task that starts the NATS container, its client port open to the other machines' hubs. +/// +/// The monitoring port stays on loopback, since only this machine's readiness checks read it. The store is a named +/// volume, so what is queued for an agent outlives a restart. +fn backplane_run_body(cluster_name: &str) -> String { + let client = Services::Backplane.port(); + let monitor = Services::BackplaneMonitor.port(); + format!( + concat!( + "image={image}\n", + "container={container}\n", + "ports=\"-p {client}:{client} -p 127.0.0.1:{monitor}:{monitor}\"\n", + "# $ports is a word-split flag list by design; do not quote it.\n", + "docker run --rm --name \"$container\" $ports -v \"$container:{data}\" \"$image\" {args}\n", + ), + args = backplane_args().join(" "), + client = client, + container = container(BACKPLANE_TASK, cluster_name), + data = DATA_DIR, + image = BACKPLANE_IMAGE, + monitor = monitor, + ) +} + +/// The body of the task that starts the rustfs container, its S3 port open to the other machines' hubs. +/// +/// `-e RUSTFS_SECRET_KEY` passes the name only, so Docker forwards the value the `[env] _.file` loaded, as the +/// collector task does with its password. +fn shared_storage_run_body(cluster_name: &str) -> String { + let settings = shared_storage_env() + .into_iter() + .map(|(name, value)| format!("-e {name}={value}")) + .collect::>() + .join(" "); + format!( + concat!( + "image={image}\n", + "container={container}\n", + "settings=\"{settings} -e RUSTFS_SECRET_KEY\"\n", + "start='{start}'\n", + "# $settings is a word-split flag list by design; do not quote it.\n", + "docker run --rm --name \"$container\" -p {port}:{port} -v \"$container:{data}\" $settings ", + "--entrypoint sh \"$image\" -c \"$start\"\n", + ), + container = container(SHARED_STORAGE_TASK, cluster_name), + data = DATA_DIR, + image = SHARED_STORAGE_IMAGE, + port = Services::SharedStorage.port(), + settings = settings, + start = shared_storage_start(), + ) +} + /// The body of the task that starts the hub. /// /// A published deployment names its modules once, as the `[tools]` that stage them, and says nothing about where they @@ -258,14 +361,14 @@ fn ws_server_run_body( /// layout (`services/ws-server/storage`) under whatever it takes for the project root -- outside this repository, /// the directory it was started in. A published hub starts in the deployment directory, so the store is named there /// directly, as the compose file does with its `/app/storage` volume. A local hub keeps the default, which is where -/// this repository's tests read agent output from. +/// this repository's tests read agent output from. Hubs that `shared` storage are pointed at it by the caller instead. #[expect( clippy::single_call_fn, reason = "distinct step of generate_mise_deployment, split out to keep it readable" )] -fn ws_server_env(scenario: &ScenarioModules<'_>, artifacts: ArtifactSource) -> Table { +fn ws_server_env(scenario: &ScenarioModules<'_>, artifacts: ArtifactSource, shared: bool) -> Table { let mut env = hub_root_env(scenario.ws_server_dir, scenario.serves_a_page); - if matches!(artifacts, ArtifactSource::Published) { + if !shared && matches!(artifacts, ArtifactSource::Published) { let _previous: Option = env.insert( "STORAGE_URL".to_string(), Value::String(format!("file://{{{{ config_root }}}}/{STORAGE_DIR}")), diff --git a/utilities/cli/src/deployment_types/mod.rs b/utilities/cli/src/deployment_types/mod.rs index b552094c..1f083231 100644 --- a/utilities/cli/src/deployment_types/mod.rs +++ b/utilities/cli/src/deployment_types/mod.rs @@ -1,6 +1,7 @@ mod dekit; mod docker_compose; mod hub; +mod hub_sharing; mod k3s; mod k3s_recovery; mod mise; @@ -12,6 +13,7 @@ pub use self::dekit::generate_dekit_deployment; pub(crate) use self::docker_compose::{HUB_IMAGE_BUILD, SCHEDULER_SERVICE, scheduler_service}; pub use self::docker_compose::{docker_image_module_paths, generate_docker_compose_deployment}; pub(crate) use self::hub::{hub_root_module, serves_a_page}; +pub(crate) use self::hub_sharing::{BACKPLANE_TASK, SHARED_STORAGE_TASK, STORAGE_SECRET_ENVS}; pub use self::k3s::generate_k3s_deployment; pub(crate) use self::k3s::{ CONTAINERD_TEMPLATE_FILE, PLUGIN_NAMESPACE as K3S_PLUGIN_NAMESPACE, needs_containerd_template, diff --git a/utilities/cli/src/deployment_types/process_compose.rs b/utilities/cli/src/deployment_types/process_compose.rs index f1de1a58..a47ab36e 100644 --- a/utilities/cli/src/deployment_types/process_compose.rs +++ b/utilities/cli/src/deployment_types/process_compose.rs @@ -4,12 +4,13 @@ use edge_toolkit::ports::Services; use fs_err as fs; use serde_yaml::{Mapping, Value}; -use super::mise::collector_container; +use super::hub_sharing::server_containers; +use super::mise::container; use super::yaml::{document, insert}; -use super::{COLLECTOR_TASK, HUB_TASK, OPENER_TASK}; +use super::{COLLECTOR_TASK, OPENER_TASK}; use crate::error::CliError; use crate::input::ClusterInput; -use crate::placement::{Site, hub_site}; +use crate::placement::{Site, hub_site, hubs, shares_hubs}; use crate::schedule::Schedule; use crate::{OutputType, scenario_runners}; @@ -22,8 +23,8 @@ const SCHEMA_VERSION: &str = "0.5"; /// SIGINT, the interrupt a mise task is written to stop on, as process-compose's numeric `signal:` takes it. const SIGINT: u8 = 2; -/// How long the collector gets to exit once stopped, matching the dekit deployment's allowance for `docker stop`. -const COLLECTOR_STOP_TIMEOUT_SECONDS: u16 = 30; +/// How long a container the deployment runs gets to exit once stopped, matching dekit's allowance for `docker stop`. +const CONTAINER_STOP_TIMEOUT_SECONDS: u16 = 30; /// Seconds between readiness checks. const READINESS_PERIOD_SECONDS: u16 = 2; @@ -49,33 +50,28 @@ pub fn generate_process_compose_deployment(cluster: &ClusterInput, output_dir: & let server = hub_site(cluster); let mut processes = Mapping::new(); - let mut collector = process(COLLECTOR_TASK, &[], &server); - insert( - &mut collector, - "readiness_probe", - http_ready(Services::OtlpCollector.port(), "/healthz"), - ); - // `docker run` is stopped through Docker rather than by signalling the client, for the reason dekit's stop gives. - if let Some(Value::Mapping(shutdown)) = collector.get_mut("shutdown") { - let container = collector_container(&cluster.cluster_name); - insert(shutdown, "command", Value::from(format!("docker stop {container}"))); - insert(shutdown, "timeout_seconds", Value::from(COLLECTOR_STOP_TIMEOUT_SECONDS)); + let containers = server_containers(shares_hubs(cluster)); + let server_hub_deps: Vec<&str> = containers.iter().map(|(name, ..)| *name).collect(); + for (name, port, path) in containers { + let mut service = process(name, &[], &server); + insert(&mut service, "readiness_probe", http_ready(port, path)); + // `docker run` is stopped through Docker rather than by signalling the client, for the reason dekit's stop + // gives. + if let Some(Value::Mapping(shutdown)) = service.get_mut("shutdown") { + let container = container(name, &cluster.cluster_name); + insert(shutdown, "command", Value::from(format!("docker stop {container}"))); + insert(shutdown, "timeout_seconds", Value::from(CONTAINER_STOP_TIMEOUT_SECONDS)); + } + insert(&mut processes, name, Value::Mapping(service)); } - insert(&mut processes, COLLECTOR_TASK, Value::Mapping(collector)); - - let mut hub = process(HUB_TASK, &[COLLECTOR_TASK], &server); - insert( - &mut hub, - "readiness_probe", - http_ready(Services::InsecureWebSocketServer.port(), "/health"), - ); - insert(&mut processes, HUB_TASK, Value::Mapping(hub)); + insert_hub_processes(&mut processes, cluster, &server_hub_deps); for runner in runners { - // A runner on another machine cannot wait on a hub process this one does not hold, so it restarts until the hub - // on the server answers instead. A scheduled one is not restarted: its next tick is its retry. + // A runner on a machine without a hub cannot wait on a hub process this one does not hold, so it restarts until + // the hub on the server answers instead. A scheduled one is not restarted: its next tick is its retry. let remote = runner.site.is_remote(); - let deps: &[&str] = if remote { &[] } else { &[HUB_TASK] }; + let hub = runner.site.hub_name(cluster); + let deps: &[&str] = if remote { &[] } else { &[&hub] }; let mut runner_process = process(&runner.name, deps, &runner.site); if runner.schedule.is_unset() { if remote { @@ -103,6 +99,29 @@ pub fn generate_process_compose_deployment(cluster: &ClusterInput, output_dir: & Ok(()) } +/// Add a process per hub, the server's waiting on `server_deps`. +/// +/// A hub off the server cannot wait on the services it shares there, so it restarts until they answer instead. +#[expect( + clippy::single_call_fn, + reason = "the per-hub step of generate_process_compose_deployment" +)] +fn insert_hub_processes(processes: &mut Mapping, cluster: &ClusterInput, server_deps: &[&str]) { + for hub in hubs(cluster) { + let deps: &[&str] = if hub.on_server() { server_deps } else { &[] }; + let mut hub_process = process(&hub.name, deps, &hub.site); + insert( + &mut hub_process, + "readiness_probe", + http_ready(Services::InsecureWebSocketServer.port(), "/health"), + ); + if !hub.on_server() { + insert(&mut hub_process, "availability", restart("on_failure")); + } + insert(processes, &hub.name, Value::Mapping(hub_process)); + } +} + /// The process that runs the mise task `name` once each of `deps` reports healthy, in the namespace of `site`. /// /// Stopped with SIGINT rather than process-compose's SIGTERM, for the reason the dekit deployment gives. diff --git a/utilities/cli/src/error.rs b/utilities/cli/src/error.rs index d7ba5092..c7128a3f 100644 --- a/utilities/cli/src/error.rs +++ b/utilities/cli/src/error.rs @@ -175,6 +175,28 @@ pub enum CliError { first: String, second: String, }, + + #[error("services.hub.machines lists {machine:?}, but the scenario has no machines: list them under `machines:`")] + HubMachinesWithoutMachines { machine: String }, + + #[error("services.hub.machines lists {machine:?}, which is not one of the scenario's machines")] + UnknownHubMachine { machine: String }, + + #[error("services.hub.machines lists {machine:?} more than once; a machine runs one hub")] + DuplicateHubMachine { machine: String }, + + #[error("Machine {machine:?} would run hub {hub:?}, which is not an RFC 1123 label: {reason}")] + InvalidHubName { + machine: String, + hub: String, + reason: String, + }, + + #[error("services.hub.machines leaves out the server {server:?}, which runs the collector and what the hubs share")] + HubMachinesWithoutServer { server: String }, + + #[error("services.{service} is set, but only several hubs run it: list them in services.hub.machines")] + SharedServiceWithoutHubs { service: String }, } /// Parse `src` as TOML into `T`, attaching `path` to the error on failure. diff --git a/utilities/cli/src/hub_ws_url.rs b/utilities/cli/src/hub_ws_url.rs index fb4d1e2a..2e7769b3 100644 --- a/utilities/cli/src/hub_ws_url.rs +++ b/utilities/cli/src/hub_ws_url.rs @@ -16,11 +16,8 @@ use edge_toolkit::ports::Services; -/// Name the in-cluster `Service` that fronts the hub. -/// -/// Duplicated from the k3s generator's own constant would be one definition too many, so the URL builder that -/// needs it owns the name and the generator names its objects from here. -pub const HUB_SERVICE: &str = "ws-server"; +/// Name the in-cluster `Service` that fronts the server's hub, which every hub's objects are named from. +const HUB_SERVICE: &str = crate::deployment_types::HUB_TASK; /// WebSocket URL for a runner sharing a host with the hub, which is how mise and compose arrange it. #[must_use] @@ -37,5 +34,11 @@ pub fn hub_ws_url_at(host: &str) -> String { /// WebSocket URL for a runner reaching the hub across a cluster, where `localhost` is the runner's own pod. #[must_use] pub fn hub_service_ws_url() -> String { - format!("ws://{HUB_SERVICE}:{}/ws", Services::InsecureWebSocketServer.port()) + hub_service_ws_url_named(HUB_SERVICE) +} + +/// WebSocket URL for a runner reaching the hub that `service` fronts, one of several in a cluster. +#[must_use] +pub fn hub_service_ws_url_named(service: &str) -> String { + format!("ws://{service}:{}/ws", Services::InsecureWebSocketServer.port()) } diff --git a/utilities/cli/src/input.rs b/utilities/cli/src/input.rs index 2e112a7c..5a49f43b 100644 --- a/utilities/cli/src/input.rs +++ b/utilities/cli/src/input.rs @@ -152,12 +152,18 @@ pub struct ClusterInput { #[serde(deny_unknown_fields)] #[non_exhaustive] pub struct Services { + /// The NATS server several hubs share, from `services.backplane:`; run only when there is more than one hub. + #[serde(default, skip_serializing_if = "Service::is_unset")] + pub backplane: Service, /// The telemetry collector, from `services.collector:`. #[serde(default, skip_serializing_if = "Service::is_unset")] pub collector: Service, /// The hub, from `services.hub:`. + #[serde(default, skip_serializing_if = "HubService::is_unset")] + pub hub: HubService, + /// The S3 object store several hubs share, from `services.storage:`; run only when there is more than one hub. #[serde(default, skip_serializing_if = "Service::is_unset")] - pub hub: Service, + pub storage: Service, } impl Services { @@ -186,6 +192,29 @@ impl Service { } } +/// The hub's settings: the limits every service takes, and the machines it runs on. +#[derive(Clone, Debug, Default, Deserialize, Eq, PartialEq, Serialize)] +#[serde(deny_unknown_fields)] +#[non_exhaustive] +pub struct HubService { + /// Resource limits for each hub process, from `limits:`; the same fields an agent's `limits:` takes. + #[serde(default, skip_serializing_if = "Limits::is_unset")] + pub limits: Limits, + /// The machines that each run a hub of their own, from `machines:`; empty runs one hub, on the server. + /// + /// Listed, the server has to be among them, since it also runs the collector and what the hubs share. + #[serde(default, skip_serializing_if = "Vec::is_empty")] + pub machines: Vec, +} + +impl HubService { + /// Whether the scenario sets nothing for the hub. + #[must_use] + pub fn is_unset(&self) -> bool { + *self == Self::default() + } +} + /// One machine of a multi-machine cluster, and the agents that run on it. /// /// The agents are listed here rather than each agent naming its machine, so that an agent that names none has no third diff --git a/utilities/cli/src/lib.rs b/utilities/cli/src/lib.rs index eec7c6bd..b852d2f9 100644 --- a/utilities/cli/src/lib.rs +++ b/utilities/cli/src/lib.rs @@ -36,8 +36,7 @@ pub(crate) use self::deployment_types::{ generate_process_compose_deployment, generate_scenario_image, }; pub use self::error::CliError; -pub(crate) use self::hub_ws_url::HUB_SERVICE; -pub use self::hub_ws_url::{hub_service_ws_url, hub_ws_url, hub_ws_url_at}; +pub use self::hub_ws_url::{hub_service_ws_url, hub_service_ws_url_named, hub_ws_url, hub_ws_url_at}; pub(crate) use self::input::running_in_this_repository; pub use self::input::{ ArtifactSource, ClusterInput, DEFAULT_CLUSTER_NAME, OutputType, Service, Services, infer_artifact_source, @@ -146,7 +145,7 @@ pub(crate) const SUPPORTED_RUNNERS: [(&str, &str); 3] = [ /// into a table and compose writes each service as a mapping key. Either way a collision replaces rather than reports /// -- an agent called `ws-server` would quietly take the hub's place, and the deployment would come up missing the /// thing it was meant to talk to. Rejecting the name is the only way that surfaces. -pub(crate) const RESERVED_RUNNER_NAMES: [&str; 7] = [ +pub(crate) const RESERVED_RUNNER_NAMES: [&str; 9] = [ "generated-scenario", "o2", deployment_types::OPENER_TASK, @@ -154,6 +153,8 @@ pub(crate) const RESERVED_RUNNER_NAMES: [&str; 7] = [ deployment_types::HUB_TASK, deployment_types::HUB_IMAGE_BUILD, deployment_types::SCHEDULER_SERVICE, + deployment_types::BACKPLANE_TASK, + deployment_types::SHARED_STORAGE_TASK, ]; /// What the README says about committing the lockfile, whichever way the deployment's artifacts come. @@ -392,6 +393,18 @@ pub(crate) fn load_cluster_input(input_file: &Path) -> Result<(ClusterInput, u64 placement::validate_machines(&cluster)?; limits::validate_limits(deployment_types::HUB_TASK, &cluster.services.hub.limits)?; limits::validate_limits(deployment_types::COLLECTOR_TASK, &cluster.services.collector.limits)?; + for (service, settings) in [ + ("backplane", &cluster.services.backplane), + ("storage", &cluster.services.storage), + ] { + if !settings.is_unset() && !placement::shares_hubs(&cluster) { + return Err(CliError::SharedServiceWithoutHubs { + service: service.to_owned(), + }); + } + } + limits::validate_limits(deployment_types::BACKPLANE_TASK, &cluster.services.backplane.limits)?; + limits::validate_limits(deployment_types::SHARED_STORAGE_TASK, &cluster.services.storage.limits)?; for agent in &cluster.agents { // An agent's limits govern its runner processes; with no runner its modules run in a browser, which no // deployment format starts, so there would be nothing for them to apply to. @@ -557,7 +570,10 @@ fn generate_deployment_outputs( // One password per scenario, shared by both deployment formats. OpenObserve and the ws-server have to agree on it: // the server authenticates its OTLP exports against the same root credentials the collector was started with. let password = scenario_password(seed); - fs::write(output_dir.join(SECRETS_ENV_FILE), secrets_env(&password))?; + fs::write( + output_dir.join(SECRETS_ENV_FILE), + secrets_env(&password, placement::shares_hubs(cluster)), + )?; write_deployment_gitignore(output_dir)?; for output_type in output_types { generate_format(cluster, output_dir, *output_type, with_process_compose)?; @@ -717,7 +733,9 @@ fn write_deployment_gitignore(output_dir: &Path) -> Result<(), CliError> { /// /// Two names for the one password because the services that share it read different variables: `OpenObserve` /// takes `ZO_ROOT_USER_PASSWORD` as its root credential, and the ws-server authenticates its OTLP exports with -/// `OTLP_AUTH_PASSWORD`. Nothing else belongs here. The account those two authenticate as is not a secret and each +/// `OTLP_AUTH_PASSWORD`. A cluster of several hubs adds the secret of the object store they share, under the name the +/// hubs' S3 client reads and the one rustfs reads. Nothing else belongs here. The accounts these authenticate as are +/// not secrets and each /// format states it outright, so keeping it in this file would have meant an uncommitted file standing between a reader /// and a value there was never any reason to withhold -- and, in the Kubernetes case, a `Secret` holding something that /// is not one. @@ -727,8 +745,8 @@ fn write_deployment_gitignore(output_dir: &Path) -> Result<(), CliError> { /// a secrets scan. The pragma sits on the line above rather than at the end of its own, because an env file has no /// inline comments: every consumer keeps what follows the `=` verbatim, so a trailing marker would become part of the /// password. -fn secrets_env(password: &str) -> String { - format!( +fn secrets_env(password: &str, shares_hubs: bool) -> String { + let mut body = format!( concat!( "# Generated from the scenario input -- regenerating this deployment rewrites it.\n", "# skipcq: SCT-A000 -- derived from the scenario input, not an independently generated secret\n", @@ -737,7 +755,21 @@ fn secrets_env(password: &str) -> String { "OTLP_AUTH_PASSWORD={password}\n", ), password = password - ) + ); + if shares_hubs { + for name in deployment_types::STORAGE_SECRET_ENVS { + let _write_result = writeln!( + body, + concat!( + "# skipcq: SCT-A000 -- derived from the scenario input, not an independently generated secret\n", + "{name}={password}", + ), + name = name, + password = password, + ); + } + } + body } fn discover_verification_scenarios(verification_root: &Path) -> Result, CliError> { @@ -883,7 +915,7 @@ fn generated_readme( install_note = install_note, output_summary = output_summary, module_summary = module_summary, - secrets_note = secrets_note(), + secrets_note = secrets_note(placement::shares_hubs(cluster)), run_instructions = run_instructions, ) } @@ -912,14 +944,20 @@ const fn artifact_source_note(artifacts: ArtifactSource) -> &'static str { /// Worth saying out loud because its absence is silent: `mise` skips an `_.file` it cannot find without a warning, /// and Docker Compose treats a missing `env_file` the same way, so a copy without it starts a collector with no root /// password rather than failing. -fn secrets_note() -> String { +fn secrets_note(shares_hubs: bool) -> String { + let credentials = if shares_hubs { + "OpenObserve, OTLP and object store credentials" + } else { + "OpenObserve and OTLP credentials" + }; format!( concat!( - "`{file}` holds the scenario's derived OpenObserve and OTLP credentials. It is derived from the\n", + "`{file}` holds the scenario's derived {credentials}. It is derived from the\n", "scenario input, so regenerating this deployment rewrites it; if it is missing, regenerate before\n", "starting the stack. A deployment generated outside the repository is written with a `.gitignore`\n", "covering it, so its credential is not committed by whatever repository it lands in.", ), + credentials = credentials, file = SECRETS_ENV_FILE ) } @@ -1066,29 +1104,49 @@ fn generated_run_instructions( ) .to_string(), OutputType::ProcessCompose => process_compose_run_instructions(artifacts).to_string(), - OutputType::DockerCompose => format!( - concat!( - "## Run With Docker Compose\n\n", - "From this directory, start the scenario with:\n\n", - "```bash\n", - "docker compose up --build\n", - "```\n\n", - "{hub}", - "`ws-server` runs with host networking so it advertises the same LAN IP as the `mise` deployment.\n\n", - "### Open The UIs\n\n", - "OpenObserve is available at .\n", - "`ws-server` is available at and .\n\n", - "Stop the scenario with:\n\n", - "```bash\n", - "docker compose down\n", - "```\n" - ), - hub = compose_hub_note(artifacts) - ), + OutputType::DockerCompose => docker_compose_run_instructions(cluster, artifacts), OutputType::K3s => k3s_run_instructions(cluster, image_build, runners, artifacts, build_contexts), } } +/// Render the README section that runs the Docker Compose deployment on one host. +#[expect( + clippy::single_call_fn, + reason = "the Docker Compose step of generated_run_instructions" +)] +fn docker_compose_run_instructions(cluster: &ClusterInput, artifacts: ArtifactSource) -> String { + let other_hubs = if placement::shares_hubs(cluster) { + concat!( + "Every hub listens on the same ports, so on one host only the server's starts: this leaves out\n", + "each other machine running a hub, which runs its share on its own machine instead.\n\n" + ) + } else { + "" + }; + format!( + concat!( + "## Run With Docker Compose\n\n", + "From this directory, start the scenario with:\n\n", + "```bash\n", + "docker compose{profiles} up --build\n", + "```\n\n", + "{other_hubs}", + "{hub}", + "`ws-server` runs with host networking so it advertises the same LAN IP as the `mise` deployment.\n\n", + "### Open The UIs\n\n", + "OpenObserve is available at .\n", + "`ws-server` is available at and .\n\n", + "Stop the scenario with:\n\n", + "```bash\n", + "docker compose{profiles} down\n", + "```\n" + ), + hub = compose_hub_note(artifacts), + other_hubs = other_hubs, + profiles = compose_single_host_profiles(cluster), + ) +} + /// Render how one format runs across the machines a scenario names, or nothing for a single-host scenario. /// /// A subsection after the single-host instructions rather than a rewrite of them, because most of what those say -- @@ -1136,17 +1194,9 @@ fn machine_instructions(output_type: OutputType, cluster: &ClusterInput) -> Stri ), OutputType::K3s => return k3s_machine_instructions(cluster, server), }; - let mut text = format!( - concat!( - "\n### Run Across Machines With {format_name}\n\n", - "{intro}`{server}` runs OpenObserve and `ws-server`, and the runners on every other machine reach\n", - "the hub at `{address}`.\n\n" - ), - address = server.address, - format_name = format_name, - intro = intro, - server = server.name, - ); + let roles = machine_roles(cluster, server); + let loopback = loopback_note(cluster); + let mut text = format!("\n### Run Across Machines With {format_name}\n\n{intro}{roles}{loopback}"); for machine in &cluster.machines { let _write_result = writeln!( text, @@ -1159,6 +1209,71 @@ fn machine_instructions(output_type: OutputType, cluster: &ClusterInput) -> Stri text } +/// Say that the scenario's machine addresses are loopback, when any is, or nothing. +/// +/// Loopback addresses run every machine's share on one host, which is what a verification scenario wants, and none of +/// them reaches another machine. The commands that follow use the addresses as written, so they have to change in the +/// input, and the deployment regenerated, before it runs across machines. +fn loopback_note(cluster: &ClusterInput) -> &'static str { + let loopback = cluster + .machines + .iter() + .any(|machine| machine.address == "localhost" || machine.address.starts_with("127.")); + if !loopback { + return ""; + } + concat!( + "The machines' addresses are loopback, which plays every machine on this one host. To run them on\n", + "separate machines, give each its reachable address in the scenario input and regenerate first.\n\n" + ) +} + +/// The `--profile` flags that start the scenario on one host, or nothing for a scenario naming no machines. +/// +/// Every machine's profile, but that of a machine running a hub besides the server: every hub listens on the same +/// ports, so on one host the server's is the only one that can start, and each runner reaches it at `localhost`. +fn compose_single_host_profiles(cluster: &ClusterInput) -> String { + let hubs = placement::hubs(cluster); + cluster + .machines + .iter() + .filter(|machine| { + !hubs + .iter() + .any(|hub| !hub.on_server() && hub.site.machine_names().contains(&machine.name)) + }) + .fold(String::default(), |mut flags, machine| { + let _write_result = write!(flags, " --profile {}", machine.name); + flags + }) +} + +/// Render what the server runs and where each runner finds its hub. +/// +/// The paragraph every format's machine section opens with. +#[expect(clippy::single_call_fn, reason = "the roles step of machine_instructions")] +fn machine_roles(cluster: &ClusterInput, server: &input::Machine) -> String { + if placement::shares_hubs(cluster) { + return format!( + concat!( + "`{server}` runs OpenObserve, `ws-server`, and the NATS server and object store the hubs share,\n", + "which the other machines' hubs reach at `{address}`. Each runner joins the hub on its own\n", + "machine, or the one on `{server}` when its machine runs none.\n\n" + ), + address = server.address, + server = server.name, + ); + } + format!( + concat!( + "`{server}` runs OpenObserve and `ws-server`, and the runners on every other machine reach\n", + "the hub at `{address}`.\n\n" + ), + address = server.address, + server = server.name, + ) +} + /// Render how the machines of a k3s scenario become one cluster and receive the images their pods run. /// /// Each machine is labelled as it joins rather than afterwards, so the manifests' node selectors have something to @@ -1167,11 +1282,11 @@ fn k3s_machine_instructions(cluster: &ClusterInput, server: &input::Machine) -> let mut joins = String::default(); for machine in &cluster.machines { if machine.name != server.name { - let _write_result = write!( + let _write_result = writeln!( joins, concat!( "sudo k3s agent --server \"https://${{server}}:6443\" --token \"$token\"", - " --node-label {label}={name}\n" + " --node-label {label}={name}" ), label = placement::MACHINE_LABEL, name = machine.name, @@ -1182,6 +1297,7 @@ fn k3s_machine_instructions(cluster: &ClusterInput, server: &input::Machine) -> format!( concat!( "\n### Run Across Machines With k3s\n\n", + "{loopback}", "The machines have to form one k3s cluster first, each labelled with the machine name the manifests\n", "pin pods to. With `k3s` installed on every machine, start the server on `{server}`. It stays in\n", "the foreground, so read the token the others join with from a second shell once it is up:\n\n", @@ -1206,19 +1322,31 @@ fn k3s_machine_instructions(cluster: &ClusterInput, server: &input::Machine) -> imports = imports, joins = joins, label = placement::MACHINE_LABEL, + loopback = loopback_note(cluster), server = server.name, ) } /// The image imports of a k3s scenario spread across machines. /// -/// The scenario image goes to the server, and each locally built runner image to each machine with a runner of that -/// kind. A published scenario's runner images are pulled by the cluster, so only the scenario image is imported for it. +/// The scenario image goes to each machine running a hub, and each locally built runner image to each machine with a +/// runner of that kind. A published scenario's runner images are pulled by the cluster, so only the scenario image is +/// imported for it. fn k3s_machine_imports(cluster: &ClusterInput, server: &input::Machine) -> String { - let mut imports = format!( - "image=\"et-ws-server-{}:latest\"\ndocker save \"$image\" | ssh {} sudo k3s ctr images import -\n", - cluster.cluster_name, server.address - ); + let mut imports = format!("image=\"et-ws-server-{}:latest\"\n", cluster.cluster_name); + for hub in placement::hubs(cluster) { + for machine in cluster + .machines + .iter() + .filter(|machine| hub.site.machine_names().contains(&machine.name)) + { + let _write_result = writeln!( + imports, + "docker save \"$image\" | ssh {} sudo k3s ctr images import -", + machine.address + ); + } + } if matches!(cluster.artifact_source, ArtifactSource::Local) { if !cluster.recovery.is_unset() { let _write_result = writeln!( @@ -2002,12 +2130,13 @@ pub(crate) fn resolve_cluster_runners( // The hub serves a module under its package name, so that is what the runner has to ask for. let module = entry.package_name.clone().unwrap_or_else(|| module_name.to_string()); let name = runner_name(&agent.name, module_name, multi_resource, &runners)?; - // A machine is a mise task of its own as well, and has a compose scheduler service, so a runner taking - // either name would replace it. + // A machine is a mise task of its own as well, has a compose scheduler service and may run a hub, so a + // runner taking any of those names would replace it. if cluster.machines.iter().any(|machine| { name == deployment_types::machine_task(&machine.name) || name == deployment_types::scheduler_service(slice::from_ref(&machine.name)) - }) { + }) || placement::hubs(cluster).iter().any(|hub| hub.name == name) + { return Err(CliError::ReservedRunnerName { agent: agent.name.clone(), name, diff --git a/utilities/cli/src/limits.rs b/utilities/cli/src/limits.rs index d9aaca2a..67f7151e 100644 --- a/utilities/cli/src/limits.rs +++ b/utilities/cli/src/limits.rs @@ -20,7 +20,7 @@ use bytesize::ByteSize; use serde::{Deserialize, Serialize}; use thiserror::Error; -use crate::deployment_types::{COLLECTOR_TASK, HUB_TASK}; +use crate::deployment_types::{BACKPLANE_TASK, COLLECTOR_TASK, HUB_TASK, SHARED_STORAGE_TASK}; use crate::error::CliError; use crate::input::{ClusterInput, OutputType}; @@ -518,8 +518,10 @@ pub(crate) fn k8s_cpus(cpus: Cpus) -> String { /// Every process the scenario sets limits for, by the name each deployment format gives it. fn limited_processes(cluster: &ClusterInput) -> Vec<&str> { let fixed = [ + (BACKPLANE_TASK, &cluster.services.backplane.limits), (COLLECTOR_TASK, &cluster.services.collector.limits), (HUB_TASK, &cluster.services.hub.limits), + (SHARED_STORAGE_TASK, &cluster.services.storage.limits), ]; let agents = cluster.agents.iter().map(|agent| (agent.name.as_str(), &agent.limits)); fixed diff --git a/utilities/cli/src/placement.rs b/utilities/cli/src/placement.rs index d102f766..678d8482 100644 --- a/utilities/cli/src/placement.rs +++ b/utilities/cli/src/placement.rs @@ -2,6 +2,7 @@ use std::collections::BTreeMap; +use crate::deployment_types::HUB_TASK; use crate::error::CliError; use crate::hub_ws_url::{hub_ws_url, hub_ws_url_at}; use crate::input::{ClusterInput, Machine}; @@ -24,9 +25,11 @@ pub(crate) const MACHINE_LABEL: &str = concat!(et_org::org!(), "/machine"); pub(crate) enum Site { /// The scenario names no machines, so everything shares the host the deployment is started on. SingleHost, - /// The server machine, the first listed, which also runs the hub and the collector. - Server { machine: String }, - /// Any other machine, which reaches the hub across the network at the server's address. + /// A machine running a hub of its own, which everything there reaches at `localhost`. + /// + /// The server, which also runs the collector, or another machine `services.hub.machines` lists. + Local { machine: String }, + /// A machine with no hub, which reaches the server's across the network at the server's address. Remote { machine: String, server_address: String }, } @@ -36,7 +39,7 @@ impl Site { pub(crate) fn hub_ws_url(&self) -> String { match self { Self::Remote { server_address, .. } => hub_ws_url_at(server_address), - Self::SingleHost | Self::Server { .. } => hub_ws_url(), + Self::SingleHost | Self::Local { .. } => hub_ws_url(), } } @@ -47,7 +50,7 @@ impl Site { pub(crate) fn machine_names(&self) -> Vec { match self { Self::SingleHost => Vec::new(), - Self::Server { machine } | Self::Remote { machine, .. } => vec![machine.clone()], + Self::Local { machine } | Self::Remote { machine, .. } => vec![machine.clone()], } } @@ -56,17 +59,95 @@ impl Site { pub(crate) const fn is_remote(&self) -> bool { matches!(self, Self::Remote { .. }) } + + /// The hub a runner here connects to: its own machine's, or the server's when its machine runs none. + #[must_use] + pub(crate) fn hub_name(&self, cluster: &ClusterInput) -> String { + match self { + Self::Local { machine } => hub_name_on(cluster, machine), + Self::SingleHost | Self::Remote { .. } => HUB_TASK.to_string(), + } + } } -/// The site of the hub and the collector: the server machine, or the single host. +/// One hub of the cluster and where it runs. +#[derive(Clone, Debug, Eq, PartialEq)] +#[non_exhaustive] +pub(crate) struct Hub { + /// Its mise task, compose service and Kubernetes `Deployment`: [`HUB_TASK`] on the server, suffixed elsewhere. + pub name: String, + /// The machine it runs on. + pub site: Site, + /// What it reaches the server's services at: `localhost` on the server, the server's address on any other machine. + pub server_host: String, +} + +impl Hub { + /// Whether this hub runs on the server, beside the collector and what the hubs share. + #[must_use] + pub(crate) fn on_server(&self) -> bool { + self.name == HUB_TASK + } +} + +/// The site of the collector, and of NATS and the object store when the hubs share them: the server, or the one host. #[must_use] pub(crate) fn hub_site(cluster: &ClusterInput) -> Site { - cluster - .machines - .first() - .map_or(Site::SingleHost, |server| Site::Server { - machine: server.name.clone(), + cluster.machines.first().map_or(Site::SingleHost, |server| Site::Local { + machine: server.name.clone(), + }) +} + +/// Every hub the cluster runs, the server's first and the rest in the order `services.hub.machines` lists them. +/// +/// One, on the server, unless that list names more. +#[must_use] +pub(crate) fn hubs(cluster: &ClusterInput) -> Vec { + let Some(server) = cluster.machines.first() else { + return vec![Hub { + name: HUB_TASK.to_string(), + site: Site::SingleHost, + server_host: "localhost".to_string(), + }]; + }; + let mut machines = vec![server.name.as_str()]; + machines.extend( + cluster + .services + .hub + .machines + .iter() + .map(String::as_str) + .filter(|machine| *machine != server.name), + ); + machines + .into_iter() + .map(|machine| Hub { + name: hub_name_on(cluster, machine), + site: Site::Local { + machine: machine.to_string(), + }, + server_host: if machine == server.name { + "localhost".to_string() + } else { + server.address.clone() + }, }) + .collect() +} + +/// Whether the cluster runs more than one hub, and so the backplane and the object store they share. +#[must_use] +pub(crate) fn shares_hubs(cluster: &ClusterInput) -> bool { + hubs(cluster).len() > 1 +} + +/// The name of the hub on `machine`: [`HUB_TASK`] on the server, so a one-hub cluster's names stay what they were. +fn hub_name_on(cluster: &ClusterInput, machine: &str) -> String { + if cluster.machines.first().is_none_or(|server| server.name == machine) { + return HUB_TASK.to_string(); + } + format!("{HUB_TASK}-{machine}") } /// The site of `agent`: the machine that lists it, or the server when none does. @@ -80,8 +161,8 @@ pub(crate) fn agent_site(cluster: &ClusterInput, agent: &str) -> Site { .iter() .find(|machine| machine.agents.iter().any(|listed| listed == agent)) .unwrap_or(server); - if machine.name == server.name { - return Site::Server { + if machine.name == server.name || cluster.services.hub.machines.contains(&machine.name) { + return Site::Local { machine: machine.name.clone(), }; } @@ -91,7 +172,7 @@ pub(crate) fn agent_site(cluster: &ClusterInput, agent: &str) -> Site { } } -/// Reject a `machines:` list that does not describe one placement for every agent. +/// Reject a `machines:` list that does not describe one placement for every agent and every hub. /// /// The names and addresses are held to narrow alphabets for the reason `cluster_name` is: both reach the generated /// README's shell commands, and a name also becomes a Kubernetes label value, a compose profile and a mise task. An @@ -131,6 +212,47 @@ pub(crate) fn validate_machines(cluster: &ClusterInput) -> Result<(), CliError> } } } + validate_hub_machines(cluster) +} + +/// Reject a `services.hub.machines` that does not name each hub's machine once, the server among them. +#[expect( + clippy::single_call_fn, + reason = "the hub step of validate_machines, separate for readability" +)] +fn validate_hub_machines(cluster: &ClusterInput) -> Result<(), CliError> { + let listed = &cluster.services.hub.machines; + let Some(server) = cluster.machines.first() else { + return listed.first().map_or(Ok(()), |machine| { + Err(CliError::HubMachinesWithoutMachines { + machine: machine.clone(), + }) + }); + }; + for (index, machine) in listed.iter().enumerate() { + if !cluster.machines.iter().any(|declared| declared.name == *machine) { + return Err(CliError::UnknownHubMachine { + machine: machine.clone(), + }); + } + if listed.iter().take(index).any(|earlier| earlier == machine) { + return Err(CliError::DuplicateHubMachine { + machine: machine.clone(), + }); + } + // The hub is named after its machine, and that name is a Kubernetes object name too. + let hub = hub_name_on(cluster, machine); + rfc1123_label(&hub, MACHINE_NAME_MAX, |reason| CliError::InvalidHubName { + machine: machine.clone(), + hub: hub.clone(), + reason, + })?; + } + if !listed.is_empty() && !listed.contains(&server.name) { + return Err(CliError::HubMachinesWithoutServer { + server: server.name.clone(), + }); + } Ok(()) } diff --git a/utilities/cli/tests/multi_hub.rs b/utilities/cli/tests/multi_hub.rs new file mode 100644 index 00000000..360baa2f --- /dev/null +++ b/utilities/cli/tests/multi_hub.rs @@ -0,0 +1,533 @@ +//! Covers a scenario that runs a hub on more than one machine: how `services.hub.machines` is validated, and how each +//! format runs every hub, joins them through the NATS backplane and points them at the one object store. +//! +//! The fixture has three machines: the server, with the trigger; `edge`, which runs a hub of its own and the twin; +//! and `far`, which runs none and so reaches the server's hub across the network. +#![cfg(test)] + +use std::collections::BTreeMap; +use std::path::{Path, PathBuf}; + +use et_cli::{ + ClusterInput, generate_deployment, hub_service_ws_url, hub_service_ws_url_named, hub_ws_url, hub_ws_url_at, + regenerate_verification, +}; +use fs_err as fs; +use k8s_openapi::api::apps::v1::Deployment; +use k8s_openapi::api::core::v1::Container; +use k8s_openapi::apimachinery::pkg::apis::meta::v1::ObjectMeta; +use serde::Deserialize as _; +use tempfile::{TempDir, tempdir}; + +/// The agents and machines every scenario here declares, with the `services:` table appended after them. +const CLUSTER: &str = r#"cluster_name: "hubs" +agents: + - name: "math1-twin" + runner: "web" + resources: + - type: "math1" + - name: "math1-trigger" + runner: "web" + resources: + - type: "math1-sender" + - name: "wasi-twin" + runner: "wasi" + resources: + - type: "wasi-math1" +machines: + - name: "server" + address: "10.0.0.1" + agents: ["math1-trigger"] + - name: "edge" + address: "10.0.0.2" + agents: ["math1-twin"] + - name: "far" + address: "10.0.0.3" + agents: ["wasi-twin"] +"#; + +/// Hubs on the server and on `edge`, with the shared services given limits of their own. +const SERVICES: &str = r#"services: + hub: + machines: ["server", "edge"] + backplane: + limits: + memory: "256m" + storage: + limits: + memory: "512m" +"#; + +/// Every format generated from `input` into a fresh `verification/` tree, with the root kept alive. +fn generated(input: &str) -> (TempDir, PathBuf) { + let test_root = tempdir().unwrap(); + let verification_root = test_root.path().join("verification"); + let input_dir = verification_root.join("local/input"); + fs::create_dir_all(&input_dir).unwrap(); + fs::write(input_dir.join("cluster.yaml"), input).unwrap(); + let _regenerated = regenerate_verification(&verification_root, None).unwrap(); + (test_root, verification_root.join("local/output/cluster")) +} + +fn read(dir: &Path, file: &str) -> String { + fs::read_to_string(dir.join(file)).unwrap() +} + +fn yaml(dir: &Path, file: &str) -> serde_yaml::Value { + serde_yaml::from_str(&read(dir, file)).unwrap() +} + +fn toml(dir: &Path) -> toml::Table { + read(dir, "mise.toml").parse().unwrap() +} + +/// The error generating `input` produces. +fn error_for(input: &str) -> String { + let test_root = tempdir().unwrap(); + let input_file = test_root.path().join("cluster.yaml"); + fs::write(&input_file, input).unwrap(); + generate_deployment(&input_file, &test_root.path().join("output"), None) + .unwrap_err() + .to_string() +} + +/// Any Kubernetes object, read only as far as its name. +#[derive(serde::Deserialize)] +struct Named { + metadata: ObjectMeta, +} + +/// Every k3s document of `kind`, by name. +fn k3s_documents(dir: &Path, kind: &str) -> BTreeMap { + let manifest = read(dir, "k3s.yaml"); + serde_yaml::Deserializer::from_str(&manifest) + .map(|document| serde_yaml::Value::deserialize(document).unwrap()) + .filter(|document| document["kind"] == kind) + .map(|document| { + let named: Named = serde_yaml::from_value(document.clone()).unwrap(); + (named.metadata.name.unwrap(), document) + }) + .collect() +} + +/// The one container of each k3s `Deployment`, with the machine its pods are pinned to. +fn k3s_containers(dir: &Path) -> BTreeMap { + k3s_documents(dir, "Deployment") + .into_iter() + .map(|(name, document)| { + let deployment: Deployment = serde_yaml::from_value(document).unwrap(); + let pod = deployment.spec.unwrap().template.spec.unwrap(); + let machine = pod.node_selector.unwrap_or_default().into_values().collect::(); + (name, (pod.containers[0].clone(), machine)) + }) + .collect() +} + +/// A k3s container's literal environment, by name. +fn container_env(container: &Container) -> BTreeMap { + container + .env + .clone() + .unwrap_or_default() + .into_iter() + .map(|var| (var.name, var.value.unwrap_or_default())) + .collect() +} + +#[test] +fn hub_machines_default_to_none_and_are_left_out_when_unset() { + let cluster: ClusterInput = serde_yaml::from_str(CLUSTER).unwrap(); + assert_eq!(cluster.services.hub.machines, Vec::::new()); + assert!(cluster.services.hub.is_unset()); + assert!(cluster.services.backplane.is_unset()); + assert!(cluster.services.storage.is_unset()); + assert!(!serde_yaml::to_string(&cluster).unwrap().contains("services")); + + let cluster: ClusterInput = serde_yaml::from_str(&format!("{CLUSTER}{SERVICES}")).unwrap(); + assert_eq!(cluster.services.hub.machines, ["server", "edge"]); + assert!(!cluster.services.hub.is_unset()); + let written = serde_yaml::to_string(&cluster).unwrap(); + assert!(written.contains("backplane:")); + assert!(written.contains("storage:")); +} + +#[test] +fn hub_machines_need_machines() { + let input = "cluster_name: \"bare\"\nservices:\n hub:\n machines: [\"server\"]\n"; + assert!(error_for(input).contains("services.hub.machines lists \"server\", but the scenario has no machines")); +} + +#[test] +fn hub_machines_must_be_listed_machines() { + let error = error_for(&format!( + "{CLUSTER}services:\n hub:\n machines: [\"server\", \"nowhere\"]\n" + )); + assert!(error.contains("services.hub.machines lists \"nowhere\", which is not one of the scenario's machines")); +} + +#[test] +fn a_hub_machine_is_listed_once() { + let error = error_for(&format!( + "{CLUSTER}services:\n hub:\n machines: [\"server\", \"edge\", \"edge\"]\n" + )); + assert!(error.contains("services.hub.machines lists \"edge\" more than once")); +} + +#[test] +fn a_hub_name_must_fit_a_kubernetes_object_name() { + // The longest machine name allowed, which leaves no room for the hub's `ws-server-` prefix. + let long = "e".repeat(63); + let cluster = CLUSTER.replace("\"edge\"", &format!("\"{long}\"")); + let input = format!("{cluster}services:\n hub:\n machines: [\"server\", \"{long}\"]\n"); + let error = error_for(&input); + assert!(error.contains(&format!("Machine \"{long}\" would run hub \"ws-server-{long}\""))); + assert!(error.contains("it is longer than 63 characters")); +} + +#[test] +fn hub_machines_include_the_server() { + let error = error_for(&format!("{CLUSTER}services:\n hub:\n machines: [\"edge\"]\n")); + assert!(error.contains("services.hub.machines leaves out the server \"server\"")); +} + +#[test] +fn a_shared_service_needs_several_hubs() { + for service in ["backplane", "storage"] { + let error = error_for(&format!( + "{CLUSTER}services:\n {service}:\n limits:\n memory: \"256m\"\n" + )); + assert!(error.contains(&format!("services.{service} is set, but only several hubs run it"))); + } +} + +#[test] +fn a_runner_cannot_take_a_hub_or_shared_service_name() { + for name in ["ws-server-edge", "nats", "rustfs"] { + let input = format!("{CLUSTER}{SERVICES}").replace("\"wasi-twin\"", &format!("\"{name}\"")); + let expected = format!("would create runner \"{name}\", a name the generated deployment"); + assert!(error_for(&input).contains(&expected)); + } +} + +#[test] +fn the_server_alone_is_one_hub_and_shares_nothing() { + let (_root, dir) = generated(&format!("{CLUSTER}services:\n hub:\n machines: [\"server\"]\n")); + let tasks = toml(&dir)["tasks"].as_table().unwrap().clone(); + assert!(tasks.contains_key("ws-server")); + assert!(!tasks.contains_key("nats")); + assert!(!tasks.contains_key("rustfs")); + assert!(!read(&dir, "secrets.env").contains("AWS_SECRET_ACCESS_KEY")); + assert!(k3s_documents(&dir, "PersistentVolumeClaim").contains_key("ws-server-storage")); +} + +#[test] +fn the_credential_file_carries_the_storage_secret_under_both_names() { + let (_root, dir) = generated(&format!("{CLUSTER}{SERVICES}")); + let secrets = read(&dir, "secrets.env"); + let password = secrets + .lines() + .find_map(|line| line.strip_prefix("OTLP_AUTH_PASSWORD=")) + .unwrap(); + assert!(secrets.contains(&format!("\nAWS_SECRET_ACCESS_KEY={password}\n"))); + assert!(secrets.contains(&format!("\nRUSTFS_SECRET_KEY={password}\n"))); +} + +/// The mise tasks generated from the fixture with its hubs, by name. +fn mise_tasks() -> toml::Table { + let (_root, dir) = generated(&format!("{CLUSTER}{SERVICES}")); + toml(&dir)["tasks"].as_table().unwrap().clone() +} + +/// The names a mise task depends on, in the order the file lists them. +fn depends(tasks: &toml::Table, task: &str) -> Vec { + tasks[task]["depends"] + .as_array() + .unwrap() + .iter() + .map(|value| value.as_str().unwrap().to_string()) + .collect() +} + +#[test] +fn mise_starts_each_hub_with_its_machine() { + let tasks = mise_tasks(); + assert_eq!( + depends(&tasks, "machine-server"), + ["math1-trigger", "nats", "openobserve", "rustfs", "ws-server"] + ); + assert_eq!(depends(&tasks, "machine-edge"), ["math1-twin", "ws-server-edge"]); + assert_eq!(depends(&tasks, "machine-far"), ["wasi-twin"]); + assert!(!depends(&tasks, "generated-scenario").contains(&"ws-server-edge".to_string())); + assert_eq!( + tasks["ws-server-edge"]["description"].as_str(), + Some("Run the WebSocket server on edge") + ); +} + +#[test] +fn mise_hubs_reach_the_shared_services_at_the_server() { + let tasks = mise_tasks(); + let server_env = tasks["ws-server"]["env"].as_table().unwrap(); + assert_eq!(server_env["WS_BACKPLANE"].as_str(), Some("nats://localhost:4222")); + assert_eq!(server_env["AWS_ENDPOINT"].as_str(), Some("http://localhost:9000")); + assert_eq!(server_env["STORAGE_URL"].as_str(), Some("s3://et-storage")); + let edge_env = tasks["ws-server-edge"]["env"].as_table().unwrap(); + assert_eq!(edge_env["WS_BACKPLANE"].as_str(), Some("nats://10.0.0.1:4222")); + assert_eq!(edge_env["AWS_ENDPOINT"].as_str(), Some("http://10.0.0.1:9000")); +} + +#[test] +fn mise_runners_join_their_own_machines_hub_or_the_servers() { + let tasks = mise_tasks(); + let url = |task: &str| tasks[task]["env"]["WS_SERVER_URL"].as_str().unwrap().to_string(); + assert_eq!(url("math1-twin"), hub_ws_url()); + assert_eq!(url("wasi-twin"), hub_ws_url_at("10.0.0.1")); +} + +#[test] +fn mise_runs_the_shared_services_as_containers() { + let tasks = mise_tasks(); + let nats = tasks["nats"]["run"].as_str().unwrap(); + assert!(nats.contains("image=nats:2.15.1-alpine\n")); + assert!(nats.contains("container=nats-hubs\n")); + assert!(nats.contains("--jetstream --store_dir /data --http_port 8222")); + let rustfs = tasks["rustfs"]["run"].as_str().unwrap(); + assert!(rustfs.contains("image=rustfs/rustfs:1.0.0-beta.12\n")); + assert!(rustfs.contains("-e RUSTFS_ACCESS_KEY=et-storage -e RUSTFS_VOLUMES=/data -e RUSTFS_SECRET_KEY")); + assert!(rustfs.contains("start='mkdir -p /data/et-storage && exec /entrypoint.sh rustfs'")); +} + +#[test] +fn dekit_waits_on_the_shared_services_on_the_server_only() { + let (_root, dir) = generated(&format!("{CLUSTER}{SERVICES}")); + let dekit = yaml(&dir, "dekit.yaml"); + let tasks = &dekit["tasks"]; + assert_eq!( + tasks["ws-server"]["deps"], + serde_yaml::from_str::("[openobserve, nats, rustfs]").unwrap() + ); + assert!(tasks["ws-server-edge"].get("deps").is_none()); + assert_eq!(tasks["ws-server-edge"]["autorestart"], "on-failure"); + assert_eq!(tasks["ws-server-edge"]["tags"][0], "edge"); + assert_eq!(tasks["math1-twin"]["deps"][0], "ws-server-edge"); + assert!(tasks["wasi-twin"].get("deps").is_none()); + assert_eq!( + tasks["nats"]["ready"]["http"], + "http://localhost:8222/healthz?js-enabled-only=true" + ); + assert_eq!(tasks["rustfs"]["stop"]["cmd"][2], "rustfs-hubs"); +} + +#[test] +fn process_compose_waits_on_the_shared_services_on_the_server_only() { + let (_root, dir) = generated(&format!("{CLUSTER}{SERVICES}")); + let process_compose = yaml(&dir, "process-compose.yaml"); + let processes = &process_compose["processes"]; + assert!(processes["ws-server"]["depends_on"].get("rustfs").is_some()); + assert!(processes["ws-server-edge"].get("depends_on").is_none()); + assert_eq!(processes["ws-server-edge"]["availability"]["restart"], "on_failure"); + assert_eq!(processes["ws-server-edge"]["namespace"], "edge"); + assert!(processes["math1-twin"]["depends_on"].get("ws-server-edge").is_some()); + assert_eq!( + processes["rustfs"]["readiness_probe"]["http_get"]["port"].as_u64(), + Some(9_000_u64) + ); + assert_eq!(processes["nats"]["shutdown"]["command"], "docker stop nats-hubs"); +} + +/// The compose services generated from the fixture with its hubs, by name, and the volumes the file declares. +fn compose_file() -> serde_yaml::Value { + let (_root, dir) = generated(&format!("{CLUSTER}{SERVICES}")); + yaml(&dir, "compose.yaml") +} + +#[test] +fn compose_runs_a_hub_per_machine_profile() { + let compose = compose_file(); + let services = &compose["services"]; + assert_eq!(services["ws-server"]["profiles"][0], "server"); + assert_eq!(services["ws-server-edge"]["profiles"][0], "edge"); + assert_eq!(services["ws-server-edge"]["restart"], "on-failure"); + assert_eq!( + services["ws-server-edge"]["depends_on"]["nats"]["required"].as_bool(), + Some(false) + ); + assert!(services["ws-server"]["depends_on"]["rustfs"].get("required").is_none()); + assert!(services["ws-server"].get("volumes").is_none()); + assert_eq!( + services["hub-image-build"]["profiles"], + serde_yaml::from_str::("[server, edge]").unwrap() + ); +} + +#[test] +fn compose_hubs_share_the_servers_services_and_runners_join_their_own_hub() { + let compose = compose_file(); + let services = &compose["services"]; + let edge_env = &services["ws-server-edge"]["environment"]; + assert_eq!(edge_env["WS_BACKPLANE"], "nats://10.0.0.1:4222"); + assert_eq!(edge_env["AWS_ALLOW_HTTP"], "true"); + assert_eq!(edge_env["STORAGE_URL"], "s3://et-storage"); + assert_eq!( + services["math1-twin"]["environment"]["WS_SERVER_URL"].as_str(), + Some(hub_ws_url().as_str()) + ); + assert!(services["math1-twin"]["depends_on"].get("ws-server-edge").is_some()); + assert_eq!( + services["wasi-twin"]["depends_on"]["ws-server"]["required"].as_bool(), + Some(false) + ); +} + +#[test] +fn compose_runs_the_shared_services_on_the_server_under_their_limits() { + let compose = compose_file(); + let services = &compose["services"]; + assert_eq!(services["nats"]["command"][4], "8222"); + assert_eq!(services["nats"]["deploy"]["resources"]["limits"]["memory"], "250000k"); + assert_eq!(services["rustfs"]["entrypoint"][0], "sh"); + assert_eq!(services["rustfs"]["environment"]["RUSTFS_ACCESS_KEY"], "et-storage"); + assert_eq!(services["rustfs"]["deploy"]["resources"]["limits"]["memory"], "500000k"); + let volumes = compose["volumes"].as_mapping().unwrap(); + let names: Vec<&str> = volumes.keys().map(|key| key.as_str().unwrap()).collect(); + assert_eq!(names, ["nats-data", "openobserve-data", "rustfs-data"]); +} + +#[test] +fn compose_keeps_the_nats_monitor_on_loopback() { + let compose = compose_file(); + let nats = &compose["services"]["nats"]; + assert!(nats.get("network_mode").is_none()); + assert_eq!( + nats["ports"], + serde_yaml::from_str::("[\"4222:4222\", \"127.0.0.1:8222:8222\"]").unwrap() + ); + assert_eq!(compose["services"]["rustfs"]["network_mode"], "host"); +} + +#[test] +fn compose_starts_on_one_host_with_every_profile_but_another_hubs() { + let (_root, dir) = generated(&format!("{CLUSTER}{SERVICES}")); + let readme = read(&dir, "README.md"); + assert!(readme.contains("docker compose --profile server --profile far up --build\n")); + assert!(readme.contains("docker compose --profile server --profile far down\n")); + assert!(readme.contains("on one host only the server's starts")); + + let (_root, dir) = generated(CLUSTER); + let readme = read(&dir, "README.md"); + assert!(readme.contains("docker compose --profile server --profile edge --profile far up --build\n")); + assert!(!readme.contains("on one host only the server's starts")); +} + +#[test] +fn the_readme_warns_when_the_machines_are_at_loopback() { + let note = "The machines' addresses are loopback"; + let (_root, dir) = generated(&format!("{CLUSTER}{SERVICES}")); + assert!(!read(&dir, "README.md").contains(note)); + + let (_root, dir) = generated(&format!("{CLUSTER}{SERVICES}").replace("10.0.0.2", "127.0.0.1")); + let readme = read(&dir, "README.md"); + // One for each format's machine section: mise, dekit, process-compose, Docker Compose and k3s. + assert_eq!(readme.matches(note).count(), 5); + + let (_root, dir) = generated(&format!("{CLUSTER}{SERVICES}").replace("10.0.0.3", "localhost")); + assert!(read(&dir, "README.md").contains(note)); +} + +#[test] +fn compose_quotes_values_yaml_would_not_read_as_strings() { + let input = format!("{CLUSTER}{SERVICES}").replace( + " runner: \"wasi\"\n", + " runner: \"wasi\"\n env:\n FLAG: \"yes\"\n COUNT: \"3\"\n NAME: \"plain\"\n", + ); + let (_root, dir) = generated(&input); + let compose = read(&dir, "compose.yaml"); + assert!(compose.contains(" FLAG: \"yes\"\n")); + assert!(compose.contains(" COUNT: \"3\"\n")); + assert!(compose.contains(" NAME: plain\n")); + assert!(compose.contains(" - \"8222\"\n")); +} + +#[test] +fn k3s_pins_a_hub_to_each_machine_and_the_shared_services_to_the_server() { + let (_root, dir) = generated(&format!("{CLUSTER}{SERVICES}")); + let containers = k3s_containers(&dir); + assert_eq!(containers["ws-server"].1, "server"); + assert_eq!(containers["ws-server-edge"].1, "edge"); + assert_eq!(containers["nats"].1, "server"); + assert_eq!(containers["rustfs"].1, "server"); +} + +#[test] +fn k3s_hubs_reach_the_shared_services_by_their_service_names() { + let (_root, dir) = generated(&format!("{CLUSTER}{SERVICES}")); + let hub_env = container_env(&k3s_containers(&dir)["ws-server-edge"].0); + assert_eq!(hub_env["WS_BACKPLANE"], "nats://nats:4222"); + // Composed from the service name rather than written out, so link checking does not read an in-cluster DNS name + // as a link it should be able to reach. + let storage_service = "rustfs"; + assert_eq!(hub_env["AWS_ENDPOINT"], format!("http://{storage_service}:9000")); + assert_eq!(hub_env["STORAGE_URL"], "s3://et-storage"); + let names: Vec<&String> = hub_env.keys().collect(); + let mut sorted = names.clone(); + sorted.sort(); + assert_eq!(names, sorted); +} + +#[test] +fn k3s_runners_join_their_own_machines_hub_or_the_servers() { + let (_root, dir) = generated(&format!("{CLUSTER}{SERVICES}")); + let containers = k3s_containers(&dir); + let url = |name: &str| container_env(&containers[name].0)["WS_SERVER_URL"].clone(); + assert_eq!(url("math1-twin"), hub_service_ws_url_named("ws-server-edge")); + assert_eq!(url("math1-trigger"), hub_service_ws_url()); + assert_eq!(url("wasi-twin"), hub_service_ws_url()); +} + +#[test] +fn k3s_runs_nats_from_its_own_command_over_a_mounted_config() { + let (_root, dir) = generated(&format!("{CLUSTER}{SERVICES}")); + let nats = k3s_containers(&dir)["nats"].0.clone(); + assert!(nats.args.is_none()); + assert!(nats.command.is_none()); + let mounts: Vec = nats + .volume_mounts + .unwrap() + .into_iter() + .map(|mount| mount.mount_path) + .collect(); + assert_eq!(mounts, ["/data", "/etc/nats"]); + assert_eq!( + k3s_documents(&dir, "ConfigMap")["nats-config"]["data"]["nats-server.conf"], + "port: 4222\nmonitor_port: 8222\njetstream {\n store_dir: /data\n}\n" + ); + let readiness = nats.readiness_probe.unwrap().http_get.unwrap(); + assert_eq!(readiness.path.as_deref(), Some("/healthz?js-enabled-only=true")); +} + +#[test] +fn k3s_gives_the_shared_services_claims_and_services_in_place_of_hub_storage() { + let (_root, dir) = generated(&format!("{CLUSTER}{SERVICES}")); + let rustfs = k3s_containers(&dir)["rustfs"].0.clone(); + assert_eq!(rustfs.command.unwrap()[0], "sh"); + assert!(rustfs.env_from.is_some()); + let claims = k3s_documents(&dir, "PersistentVolumeClaim"); + assert!(claims.contains_key("nats-data")); + assert!(claims.contains_key("rustfs-data")); + assert!(!claims.contains_key("ws-server-storage")); + let services = k3s_documents(&dir, "Service"); + assert!(services.contains_key("ws-server-edge")); + assert_eq!(services["nats"]["spec"]["ports"][1]["port"].as_u64(), Some(8_222_u64)); +} + +#[test] +fn the_readme_says_which_machine_runs_what_and_ships_the_hub_image_to_each() { + let (_root, dir) = generated(&format!("{CLUSTER}{SERVICES}")); + let readme = read(&dir, "README.md"); + assert!(readme.contains("OpenObserve, OTLP and object store credentials")); + assert!(readme.contains("`server` runs OpenObserve, `ws-server`, and the NATS server and object store the hubs")); + assert!(readme.contains("Each runner joins the hub on its own\nmachine, or the one on `server`")); + assert!(readme.contains("docker save \"$image\" | ssh 10.0.0.1 sudo k3s ctr images import -\n")); + assert!(readme.contains("docker save \"$image\" | ssh 10.0.0.2 sudo k3s ctr images import -\n")); + assert!(!readme.contains("docker save \"$image\" | ssh 10.0.0.3 ")); +} diff --git a/verification/README.md b/verification/README.md new file mode 100644 index 00000000..82a0222b --- /dev/null +++ b/verification/README.md @@ -0,0 +1,22 @@ +# Verification scenarios + +Scenario inputs and the deployments `et-cli` generates from them, committed so that any change to a generator shows +up as a diff here. + +- `local/` renders each scenario against builds of this tree, and `published/` renders the same scenarios against + the released artifacts. +- `/input/.yaml` is one scenario, and its header comment says what it covers. +- `/output//` is every format generated from it, with a `README.md` on running each. A scheduled scenario + has no `mise.toml` or `dekit.yaml`, which cannot schedule, and a `recovery:` one is k3s only. + +`mise run regen-verification` rewrites every output, and `mise run verification-check` fails when a committed one +has drifted from what its input generates. + +The math1-shaped scenarios also run end to end, each checking the model the twin stores: + +- `utilities/cli/tests/scenario_runners.rs` runs the generated mise runner tasks against an in-process hub. +- `mise run k3s-verify [local|published] --scenario ` applies `k3s.yaml` to a throwaway k3d cluster with one + node per machine, and checks each pod landed on its machine's node. It expects the images built first. +- `mise run compose-verify --scenario ` brings `compose.yaml` up one machine profile at a time. + +Both also check a scenario's resource limits in the running containers when it sets any. diff --git a/verification/local/input/math1-multi-hub.yaml b/verification/local/input/math1-multi-hub.yaml new file mode 100644 index 00000000..0cead6b5 --- /dev/null +++ b/verification/local/input/math1-multi-hub.yaml @@ -0,0 +1,31 @@ +cluster_name: "math1-multi-hub" +deployment_type: "k3s" + +# The math1 pair split across two machines that each run a hub, so the exchange crosses from one hub to the other. +# The trigger joins the server's hub and the twin the edge machine's own, and the two hubs relay between them over +# the NATS backplane on the server. The twin stores its model through the edge hub into the shared object store, +# where the server's hub reads it back. Each hub listens on the same ports, so only k3s, where every pod has a +# network of its own, runs both on one host; the addresses are loopback as in math1-multi. +agents: + - name: "math1-twin" + runner: "web" + resources: + - type: "math1" + + - name: "math1-trigger" + runner: "web" + resources: + - type: "math1-sender" + +machines: + - name: "server" + address: "127.0.0.1" + agents: ["math1-trigger"] + + - name: "edge" + address: "127.0.0.1" + agents: ["math1-twin"] + +services: + hub: + machines: ["server", "edge"] diff --git a/verification/local/output/math1-multi-hub/.gitignore b/verification/local/output/math1-multi-hub/.gitignore new file mode 100644 index 00000000..4b22f893 --- /dev/null +++ b/verification/local/output/math1-multi-hub/.gitignore @@ -0,0 +1,7 @@ +# Written by `et-cli`. +# What the hub writes into the directory it runs from: its self-signed TLS pair (the key is a +# credential too), the agent registry it saves on shutdown, and agent storage. +cert.pem +key.pem +registry.yaml +storage/ diff --git a/verification/local/output/math1-multi-hub/Dockerfile b/verification/local/output/math1-multi-hub/Dockerfile new file mode 100644 index 00000000..64c58483 --- /dev/null +++ b/verification/local/output/math1-multi-hub/Dockerfile @@ -0,0 +1,29 @@ +# syntax=docker/dockerfile:1 + +# AUTO-GENERATED by `et-cli` for the `math1-multi-hub` scenario -- do not edit. +# Regenerate with `mise run regen-verification`. +# +# The hub image (services/ws-server/Dockerfile) carries no workflow modules at all, because which +# modules a deployment serves is a per-cluster decision. This image is that decision made concrete: it +# stages exactly the modules this cluster's agents declare, plus the packages those modules depend on, +# into the `/app` paths the hub already scans. +# +# The sibling `Dockerfile.dockerignore` is what makes the COPYs below possible. The repository's root +# `.dockerignore` drops every `*.wasm` and `*.onnx` so the CI images built from `./Dockerfile` stay +# small; BuildKit uses this Dockerfile's own ignore file instead, which re-admits the built `pkg/` +# directories and nothing else. + +# `hub` is a named build context, not a stage defined here. +# The generated compose file wires it to its `hub-image-build` service, so compose builds the hub image +# first and hands the result over under this name. That is also why the version-tag rule is +# suppressed on the next line: hadolint reads `hub` as an untagged image, but there is no tag to give a +# build context, and the image it resolves to is pinned by the hub Dockerfile rather than here. +# hadolint ignore=DL3006 +FROM hub + +LABEL org.opencontainers.image.source="https://github.com/edge-toolkit/core" + +COPY --chown=10001:10001 services/ws-modules/math1/pkg /app/services/ws-modules/math1/pkg +COPY --chown=10001:10001 services/ws-modules/math1-sender/pkg /app/services/ws-modules/math1-sender/pkg + +USER 10001 diff --git a/verification/local/output/math1-multi-hub/Dockerfile.dockerignore b/verification/local/output/math1-multi-hub/Dockerfile.dockerignore new file mode 100644 index 00000000..1e6b10d4 --- /dev/null +++ b/verification/local/output/math1-multi-hub/Dockerfile.dockerignore @@ -0,0 +1,85 @@ +# AUTO-GENERATED by `et-cli` for the `math1-multi-hub` scenario -- do not edit. +# Regenerate with `mise run regen-verification`. +# +# BuildKit reads this file instead of the repository root `.dockerignore` when building the +# Dockerfile beside it, so the root file keeps governing the CI images built from `./Dockerfile` +# and only this build sees the re-includes at the end. + +# AUTO-GENERATED from .gitignore by 'mise run gen:dockerignore' -- do not edit. +# Docker reads only this file; patterns are **/-prefixed to match at any depth +# like .gitignore. Edit .gitignore and regenerate. + +# `.dockerignore` is generated from `.gitignore` by `mise run gen:dockerignore`. +# After editing `.gitignore`, regenerate it and commit both, or the dockerignore-check task fails on drift. +**/.claude/ +**/*.wasm +**/*.onnx +**/target/ +**/.DS_Store +services/ws-wasm-agent/pkg/ +services/ws-modules/pywasm1/pkg/ +services/ws-modules/rdata1/pkg/webr/ +services/ws-modules/rcomm1/pkg/webr/ +services/ws-modules/rmath1/pkg/webr/ +services/ws-modules/js-data1/pkg/et_ws_js_data1.js +services/ws-server/static/models/ +**/.zig-cache/ +**/zig-out/ +**/*.o +**/*.pem +**/mprocs.log +**/__pycache__/ +**/.pytest_cache/ +**/.python-version +**/node_modules/ +**/.venv/ +# .NET build output. +# `obj/` is safe globally (nothing tracked is named obj/), but `bin/` is scoped to the module so it never +# matches a Rust crate's `src/bin/` (e.g. utilities/int-gen/src/bin/). +**/obj/ +services/ws-modules/dotnet-data1/bin/ +# Gradle project caches and build output plus the Kotlin session dir, all regenerated by any Gradle run. +**/.gradle/ +**/.kotlin/ +services/ws-modules/kotlin-data1/build/ +services/ws-modules/kotlin-math1/build/ +# Coverage-report artifacts produced by the coverage workflow, uploaded to Codecov and never committed. +# Covers cargo-llvm-cov's lcov, pytest-cov's Cobertura xml, and pytest-cov's .coverage data file. +lcov.info +**/coverage-python.xml +**/.coverage +# minicov/llvm raw profiles dropped by instrumented binaries run during coverage builds. +**/*.profraw +# Editor dir, tool caches, and the ws-server's runtime file storage. +# The .vscode dir is ignored except its shared recommended-extensions list (negated below). +.vscode/* +!.vscode/extensions.json +**/.ruff_cache/ +**/.ryl_cache/ +**/.lycheecache +# Dart pub tool dir, regenerated by `dart pub get`; one global rule for the whole workspace (root + members). +**/.dart_tool/ +services/ws-server/storage/ +# Lock files a generated scenario leaves behind when it is run. +# `lockfile = true` makes mise write one beside whichever config it resolved tools for, so running a scenario +# drops `mise.lock` into that scenario's output directory. It is a by-product of running the deployment rather +# than part of it: `regen-verification` never writes one, the drift check never reads one, and what it pins is +# whatever the machine that ran it happened to resolve. One rule here rather than a `.gitignore` generated into +# each scenario, so a scenario added later is covered without anything having to be written into it. +verification/**/*.lock +# The npm dependency graphs `mise lock` records beside that lockfile, a by-product for the same reason. +verification/**/.mise/locks/ +# No rule for the per-scenario credential file, deliberately. +# Under verification/ it is committed like every other generated artifact: the password is derived from an +# input this repository already carries, so the file reproduces from public material rather than keeping a +# secret, and the drift check has to be able to read it. A deployment generated anywhere else gets its own +# `.gitignore` written beside it by `et-cli`, which is what keeps a real credential out of a real repository. +**/.git/ +**/Dockerfile* +README.md + +# Re-admit the built module artifacts this scenario serves. +# The rules above drop every `*.wasm` and `*.onnx`, which is what keeps the CI images small; these +# negations put back the `pkg/` directories this image copies, and nothing else. +!services/ws-modules/math1-sender/pkg/** +!services/ws-modules/math1/pkg/** diff --git a/verification/local/output/math1-multi-hub/README.md b/verification/local/output/math1-multi-hub/README.md new file mode 100644 index 00000000..2d75ebea --- /dev/null +++ b/verification/local/output/math1-multi-hub/README.md @@ -0,0 +1,289 @@ +# math1-multi-hub + +This directory contains generated deployment configs for the `math1-multi-hub` scenario. +Files: `mise.toml`, `dekit.yaml`, `process-compose.yaml`, `compose.yaml`, `k3s.yaml`. + +The scenario exposes these workflow modules: math1, math1-sender. + +`secrets.env` holds the scenario's derived OpenObserve, OTLP and object store credentials. It is derived from the +scenario input, so regenerating this deployment rewrites it; if it is missing, regenerate before +starting the stack. A deployment generated outside the repository is written with a `.gitignore` +covering it, so its credential is not committed by whatever repository it lands in. + +## Lock The Tools + +Before the first run, lock the tools `mise.toml` names: + +```bash +mise lock +``` + +Commit `mise.lock` alongside the deployment, together with the `.mise/locks/` directory when +mise writes one -- it holds the dependency graph of each npm tool, which `mise.lock` refers to. +The two record the version and checksum every `latest` in `mise.toml` resolved to, for each of +mise's common platforms, so every later install -- on any machine -- fetches the same binaries +rather than whatever is newest that day. Move to newer releases deliberately, with +`mise lock --bump`, and commit the result. + +## Run With Mise + +From this directory, start the scenario with: + +```bash +mise run generated-scenario +``` + +That task starts both OpenObserve and `ws-server` for this scenario. + +### Open The OpenObserve UI + +From this directory, open the OpenObserve UI with: + +```bash +mise run open-o2 +``` + +### Run Across Machines With Mise + +Copy this directory to each machine -- for a scenario built from the repository, into a checkout +of it at the same path -- and start that machine's share of the scenario there. +`server` runs OpenObserve, `ws-server`, and the NATS server and object store the hubs share, +which the other machines' hubs reach at `127.0.0.1`. Each runner joins the hub on its own +machine, or the one on `server` when its machine runs none. + +The machines' addresses are loopback, which plays every machine on this one host. To run them on +separate machines, give each its reachable address in the scenario input and regenerate first. + +On `server`: + +```bash +mise run machine-server +``` + +On `edge`: + +```bash +mise run machine-edge +``` + +## Run With dekit + +From this directory, start the scenario and watch each process in its own pane with: + +```bash +dekit up +dekit attach +``` + +Each task runs one task of `mise.toml`, so these are the processes `mise run generated-scenario` +starts, but in order: `ws-server` waits for OpenObserve to report healthy, and every runner +waits for `ws-server`. The tasks keep running after the terminal closes; stop them with: + +```bash +dekit down +``` + +Open the OpenObserve UI, once it is up, with: + +```bash +dekit start open-o2 +``` + +### Run Across Machines With dekit + +Copy this directory to each machine and start that machine's share there. Every task is tagged +with its machine and none starts on `dekit up` alone, so `dekit up` on any machine starts nothing +until its tag is started. +`server` runs OpenObserve, `ws-server`, and the NATS server and object store the hubs share, +which the other machines' hubs reach at `127.0.0.1`. Each runner joins the hub on its own +machine, or the one on `server` when its machine runs none. + +The machines' addresses are loopback, which plays every machine on this one host. To run them on +separate machines, give each its reachable address in the scenario input and regenerate first. + +On `server`: + +```bash +dekit up +dekit start +server +``` + +On `edge`: + +```bash +dekit up +dekit start +edge +``` + +## Run With process-compose + +From this directory, install the process-compose `mise.toml` declares, then start the scenario +and watch each process with: + +```bash +mise install +process-compose up --no-server -f process-compose.yaml +``` + +Each process runs one task of `mise.toml`, in order: `ws-server` waits for OpenObserve to report +healthy, and every runner waits for `ws-server`. A scheduled agent's runner is started on each +tick of its schedule and runs its module once; a tick that finds the previous run still going is +skipped. `--no-server` keeps process-compose's own API off port 8080, where `ws-server` listens. +Quitting the TUI stops every process. Open the OpenObserve UI, once it is up, by starting the +`open-o2` process from the TUI. + +### Run Across Machines With process-compose + +Copy this directory to each machine -- for a scenario built from the repository, into a checkout +of it at the same path -- and start only that machine's namespace there. +`server` runs OpenObserve, `ws-server`, and the NATS server and object store the hubs share, +which the other machines' hubs reach at `127.0.0.1`. Each runner joins the hub on its own +machine, or the one on `server` when its machine runs none. + +The machines' addresses are loopback, which plays every machine on this one host. To run them on +separate machines, give each its reachable address in the scenario input and regenerate first. + +On `server`: + +```bash +process-compose up --no-server -f process-compose.yaml -n server +``` + +On `edge`: + +```bash +process-compose up --no-server -f process-compose.yaml -n edge +``` + +## Run With Docker Compose + +From this directory, start the scenario with: + +```bash +docker compose --profile server up --build +``` + +Every hub listens on the same ports, so on one host only the server's starts: this leaves out +each other machine running a hub, which runs its share on its own machine instead. + +The compose stack starts OpenObserve and builds `ws-server` in two layers: the module-less hub +image from the repository's `services/ws-server/Dockerfile`, then the `Dockerfile` in this +directory, which stages this scenario's modules onto it. +`ws-server` runs with host networking so it advertises the same LAN IP as the `mise` deployment. + +### Open The UIs + +OpenObserve is available at . +`ws-server` is available at and . + +Stop the scenario with: + +```bash +docker compose --profile server down +``` + +### Run Across Machines With Docker Compose + +Copy this directory to each machine -- for a scenario built from the repository, into a checkout +of it at the same path -- and start only that machine's profile there. +`server` runs OpenObserve, `ws-server`, and the NATS server and object store the hubs share, +which the other machines' hubs reach at `127.0.0.1`. Each runner joins the hub on its own +machine, or the one on `server` when its machine runs none. + +The machines' addresses are loopback, which plays every machine on this one host. To run them on +separate machines, give each its reachable address in the scenario input and regenerate first. + +On `server`: + +```bash +docker compose --profile server up --build +``` + +On `edge`: + +```bash +docker compose --profile edge up --build +``` + +## Run With k3s + +The manifests reference images by name and never build them, so build and import each one first. +This scenario's image layers its modules onto the module-less hub image and takes that hub as a +_named build context_ rather than building it, so the hub has to exist first -- a plain +`docker build` of the scenario Dockerfile fails on `FROM hub`. From the repository root: + +```bash +scenario=math1-multi-hub +hub=et-ws-server-hub:latest +image="et-ws-server-$scenario:latest" +dockerfile="verification/local/output/$scenario/Dockerfile" +docker build -t "$hub" -f services/ws-server/Dockerfile . +docker build --build-context "hub=docker-image://$hub" -t "$image" -f "$dockerfile" . +docker save "$image" | sudo k3s ctr images import - +``` + +Then this scenario's runner images, which need no build context of their own: + +```bash +docker build -t et-ws-web-runner:latest -f services/ws-web-runner/Dockerfile . +docker save et-ws-web-runner:latest | sudo k3s ctr images import - +``` + +### Load The Credential + +The credential reaches the pods as a `Secret` created from `secrets.env`, rather than written into +`k3s.yaml` where it would be committed alongside the manifests. From this directory: + +```bash +ns=et-math1-multi-hub +kubectl create namespace "$ns" --save-config +kubectl create secret generic "$ns-secrets" --from-env-file=secrets.env -n "$ns" +``` + +### Apply + +```bash +kubectl apply -f k3s.yaml +``` + +The runners exit and restart until the hub reports ready, so `CrashLoopBackOff` while the hub +starts is expected here rather than a fault. Watch it settle with: + +```bash +kubectl get pods -n "$ns" --watch +``` + +### Run Across Machines With k3s + +The machines' addresses are loopback, which plays every machine on this one host. To run them on +separate machines, give each its reachable address in the scenario input and regenerate first. + +The machines have to form one k3s cluster first, each labelled with the machine name the manifests +pin pods to. With `k3s` installed on every machine, start the server on `server`. It stays in +the foreground, so read the token the others join with from a second shell once it is up: + +```bash +sudo k3s server --node-label edge-toolkit/machine=server +``` + +```bash +sudo cat /var/lib/rancher/k3s/server/node-token +``` + +Then join each other machine, with that token in `token`: + +```bash +server=127.0.0.1 +sudo k3s agent --server "https://${server}:6443" --token "$token" --node-label edge-toolkit/machine=edge +``` + +Each image goes to the machines that run it rather than to one node, so the import steps above +become these, with each machine reachable over `ssh` by its address: + +```bash +image="et-ws-server-math1-multi-hub:latest" +docker save "$image" | ssh 127.0.0.1 sudo k3s ctr images import - +docker save "$image" | ssh 127.0.0.1 sudo k3s ctr images import - +docker save et-ws-web-runner:latest | ssh 127.0.0.1 sudo k3s ctr images import - +docker save et-ws-web-runner:latest | ssh 127.0.0.1 sudo k3s ctr images import - +``` diff --git a/verification/local/output/math1-multi-hub/compose.yaml b/verification/local/output/math1-multi-hub/compose.yaml new file mode 100644 index 00000000..8887d41b --- /dev/null +++ b/verification/local/output/math1-multi-hub/compose.yaml @@ -0,0 +1,245 @@ +services: + openobserve: + profiles: + - server + image: openobserve/openobserve:v1.1.1 + healthcheck: + test: + - CMD + - /openobserve + - node + - status + interval: 5s + timeout: 3s + retries: 20 + start_period: 10s + ports: + - 127.0.0.1:5080:5080 + env_file: + - secrets.env + environment: + RUST_LOG: warn + ZO_ROOT_USER_EMAIL: root@example.com + ZO_DATA_DIR: /data + volumes: + - openobserve-data:/data + deploy: + resources: + limits: + cpus: "2" + memory: 2g + cpu_shares: 102 + nats: + profiles: + - server + image: nats:2.15.1-alpine + command: + - --jetstream + - --store_dir + - /data + - --http_port + - "8222" + healthcheck: + test: + - CMD-SHELL + - printf 'GET /healthz?js-enabled-only=true HTTP/1.0\r\n\r\n' | nc localhost 8222 | head -1 | grep -q ' 200 ' + interval: 5s + timeout: 3s + retries: 20 + start_period: 10s + ports: + - 4222:4222 + - 127.0.0.1:8222:8222 + volumes: + - nats-data:/data + deploy: + resources: + limits: + cpus: "2" + memory: 2g + cpu_shares: 102 + rustfs: + profiles: + - server + image: rustfs/rustfs:1.0.0-beta.12 + entrypoint: + - sh + - -c + command: + - mkdir -p /data/et-storage && exec /entrypoint.sh rustfs + healthcheck: + test: + - CMD + - curl + - -fsS + - http://localhost:9000/health + interval: 5s + timeout: 3s + retries: 20 + start_period: 10s + env_file: + - secrets.env + network_mode: host + environment: + RUSTFS_ACCESS_KEY: et-storage + RUSTFS_VOLUMES: /data + volumes: + - rustfs-data:/data + deploy: + resources: + limits: + cpus: "2" + memory: 2g + cpu_shares: 102 + hub-image-build: + profiles: + - server + - edge + build: + context: ../../../.. + dockerfile: services/ws-server/Dockerfile + scale: 0 + deploy: + resources: + limits: + cpus: "2" + memory: 2g + cpu_shares: 102 + ws-server: + profiles: + - server + healthcheck: + test: + - CMD + - curl + - -fsS + - http://localhost:8080/health + interval: 5s + timeout: 3s + retries: 20 + start_period: 10s + env_file: + - secrets.env + build: + context: ../../../.. + dockerfile: verification/local/output/math1-multi-hub/Dockerfile + additional_contexts: + hub: service:hub-image-build + network_mode: host + environment: + AWS_ACCESS_KEY_ID: et-storage + AWS_ALLOW_HTTP: "true" + AWS_DEFAULT_REGION: us-east-1 + AWS_ENDPOINT: http://localhost:9000 + MODULES_PATHS: "/app/services/ws-modules/math1, + /app/services/ws-modules/math1-sender" + MODULES_ROOT: "" + OTLP_AUTH_USERNAME: root@example.com + OTLP_COLLECTOR_URL: http://127.0.0.1:5080/api/default/v1 + STORAGE_URL: s3://et-storage + WS_BACKPLANE: nats://localhost:4222 + depends_on: + openobserve: + condition: service_healthy + nats: + condition: service_healthy + rustfs: + condition: service_healthy + deploy: + resources: + limits: + cpus: "2" + memory: 2g + cpu_shares: 102 + ws-server-edge: + profiles: + - edge + healthcheck: + test: + - CMD + - curl + - -fsS + - http://localhost:8080/health + interval: 5s + timeout: 3s + retries: 20 + start_period: 10s + env_file: + - secrets.env + build: + context: ../../../.. + dockerfile: verification/local/output/math1-multi-hub/Dockerfile + additional_contexts: + hub: service:hub-image-build + network_mode: host + restart: on-failure + environment: + AWS_ACCESS_KEY_ID: et-storage + AWS_ALLOW_HTTP: "true" + AWS_DEFAULT_REGION: us-east-1 + AWS_ENDPOINT: http://127.0.0.1:9000 + MODULES_PATHS: "/app/services/ws-modules/math1, + /app/services/ws-modules/math1-sender" + MODULES_ROOT: "" + OTLP_AUTH_USERNAME: root@example.com + OTLP_COLLECTOR_URL: http://127.0.0.1:5080/api/default/v1 + STORAGE_URL: s3://et-storage + WS_BACKPLANE: nats://127.0.0.1:4222 + depends_on: + openobserve: + condition: service_healthy + required: false + nats: + condition: service_healthy + required: false + rustfs: + condition: service_healthy + required: false + deploy: + resources: + limits: + cpus: "2" + memory: 2g + cpu_shares: 102 + math1-twin: + profiles: + - edge + build: + context: ../../../.. + dockerfile: services/ws-web-runner/Dockerfile + network_mode: host + environment: + RUNNER_MODULE: "@edge-toolkit/et-ws-math1" + WS_SERVER_URL: ws://localhost:8080/ws + depends_on: + ws-server-edge: + condition: service_healthy + deploy: + resources: + limits: + cpus: "2" + memory: 2g + cpu_shares: 102 + math1-trigger: + profiles: + - server + build: + context: ../../../.. + dockerfile: services/ws-web-runner/Dockerfile + network_mode: host + environment: + RUNNER_MODULE: "@edge-toolkit/et-ws-math1-sender" + WS_SERVER_URL: ws://localhost:8080/ws + depends_on: + ws-server: + condition: service_healthy + deploy: + resources: + limits: + cpus: "2" + memory: 2g + cpu_shares: 102 +volumes: + nats-data: {} + openobserve-data: {} + rustfs-data: {} diff --git a/verification/local/output/math1-multi-hub/dekit.yaml b/verification/local/output/math1-multi-hub/dekit.yaml new file mode 100644 index 00000000..714ed54b --- /dev/null +++ b/verification/local/output/math1-multi-hub/dekit.yaml @@ -0,0 +1,99 @@ +defaults: + stop: SIGINT +tasks: + openobserve: + cmd: + - mise + - run + - openobserve + tags: + - server + ready: + http: http://localhost:5080/healthz + stop: + cmd: + - docker + - stop + - openobserve-math1-multi-hub + timeout: 30s + nats: + cmd: + - mise + - run + - nats + tags: + - server + ready: + http: http://localhost:8222/healthz?js-enabled-only=true + stop: + cmd: + - docker + - stop + - nats-math1-multi-hub + timeout: 30s + rustfs: + cmd: + - mise + - run + - rustfs + tags: + - server + ready: + http: http://localhost:9000/health + stop: + cmd: + - docker + - stop + - rustfs-math1-multi-hub + timeout: 30s + ws-server: + cmd: + - mise + - run + - ws-server + deps: + - openobserve + - nats + - rustfs + tags: + - server + ready: + http: http://localhost:8080/health + ws-server-edge: + cmd: + - mise + - run + - ws-server-edge + tags: + - edge + ready: + http: http://localhost:8080/health + autorestart: on-failure + math1-twin: + cmd: + - mise + - run + - math1-twin + deps: + - ws-server-edge + tags: + - edge + math1-trigger: + cmd: + - mise + - run + - math1-trigger + deps: + - ws-server + tags: + - server + open-o2: + type: job + cmd: + - mise + - run + - open-o2 + deps: + - openobserve + tags: + - server diff --git a/verification/local/output/math1-multi-hub/k3s.yaml b/verification/local/output/math1-multi-hub/k3s.yaml new file mode 100644 index 00000000..8c5a19ec --- /dev/null +++ b/verification/local/output/math1-multi-hub/k3s.yaml @@ -0,0 +1,656 @@ +apiVersion: v1 +kind: Namespace +metadata: + name: et-math1-multi-hub +--- +apiVersion: v1 +kind: ConfigMap +data: + RUST_LOG: warn + ZO_DATA_DIR: /data + ZO_ROOT_USER_EMAIL: root@example.com +metadata: + name: openobserve-config + namespace: et-math1-multi-hub +--- +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: openobserve-data + namespace: et-math1-multi-hub +spec: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 1Gi +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + labels: + app: openobserve + name: openobserve + namespace: et-math1-multi-hub +spec: + replicas: 1 + selector: + matchLabels: + app: openobserve + template: + metadata: + labels: + app: openobserve + spec: + containers: + - envFrom: + - configMapRef: + name: openobserve-config + - secretRef: + name: et-math1-multi-hub-secrets + image: openobserve/openobserve:v1.1.1 + imagePullPolicy: IfNotPresent + name: openobserve + ports: + - containerPort: 5080 + readinessProbe: + exec: + command: + - /openobserve + - node + - status + failureThreshold: 20 + periodSeconds: 5 + resources: + limits: + cpu: "2" + memory: 2Gi + requests: + cpu: 100m + memory: 128Mi + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + readOnlyRootFilesystem: true + runAsNonRoot: true + volumeMounts: + - mountPath: /data + name: openobserve-data + - mountPath: /tmp + name: openobserve-tmp + nodeSelector: + edge-toolkit/machine: server + securityContext: + fsGroup: 10001 + runAsGroup: 10001 + runAsNonRoot: true + runAsUser: 10001 + seccompProfile: + type: RuntimeDefault + volumes: + - name: openobserve-data + persistentVolumeClaim: + claimName: openobserve-data + - emptyDir: {} + name: openobserve-tmp +--- +apiVersion: v1 +kind: Service +metadata: + labels: + app: openobserve + name: openobserve + namespace: et-math1-multi-hub +spec: + ports: + - name: http + port: 5080 + targetPort: 5080 + selector: + app: openobserve +--- +apiVersion: v1 +kind: ConfigMap +data: + nats-server.conf: | + port: 4222 + monitor_port: 8222 + jetstream { + store_dir: /data + } +metadata: + name: nats-config + namespace: et-math1-multi-hub +--- +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: nats-data + namespace: et-math1-multi-hub +spec: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 1Gi +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + labels: + app: nats + name: nats + namespace: et-math1-multi-hub +spec: + replicas: 1 + selector: + matchLabels: + app: nats + template: + metadata: + labels: + app: nats + spec: + containers: + - image: nats:2.15.1-alpine + imagePullPolicy: IfNotPresent + name: nats + ports: + - containerPort: 4222 + - containerPort: 8222 + readinessProbe: + failureThreshold: 20 + httpGet: + path: /healthz?js-enabled-only=true + port: 8222 + periodSeconds: 5 + resources: + limits: + cpu: "2" + memory: 2Gi + requests: + cpu: 100m + memory: 128Mi + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + readOnlyRootFilesystem: true + runAsNonRoot: true + volumeMounts: + - mountPath: /data + name: nats-data + - mountPath: /etc/nats + name: nats-config + nodeSelector: + edge-toolkit/machine: server + securityContext: + fsGroup: 10001 + runAsGroup: 10001 + runAsNonRoot: true + runAsUser: 10001 + seccompProfile: + type: RuntimeDefault + volumes: + - name: nats-data + persistentVolumeClaim: + claimName: nats-data + - configMap: + name: nats-config + name: nats-config +--- +apiVersion: v1 +kind: Service +metadata: + labels: + app: nats + name: nats + namespace: et-math1-multi-hub +spec: + ports: + - name: client + port: 4222 + targetPort: 4222 + - name: monitor + port: 8222 + targetPort: 8222 + selector: + app: nats +--- +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: rustfs-data + namespace: et-math1-multi-hub +spec: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 1Gi +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + labels: + app: rustfs + name: rustfs + namespace: et-math1-multi-hub +spec: + replicas: 1 + selector: + matchLabels: + app: rustfs + template: + metadata: + labels: + app: rustfs + spec: + containers: + - command: + - sh + - -c + - mkdir -p /data/et-storage && exec /entrypoint.sh rustfs + env: + - name: RUSTFS_ACCESS_KEY + value: et-storage + - name: RUSTFS_VOLUMES + value: /data + envFrom: + - secretRef: + name: et-math1-multi-hub-secrets + image: rustfs/rustfs:1.0.0-beta.12 + imagePullPolicy: IfNotPresent + name: rustfs + ports: + - containerPort: 9000 + readinessProbe: + failureThreshold: 20 + httpGet: + path: /health + port: 9000 + periodSeconds: 5 + resources: + limits: + cpu: "2" + memory: 2Gi + requests: + cpu: 100m + memory: 128Mi + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + readOnlyRootFilesystem: true + runAsNonRoot: true + volumeMounts: + - mountPath: /data + name: rustfs-data + - mountPath: /logs + name: rustfs-logs + - mountPath: /tmp + name: rustfs-tmp + nodeSelector: + edge-toolkit/machine: server + securityContext: + fsGroup: 10001 + runAsGroup: 10001 + runAsNonRoot: true + runAsUser: 10001 + seccompProfile: + type: RuntimeDefault + volumes: + - name: rustfs-data + persistentVolumeClaim: + claimName: rustfs-data + - emptyDir: {} + name: rustfs-logs + - emptyDir: {} + name: rustfs-tmp +--- +apiVersion: v1 +kind: Service +metadata: + labels: + app: rustfs + name: rustfs + namespace: et-math1-multi-hub +spec: + ports: + - name: s3 + port: 9000 + targetPort: 9000 + selector: + app: rustfs +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + labels: + app: ws-server + name: ws-server + namespace: et-math1-multi-hub +spec: + replicas: 1 + selector: + matchLabels: + app: ws-server + template: + metadata: + labels: + app: ws-server + spec: + containers: + - env: + - name: AWS_ACCESS_KEY_ID + value: et-storage + - name: AWS_ALLOW_HTTP + value: "true" + - name: AWS_DEFAULT_REGION + value: us-east-1 + - name: AWS_ENDPOINT + value: http://rustfs:9000 + - name: MODULES_PATHS + value: /app/services/ws-modules/math1,/app/services/ws-modules/math1-sender + - name: MODULES_ROOT + value: "" + - name: OTLP_AUTH_USERNAME + value: root@example.com + - name: OTLP_COLLECTOR_URL + value: http://openobserve:5080/api/default/v1 + - name: STORAGE_URL + value: s3://et-storage + - name: TLS_CERT_FILE + value: /app/runtime/cert.pem + - name: TLS_KEY_FILE + value: /app/runtime/key.pem + - name: WS_AGENT_REGISTRY + value: /app/runtime/registry.yaml + - name: WS_BACKPLANE + value: nats://nats:4222 + envFrom: + - secretRef: + name: et-math1-multi-hub-secrets + image: et-ws-server-math1-multi-hub:latest + imagePullPolicy: IfNotPresent + name: ws-server + ports: + - containerPort: 8080 + - containerPort: 8443 + readinessProbe: + failureThreshold: 20 + httpGet: + path: /health + port: 8080 + periodSeconds: 5 + resources: + limits: + cpu: "2" + memory: 2Gi + requests: + cpu: 100m + memory: 128Mi + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + readOnlyRootFilesystem: true + runAsNonRoot: true + volumeMounts: + - mountPath: /app/runtime + name: ws-server-runtime + nodeSelector: + edge-toolkit/machine: server + securityContext: + fsGroup: 10001 + runAsGroup: 10001 + runAsNonRoot: true + runAsUser: 10001 + seccompProfile: + type: RuntimeDefault + volumes: + - emptyDir: {} + name: ws-server-runtime +--- +apiVersion: v1 +kind: Service +metadata: + labels: + app: ws-server + name: ws-server + namespace: et-math1-multi-hub +spec: + ports: + - name: ws + port: 8080 + targetPort: 8080 + - name: wss + port: 8443 + targetPort: 8443 + selector: + app: ws-server +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + labels: + app: ws-server-edge + name: ws-server-edge + namespace: et-math1-multi-hub +spec: + replicas: 1 + selector: + matchLabels: + app: ws-server-edge + template: + metadata: + labels: + app: ws-server-edge + spec: + containers: + - env: + - name: AWS_ACCESS_KEY_ID + value: et-storage + - name: AWS_ALLOW_HTTP + value: "true" + - name: AWS_DEFAULT_REGION + value: us-east-1 + - name: AWS_ENDPOINT + value: http://rustfs:9000 + - name: MODULES_PATHS + value: /app/services/ws-modules/math1,/app/services/ws-modules/math1-sender + - name: MODULES_ROOT + value: "" + - name: OTLP_AUTH_USERNAME + value: root@example.com + - name: OTLP_COLLECTOR_URL + value: http://openobserve:5080/api/default/v1 + - name: STORAGE_URL + value: s3://et-storage + - name: TLS_CERT_FILE + value: /app/runtime/cert.pem + - name: TLS_KEY_FILE + value: /app/runtime/key.pem + - name: WS_AGENT_REGISTRY + value: /app/runtime/registry.yaml + - name: WS_BACKPLANE + value: nats://nats:4222 + envFrom: + - secretRef: + name: et-math1-multi-hub-secrets + image: et-ws-server-math1-multi-hub:latest + imagePullPolicy: IfNotPresent + name: ws-server-edge + ports: + - containerPort: 8080 + - containerPort: 8443 + readinessProbe: + failureThreshold: 20 + httpGet: + path: /health + port: 8080 + periodSeconds: 5 + resources: + limits: + cpu: "2" + memory: 2Gi + requests: + cpu: 100m + memory: 128Mi + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + readOnlyRootFilesystem: true + runAsNonRoot: true + volumeMounts: + - mountPath: /app/runtime + name: ws-server-runtime + nodeSelector: + edge-toolkit/machine: edge + securityContext: + fsGroup: 10001 + runAsGroup: 10001 + runAsNonRoot: true + runAsUser: 10001 + seccompProfile: + type: RuntimeDefault + volumes: + - emptyDir: {} + name: ws-server-runtime +--- +apiVersion: v1 +kind: Service +metadata: + labels: + app: ws-server-edge + name: ws-server-edge + namespace: et-math1-multi-hub +spec: + ports: + - name: ws + port: 8080 + targetPort: 8080 + - name: wss + port: 8443 + targetPort: 8443 + selector: + app: ws-server-edge +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + labels: + app: math1-twin + name: math1-twin + namespace: et-math1-multi-hub +spec: + replicas: 1 + selector: + matchLabels: + app: math1-twin + template: + metadata: + labels: + app: math1-twin + spec: + containers: + - env: + - name: RUNNER_MODULE + value: "@edge-toolkit/et-ws-math1" + - name: WS_SERVER_URL + value: ws://ws-server-edge:8080/ws + image: et-ws-web-runner:latest + imagePullPolicy: IfNotPresent + name: math1-twin + resources: + limits: + cpu: "2" + memory: 2Gi + requests: + cpu: 100m + memory: 128Mi + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + readOnlyRootFilesystem: true + runAsNonRoot: true + volumeMounts: + - mountPath: /tmp + name: runner-tmp + nodeSelector: + edge-toolkit/machine: edge + securityContext: + fsGroup: 10001 + runAsGroup: 10001 + runAsNonRoot: true + runAsUser: 10001 + seccompProfile: + type: RuntimeDefault + volumes: + - emptyDir: {} + name: runner-tmp +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + labels: + app: math1-trigger + name: math1-trigger + namespace: et-math1-multi-hub +spec: + replicas: 1 + selector: + matchLabels: + app: math1-trigger + template: + metadata: + labels: + app: math1-trigger + spec: + containers: + - env: + - name: RUNNER_MODULE + value: "@edge-toolkit/et-ws-math1-sender" + - name: WS_SERVER_URL + value: ws://ws-server:8080/ws + image: et-ws-web-runner:latest + imagePullPolicy: IfNotPresent + name: math1-trigger + resources: + limits: + cpu: "2" + memory: 2Gi + requests: + cpu: 100m + memory: 128Mi + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + readOnlyRootFilesystem: true + runAsNonRoot: true + volumeMounts: + - mountPath: /tmp + name: runner-tmp + nodeSelector: + edge-toolkit/machine: server + securityContext: + fsGroup: 10001 + runAsGroup: 10001 + runAsNonRoot: true + runAsUser: 10001 + seccompProfile: + type: RuntimeDefault + volumes: + - emptyDir: {} + name: runner-tmp diff --git a/verification/local/output/math1-multi-hub/mise.toml b/verification/local/output/math1-multi-hub/mise.toml new file mode 100644 index 00000000..0d3dc559 --- /dev/null +++ b/verification/local/output/math1-multi-hub/mise.toml @@ -0,0 +1,112 @@ +[env] +OTLP_AUTH_USERNAME = "root@example.com" + +[env._] +file = "secrets.env" + +[tasks.generated-scenario] +depends = ["math1-trigger", "math1-twin", "nats", "openobserve", "rustfs", "ws-server"] +description = "Run generated scenario for math1-multi-hub" + +[tasks.machine-edge] +depends = ["math1-twin", "ws-server-edge"] +description = "Run what machine edge runs" + +[tasks.machine-server] +depends = ["math1-trigger", "nats", "openobserve", "rustfs", "ws-server"] +description = "Run what machine server runs" + +[tasks.math1-trigger] +description = "Run @edge-toolkit/et-ws-math1-sender in the web runner" +dir = "../../../.." +run = """ +cargo run --quiet -p et-ws-web-runner +""" + +[tasks.math1-trigger.env] +RUNNER_MODULE = "@edge-toolkit/et-ws-math1-sender" +WS_SERVER_URL = "ws://localhost:8080/ws" + +[tasks.math1-twin] +description = "Run @edge-toolkit/et-ws-math1 in the web runner" +dir = "../../../.." +run = """ +cargo run --quiet -p et-ws-web-runner +""" + +[tasks.math1-twin.env] +RUNNER_MODULE = "@edge-toolkit/et-ws-math1" +WS_SERVER_URL = "ws://localhost:8080/ws" + +[tasks.nats] +description = "Run the NATS server the hubs share" +run = """ +image=nats:2.15.1-alpine +container=nats-math1-multi-hub +ports="-p 4222:4222 -p 127.0.0.1:8222:8222" +# $ports is a word-split flag list by design; do not quote it. +docker run --rm --name "$container" $ports -v "$container:/data" "$image" --jetstream --store_dir /data --http_port 8222 +""" + +[tasks.open-o2] +description = "Open the OpenObserve UI" +run = "open http://localhost:5080/" + +[tasks.openobserve] +alias = "o2" +run = """ +image=openobserve/openobserve:v1.1.1 +container=openobserve-math1-multi-hub +settings="-e RUST_LOG=warn -e ZO_ROOT_USER_EMAIL=root@example.com" +# $settings is a word-split flag list by design; do not quote it. +docker run --rm --name "$container" -p 127.0.0.1:5080:5080 $settings -e ZO_ROOT_USER_PASSWORD "$image" +""" + +[tasks.rustfs] +description = "Run the object store the hubs share" +run = """ +image=rustfs/rustfs:1.0.0-beta.12 +container=rustfs-math1-multi-hub +settings="-e RUSTFS_ACCESS_KEY=et-storage -e RUSTFS_VOLUMES=/data -e RUSTFS_SECRET_KEY" +start='mkdir -p /data/et-storage && exec /entrypoint.sh rustfs' +# $settings is a word-split flag list by design; do not quote it. +docker run --rm --name "$container" -p 9000:9000 -v "$container:/data" $settings --entrypoint sh "$image" -c "$start" +""" + +[tasks.ws-server] +description = "Run the WebSocket server" +dir = "../../../../services/ws-server" +run = """ +MODULES_PATHS="../ws-modules/math1, ../ws-modules/math1-sender" +export MODULES_PATHS +cargo run +""" + +[tasks.ws-server.env] +AWS_ACCESS_KEY_ID = "et-storage" +AWS_ALLOW_HTTP = "true" +AWS_DEFAULT_REGION = "us-east-1" +AWS_ENDPOINT = "http://localhost:9000" +STORAGE_URL = "s3://et-storage" +WS_BACKPLANE = "nats://localhost:4222" + +[tasks.ws-server-edge] +description = "Run the WebSocket server on edge" +dir = "../../../../services/ws-server" +run = """ +MODULES_PATHS="../ws-modules/math1, ../ws-modules/math1-sender" +export MODULES_PATHS +cargo run +""" + +[tasks.ws-server-edge.env] +AWS_ACCESS_KEY_ID = "et-storage" +AWS_ALLOW_HTTP = "true" +AWS_DEFAULT_REGION = "us-east-1" +AWS_ENDPOINT = "http://127.0.0.1:9000" +STORAGE_URL = "s3://et-storage" +WS_BACKPLANE = "nats://127.0.0.1:4222" + +[tools] +"aqua:F1bonacc1/process-compose" = "latest" +"cargo:open" = "latest" diff --git a/verification/local/output/math1-multi-hub/process-compose.yaml b/verification/local/output/math1-multi-hub/process-compose.yaml new file mode 100644 index 00000000..b689e26c --- /dev/null +++ b/verification/local/output/math1-multi-hub/process-compose.yaml @@ -0,0 +1,107 @@ +version: "0.5" +processes: + openobserve: + command: mise run openobserve + namespace: server + shutdown: + signal: 2 + command: docker stop openobserve-math1-multi-hub + timeout_seconds: 30 + readiness_probe: + http_get: + host: localhost + scheme: http + path: /healthz + port: 5080 + period_seconds: 2 + failure_threshold: 1800 + nats: + command: mise run nats + namespace: server + shutdown: + signal: 2 + command: docker stop nats-math1-multi-hub + timeout_seconds: 30 + readiness_probe: + http_get: + host: localhost + scheme: http + path: /healthz?js-enabled-only=true + port: 8222 + period_seconds: 2 + failure_threshold: 1800 + rustfs: + command: mise run rustfs + namespace: server + shutdown: + signal: 2 + command: docker stop rustfs-math1-multi-hub + timeout_seconds: 30 + readiness_probe: + http_get: + host: localhost + scheme: http + path: /health + port: 9000 + period_seconds: 2 + failure_threshold: 1800 + ws-server: + command: mise run ws-server + depends_on: + openobserve: + condition: process_healthy + nats: + condition: process_healthy + rustfs: + condition: process_healthy + namespace: server + shutdown: + signal: 2 + readiness_probe: + http_get: + host: localhost + scheme: http + path: /health + port: 8080 + period_seconds: 2 + failure_threshold: 1800 + ws-server-edge: + command: mise run ws-server-edge + namespace: edge + shutdown: + signal: 2 + readiness_probe: + http_get: + host: localhost + scheme: http + path: /health + port: 8080 + period_seconds: 2 + failure_threshold: 1800 + availability: + restart: on_failure + math1-twin: + command: mise run math1-twin + depends_on: + ws-server-edge: + condition: process_healthy + namespace: edge + shutdown: + signal: 2 + math1-trigger: + command: mise run math1-trigger + depends_on: + ws-server: + condition: process_healthy + namespace: server + shutdown: + signal: 2 + open-o2: + command: mise run open-o2 + depends_on: + openobserve: + condition: process_healthy + namespace: server + shutdown: + signal: 2 + disabled: true diff --git a/verification/local/output/math1-multi-hub/secrets.env b/verification/local/output/math1-multi-hub/secrets.env new file mode 100644 index 00000000..f0694a8a --- /dev/null +++ b/verification/local/output/math1-multi-hub/secrets.env @@ -0,0 +1,9 @@ +# Generated from the scenario input -- regenerating this deployment rewrites it. +# skipcq: SCT-A000 -- derived from the scenario input, not an independently generated secret +ZO_ROOT_USER_PASSWORD=Dolorem.labore4334 +# skipcq: SCT-A000 -- derived from the scenario input, not an independently generated secret +OTLP_AUTH_PASSWORD=Dolorem.labore4334 +# skipcq: SCT-A000 -- derived from the scenario input, not an independently generated secret +AWS_SECRET_ACCESS_KEY=Dolorem.labore4334 +# skipcq: SCT-A000 -- derived from the scenario input, not an independently generated secret +RUSTFS_SECRET_KEY=Dolorem.labore4334 diff --git a/verification/local/output/math1-multi/README.md b/verification/local/output/math1-multi/README.md index 1b5e104b..619fb4f9 100644 --- a/verification/local/output/math1-multi/README.md +++ b/verification/local/output/math1-multi/README.md @@ -50,6 +50,9 @@ of it at the same path -- and start that machine's share of the scenario there. `server` runs OpenObserve and `ws-server`, and the runners on every other machine reach the hub at `127.0.0.1`. +The machines' addresses are loopback, which plays every machine on this one host. To run them on +separate machines, give each its reachable address in the scenario input and regenerate first. + On `server`: ```bash @@ -93,6 +96,9 @@ until its tag is started. `server` runs OpenObserve and `ws-server`, and the runners on every other machine reach the hub at `127.0.0.1`. +The machines' addresses are loopback, which plays every machine on this one host. To run them on +separate machines, give each its reachable address in the scenario input and regenerate first. + On `server`: ```bash @@ -131,6 +137,9 @@ of it at the same path -- and start only that machine's namespace there. `server` runs OpenObserve and `ws-server`, and the runners on every other machine reach the hub at `127.0.0.1`. +The machines' addresses are loopback, which plays every machine on this one host. To run them on +separate machines, give each its reachable address in the scenario input and regenerate first. + On `server`: ```bash @@ -148,7 +157,7 @@ process-compose up --no-server -f process-compose.yaml -n edge From this directory, start the scenario with: ```bash -docker compose up --build +docker compose --profile server --profile edge up --build ``` The compose stack starts OpenObserve and builds `ws-server` in two layers: the module-less hub @@ -164,7 +173,7 @@ OpenObserve is available at . Stop the scenario with: ```bash -docker compose down +docker compose --profile server --profile edge down ``` ### Run Across Machines With Docker Compose @@ -174,6 +183,9 @@ of it at the same path -- and start only that machine's profile there. `server` runs OpenObserve and `ws-server`, and the runners on every other machine reach the hub at `127.0.0.1`. +The machines' addresses are loopback, which plays every machine on this one host. To run them on +separate machines, give each its reachable address in the scenario input and regenerate first. + On `server`: ```bash @@ -236,6 +248,9 @@ kubectl get pods -n "$ns" --watch ### Run Across Machines With k3s +The machines' addresses are loopback, which plays every machine on this one host. To run them on +separate machines, give each its reachable address in the scenario input and regenerate first. + The machines have to form one k3s cluster first, each labelled with the machine name the manifests pin pods to. With `k3s` installed on every machine, start the server on `server`. It stays in the foreground, so read the token the others join with from a second shell once it is up: diff --git a/verification/published/output/math1-multi/README.md b/verification/published/output/math1-multi/README.md index 052ddad3..d175aae9 100644 --- a/verification/published/output/math1-multi/README.md +++ b/verification/published/output/math1-multi/README.md @@ -63,6 +63,9 @@ of it at the same path -- and start that machine's share of the scenario there. `server` runs OpenObserve and `ws-server`, and the runners on every other machine reach the hub at `127.0.0.1`. +The machines' addresses are loopback, which plays every machine on this one host. To run them on +separate machines, give each its reachable address in the scenario input and regenerate first. + On `server`: ```bash @@ -106,6 +109,9 @@ until its tag is started. `server` runs OpenObserve and `ws-server`, and the runners on every other machine reach the hub at `127.0.0.1`. +The machines' addresses are loopback, which plays every machine on this one host. To run them on +separate machines, give each its reachable address in the scenario input and regenerate first. + On `server`: ```bash @@ -142,6 +148,9 @@ of it at the same path -- and start only that machine's namespace there. `server` runs OpenObserve and `ws-server`, and the runners on every other machine reach the hub at `127.0.0.1`. +The machines' addresses are loopback, which plays every machine on this one host. To run them on +separate machines, give each its reachable address in the scenario input and regenerate first. + On `server`: ```bash @@ -159,7 +168,7 @@ process-compose up --no-server -f process-compose.yaml -n edge From this directory, start the scenario with: ```bash -docker compose up --build +docker compose --profile server --profile edge up --build ``` The compose stack starts OpenObserve and builds one image: the `Dockerfile` in this directory, @@ -174,7 +183,7 @@ OpenObserve is available at . Stop the scenario with: ```bash -docker compose down +docker compose --profile server --profile edge down ``` ### Run Across Machines With Docker Compose @@ -184,6 +193,9 @@ of it at the same path -- and start only that machine's profile there. `server` runs OpenObserve and `ws-server`, and the runners on every other machine reach the hub at `127.0.0.1`. +The machines' addresses are loopback, which plays every machine on this one host. To run them on +separate machines, give each its reachable address in the scenario input and regenerate first. + On `server`: ```bash @@ -242,6 +254,9 @@ kubectl get pods -n "$ns" --watch ### Run Across Machines With k3s +The machines' addresses are loopback, which plays every machine on this one host. To run them on +separate machines, give each its reachable address in the scenario input and regenerate first. + The machines have to form one k3s cluster first, each labelled with the machine name the manifests pin pods to. With `k3s` installed on every machine, start the server on `server`. It stays in the foreground, so read the token the others join with from a second shell once it is up: