diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 0b396c11..3d843762 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -378,13 +378,26 @@ jobs: load: true tags: ${{ env.STACK_IMAGE }}:smoke-test labels: ${{ steps.meta.outputs.labels }} + # `#` inside a `|` block scalar is literal content, not a YAML + # comment, and build-push-action parses the input without a comment + # option — so a note in there reaches buildx as a build arg. It goes + # here instead, as it does in `build.yml`. + # + # The generic shape: the agent CLIs the product exists to run, and + # none of the maintainer's estate integrations. A release is the + # public artifact, so it carries what a consumer needs to run a + # session and nothing that only means something inside one estate. + # This is a third shape deliberately chosen, not a drift toward + # `build.yml`'s: those dev/prod branch builds keep the estate rule + # and add cadastre + theclawbay on top of this same base. build-args: | POD_BASE_IMAGE=${{ needs.pod-base.outputs.image }} CORE_IMAGE=${{ env.IMAGE }}@${{ needs.image.outputs.digest }} VOGT_SOURCE_REF=${{ github.ref_name }} VOGT_SOURCE_SHA=${{ github.sha }} VOGT_PRODUCT_VERSION=${{ github.ref_name }} - INSTALL_CADASTRE_MCP=true + INSTALL_AI_CLIENTS=true + INSTALL_CADASTRE_MCP=false cache-from: type=registry,ref=${{ env.CACHE_IMAGE }}:stack-release cache-to: type=registry,ref=${{ env.CACHE_IMAGE }}:stack-release,mode=max @@ -398,6 +411,30 @@ jobs: docker run --rm --entrypoint vogt-verify-agent-clis "$CANDIDATE" echo "both halves are present and start" + # `vogt-verify-agent-clis` above does NOT prove the CLIs are here. It + # is a *shadow* detector: `check_tool` returns 0 when the system path + # does not exist, `check_package` returns 0 unless the binary is + # executable, and `agent-versions.resolved` is written unconditionally + # — so it is green on a CLI-free image. It answers "is a persisted + # $HOME copy shadowing the image's?", not "is there an image copy?". + # + # The release now carries the clients by decision, and a build arg + # that nothing executes is a default waiting to be forgotten — which + # is #23 exactly: INSTALL_AI_CLIENTS defaulted to false, no build + # turned it on, `vogt-dev` ran without claude or codex and nothing was + # red. Running them is the only thing that distinguishes a generic + # release from a silent reversion to the CLI-free one. + # + # Ungated, unlike `build.yml`'s equivalent: every release is the + # generic shape, so there is no ref to condition on. `flutter` and + # `theclawbay` are absent from the list on purpose — a release is the + # `lean` pod base and carries no estate integrations. + for tool in claude codex; do + docker run --rm --entrypoint "$tool" "$CANDIDATE" --version >/dev/null \ + || { echo "the release image is missing $tool" >&2; exit 1; } + done + echo "the release image carries the agent CLIs" + - uses: docker/build-push-action@10e90e3645eae34f1e60eeb005ba3a3d33f178e8 id: push with: @@ -406,13 +443,18 @@ jobs: push: true tags: ${{ steps.meta.outputs.tags }} labels: ${{ steps.meta.outputs.labels }} + # Identical to the candidate's, so the pushed image is the layers the + # smoke test just ran rather than a rebuild that resembles them. The + # two lists drifting apart is how an image gets smoke-tested in one + # shape and published in another. build-args: | POD_BASE_IMAGE=${{ needs.pod-base.outputs.image }} CORE_IMAGE=${{ env.IMAGE }}@${{ needs.image.outputs.digest }} VOGT_SOURCE_REF=${{ github.ref_name }} VOGT_SOURCE_SHA=${{ github.sha }} VOGT_PRODUCT_VERSION=${{ github.ref_name }} - INSTALL_CADASTRE_MCP=true + INSTALL_AI_CLIENTS=true + INSTALL_CADASTRE_MCP=false provenance: true sbom: true cache-from: type=registry,ref=${{ env.CACHE_IMAGE }}:stack-release diff --git a/README.md b/README.md index 3cd0bf97..dc88502d 100644 --- a/README.md +++ b/README.md @@ -27,11 +27,23 @@ the write API is isolated, so nothing there persists and no sign-in is needed: ## Run it -The stack is two Compose files layered together: the base -(`deploy/vogt.compose.yml`) defines the core service, and the engine overlay -(`deploy/engine.overlay.yml`) adds the Rust engine — the PWA front end — in -front of it. No engine image is published, so that overlay always builds one -from this checkout. +There are three shapes, and they trade containment against convenience: + +| | What it is | Web UI | Build | +|---|---|---|---| +| **Core only** | `deploy/vogt.compose.yml` — the API, hardened | no | no | +| **Core + engine** | the above + `deploy/engine.overlay.yml` | yes | the engine | +| **All-in-one** | `deploy/stack.compose.yml` — one published image | yes | no | + +The first two keep the core in a hardened container: slim base, `read_only`, +all capabilities dropped. The all-in-one is a **development pod** — writable +home, `sudo`, sshd, and the `claude` and `codex` CLIs — because it exists to +run coding agents, and an agent session needs a machine. Pick it where you +would run a dev box, not where you would run a hardened service. +[`docs/DEPLOYMENT.md`](docs/DEPLOYMENT.md) §1.1 lays the three out in full. + +No image of the engine alone is published, so the overlay path always builds +one from this checkout; the all-in-one image carries it with a core inside. The quickstart below **builds the core from this checkout** with the one-service build overlay `deploy/vogt.build.yml`, a path that works without diff --git a/deploy/stack.compose.yml b/deploy/stack.compose.yml new file mode 100644 index 00000000..f2341205 --- /dev/null +++ b/deploy/stack.compose.yml @@ -0,0 +1,168 @@ +# The all-in-one Vogt stack: core, engine and PWA in one container. +# +# cp deploy/stack.env.example deploy/.env # then fill in ENGINE_TOKEN +# openssl rand -hex 32 > deploy/vogt-core-token +# docker compose -f deploy/stack.compose.yml up -d --wait +# +# There is no `--build`: the published image already carries all three. +# +# This is a *base*, not an overlay. It stands alone and does not layer onto +# `vogt.compose.yml` — that file runs a core, and so does this image, so +# combining them would run two. +# +# ── Which shape is this, and do you want it? ──────────────────────────────── +# +# Three supported shapes, and this is the least contained of them: +# +# core only (`vogt.compose.yml`) hardened; no UI, no sessions +# core + engine (`engine.overlay.yml`) core stays hardened; engine builds +# from the checkout +# the AIO (this file) one container, no build, full pod +# +# The AIO is a **development pod**, and that is a deliberate posture rather +# than an oversight. It carries a writable home, passwordless `sudo`, an SSH +# server, a Docker CLI that talks to a socket you may mount, and the `claude` +# and `codex` agent CLIs. It cannot run `read_only`, and it does not drop +# capabilities. The core-only image is hardened by construction — slim base, +# `nologin`, `read_only`, `cap_drop: [ALL]`, one writable volume — and none of +# that survives being lifted into a pod that exists to run arbitrary agent +# sessions. `engine/Dockerfile`'s header says why the two images are kept +# separate: collapsing them would quietly delete Vogt's containment story +# under an unchanged image name. +# +# So: run this where you would run a dev box, not where you would run a +# hardened service. If you want the containment, run the core alone and put +# the engine somewhere you trust it. +# +# Nothing here names a host, a path, or a tailnet. Every value an operator must +# choose is a `${VAR}`; where one is genuinely required the reference fails the +# deploy with a message rather than guessing. + +services: + vogt: + # Digest-pin this for a real deployment (NFR-D10) — a tag can move, and + # the whole point of a signed release is that the bytes cannot. The default + # is a tag so the file is runnable as-is. + # + # Pin the *release* family (`X.Y.Z`, `latest`). The `dev-` and + # `prod-` tags in the same repository are the maintainer's own pods: + # same CLIs, plus estate integrations that address one estate's + # infrastructure and mean nothing outside it. See DEPLOYMENT.md §7. + image: ${VOGT_STACK_IMAGE:-ghcr.io/thedancingdeveloper-org/vogt-stack:0.4.0} + restart: unless-stopped + + # tini as PID 1 reaps the processes agent sessions leave behind. The + # entrypoint deliberately does *not* adopt a supervision framework, and + # this is the half of that decision the compose owns. + init: true + + # The engine is the only way in (NFR-D11). The core listens on loopback + # *inside* the container and is never published — the entrypoint enforces + # that rather than trusting this comment, refusing to start if + # `VOGT_CORE_URL` names anything but a loopback address. + # + # Loopback on the host too, unless told otherwise: this file will not put a + # pod carrying sudo and agent CLIs on a network interface because nobody + # said to. Set `ENGINE_BIND` to a real address to expose it, and put + # something that terminates TLS in front. + ports: + - "${ENGINE_BIND:-127.0.0.1}:${ENGINE_PORT:-8910}:8910" + + environment: + # The engine's own listen socket, inside the container's namespace — all + # interfaces so the published port above can reach it. Fixed, not an + # exposure decision: `ENGINE_BIND` in `.env` decides exposure, on the + # host side of the port mapping. + ENGINE_BIND: "0.0.0.0:8910" + # The engine's bearer token (at least 16 characters), required for every + # API and WebSocket call. Keep it in `deploy/.env` (git-ignored) rather + # than your shell. + ENGINE_TOKEN: "${ENGINE_TOKEN:?Set ENGINE_TOKEN in deploy/.env — the engine bearer token, at least 16 characters}" + # The address clients reach the front door at. The engine cannot + # advertise an address it is not told, so set it once the stack is + # reachable at a stable URL. + ENGINE_PUBLIC_URL: "${ENGINE_PUBLIC_URL:-}" + + # One value both starts the core and tells the engine where to proxy. + # Loopback means "the core is ours to run", so the entrypoint supervises + # `vogt init && vogt serve` here; the engine proxies `/api/vogt`, `/mcp` + # and `/ui-legacy` to the same address. Deriving one from the other is + # what stops a front door confidently pointed at a port nothing serves. + VOGT_CORE_URL: "http://127.0.0.1:8000" + # The token the engine presents on `/api/vogt`; the core adopts the same + # file at `init` if it does not already hold one. Both halves reading one + # file is the whole bootstrap mechanism — no second deploy. Brokered as a + # file, never a bare value, so it stays out of every `docker inspect`. + VOGT_CORE_TOKEN_FILE: /run/secrets/vogt_core_token + VOGT_BOOTSTRAP_CORE_TOKEN_FILE: /run/secrets/vogt_core_token + # Who that adopted token acts as, and how much it may do. Everything that + # can read the file runs as this identity, so this scope is the blast + # radius; narrow it if you do not need project writes. + VOGT_BOOTSTRAP_CORE_TOKEN_ACTOR: "${VOGT_BOOTSTRAP_CORE_TOKEN_ACTOR:-agent:engine}" + VOGT_BOOTSTRAP_CORE_TOKEN_SCOPES: "${VOGT_BOOTSTRAP_CORE_TOKEN_SCOPES:-read,work.write,project.write}" + # The engine is the front door, so `connect` and `/connection-info` + # render against the identity the engine states per request (its + # `X-Vogt-*` headers) rather than the core's own loopback address. + # `VOGT_PUBLIC_URL` is deliberately absent: when fronted, the door is the + # only thing that knows where clients arrive, and it says so per request. + VOGT_FRONTED: "true" + + # Both halves must agree about where the estate is: the core's import + # root and the engine's workspace root are the same tree, or a session + # opened "for" a project opens somewhere the registry never recorded. + # The entrypoint warns at boot if these drift apart. + VOGT_DATA_DIR: /var/lib/vogt + VOGT_IMPORT_ROOT: /home/sprooty/Working + + VOGT_LIFECYCLE_STATE_DIR: /home/sprooty/.vogt-lifecycle + VOGT_LIFECYCLE_WORKDIR: /home/sprooty/Working + VOGT_LIFECYCLE_HEALTHCHECK_URL: http://127.0.0.1:8910/readyz + VOGT_HOOK_DIR: /run/vogt/hooks + VOGT_HOOKS_REQUIRED: "${VOGT_HOOKS_REQUIRED:-false}" + + # Named volumes, not host binds. Docker seeds a fresh named volume from the + # image, so the `sprooty` ownership the image built is preserved — a host + # directory would arrive root-owned and break the pod. No host paths appear + # in this file by design. + volumes: + # The pod's writable home: agent state, caches, session scratch, and the + # `Working` tree sessions run in. + - engine-home:/home/sprooty + # The core's SQLite databases and backups. Separate from the home volume + # so the data outlives a pod you decide to reset. + - vogt-data:/var/lib/vogt + + # `/readyz` is the aggregate. It reports the core's state but deliberately + # stays ready when the core is absent — restarting the container would not + # revive a core and would kill every live terminal (FR-E9). + healthcheck: + test: ["CMD", "/usr/local/bin/vogt-lifecycle", "health"] + interval: 30s + timeout: 10s + retries: 3 + start_period: 60s + + logging: + driver: json-file + options: + max-size: "10m" + max-file: "3" + + secrets: + - vogt_core_token + +volumes: + engine-home: + vogt-data: + +secrets: + # The core token, shared by both halves inside the container. Create it + # beside this file before the first deploy: + # + # openssl rand -hex 32 > deploy/vogt-core-token + # + # A relative path, so nothing here is tied to one host. Legitimately empty + # until you mint one — an empty file just leaves `/api/vogt` answering 401, + # which is the pre-bootstrap behaviour rather than a broken one. + vogt_core_token: + file: ./vogt-core-token diff --git a/deploy/stack.env.example b/deploy/stack.env.example new file mode 100644 index 00000000..6e4b454c --- /dev/null +++ b/deploy/stack.env.example @@ -0,0 +1,66 @@ +# Settings for the all-in-one stack. Copy to deploy/.env before starting it: +# +# cp deploy/stack.env.example deploy/.env +# docker compose -f deploy/stack.compose.yml up -d --wait +# +# Separate from `.env.example` on purpose: that file configures the core-only +# base and its optional engine overlay, and half of it (`VOGT_PORT`, +# `VOGT_UID`, `VOGT_IMAGE`) means nothing to the AIO, which publishes one port +# and runs its core on loopback inside the container. Use one or the other, +# not both. + +# ── Required ──────────────────────────────────────────────────────────────── + +# The engine's bearer token, required for every API and WebSocket call. At +# least 16 characters. The stack refuses to start without it rather than +# generating one, because a token this file invented would be a token nobody +# knows they are trusting. Mint one with: +# +# ENGINE_TOKEN=$(openssl rand -hex 24) +ENGINE_TOKEN= + +# The core token is *not* an env var — it is a file both halves read, so it +# stays out of `docker inspect`. Create it beside this file before the first +# deploy: +# +# openssl rand -hex 32 > deploy/vogt-core-token +# +# Leaving it empty is legitimate: `/api/vogt` then answers 401 until a token +# is minted by hand, which is the pre-bootstrap behaviour rather than a break. + +# ── Exposure ──────────────────────────────────────────────────────────────── + +# Which host interface the engine is published on. Loopback by default — this +# is a development pod carrying sudo, sshd and agent CLIs, so it does not go +# onto a network interface because nobody said to. Set a real address only +# when you mean to expose it, and put something that terminates TLS in front. +ENGINE_BIND=127.0.0.1 + +# Host port. Change it if 8910 is taken. +ENGINE_PORT=8910 + +# The URL clients actually reach the stack at. Optional to start with, but the +# engine cannot advertise an address it has not been told, so set it once the +# stack has a stable one — `connect` and `/connection-info` render against it. +ENGINE_PUBLIC_URL= + +# ── The image ─────────────────────────────────────────────────────────────── + +# Pin a digest in anything you would call a deployment: a tag can move, and a +# signed release exists so the bytes cannot. Pin the release family +# (`X.Y.Z`, `latest`) — the `dev-` / `prod-` tags in the same +# repository are the maintainer's own pods, carrying estate integrations that +# address one estate's infrastructure. See docs/DEPLOYMENT.md §7. +# VOGT_STACK_IMAGE=ghcr.io/thedancingdeveloper-org/vogt-stack@sha256:... + +# ── Optional ──────────────────────────────────────────────────────────────── + +# Who the bootstrapped core token acts as, and how much it may do. Everything +# in the container that can read the token file runs as this identity, so this +# scope is the blast radius; narrow it if you do not need project writes. +VOGT_BOOTSTRAP_CORE_TOKEN_ACTOR=agent:engine +VOGT_BOOTSTRAP_CORE_TOKEN_SCOPES=read,work.write,project.write + +# Whether a missing lifecycle hook is fatal. Off by default: hooks are a +# deployment's extension point, and a deployment that installs none is normal. +VOGT_HOOKS_REQUIRED=false diff --git a/docs/CUSTOMISATION.md b/docs/CUSTOMISATION.md index a901c7b8..8c140c3f 100644 --- a/docs/CUSTOMISATION.md +++ b/docs/CUSTOMISATION.md @@ -425,9 +425,10 @@ It adds one service and some configuration. It does not rebuild, repin, or restate the core — that is the published image, unmodified, which is what makes "a customised deployment is the public image plus configuration" a claim you can check rather than one you have to believe -(`tests/test_public_delivery.py` checks it). No engine image is published, so -the overlay always builds one from this checkout; it carries no host paths, no -tailnet and no maintainer integrations, so it runs on any host unchanged. +(`tests/test_public_delivery.py` checks it). No image of the engine alone is +published, so the overlay always builds one from this checkout; it carries no +host paths, no tailnet and no maintainer integrations, so it runs on any host +unchanged. The maintainer's own estate layers its host mounts, tailnet, and secret integrations on top of this same base. That overlay is not tracked in this @@ -437,6 +438,43 @@ repository instead. Treat `engine.overlay.yml` as the pattern: every estate-specific value it would add is an environment value or a mount an operator supplies, never a default baked into a file a stranger clones. +## Extending the stack image + +The same claim holds one level up. `engine/Dockerfile`'s final stage says of +the core it lifts in that "the private path is the public path plus +configuration — now it is a digest", and the all-in-one `vogt-stack` image is +where that stops being a statement about one image and becomes the deployment +model. A deployment that needs tools the public image does not carry adds them +in a layer of its own: + +```dockerfile +FROM ghcr.io/thedancingdeveloper-org/vogt-stack@sha256:... + +# The image ends as `USER sprooty`, so an extending build must take root back +# for anything that installs, and hand it over again at the end. Forgetting the +# second half yields a pod running as root, which is a different container from +# the one whose digest you pinned. +USER root +RUN apt-get update \ + && apt-get install -y --no-install-recommends your-tool \ + && rm -rf /var/lib/apt/lists/* +USER sprooty +``` + +That is not a hypothetical: it is exactly how the maintainer's own `vogt-dev` +and `vogt-prod` images are built — the public digest, plus the estate's +integrations, plus nothing else. Whatever a private deployment adds is +therefore visible as a Dockerfile a few lines long, and what it *started* from +is a digest anyone can pull and verify. A private image built any other way +would be an assertion about the public one rather than a layer on it. + +Two things about the base are worth knowing before you extend it. It is a +development pod, not a hardened service image (see +[`DEPLOYMENT.md`](DEPLOYMENT.md) §1.1), so it will not accept `read_only` and +its uid is fixed. And it contains a core, so a layer that adds a *second* one +— or a compose file that layers `stack.compose.yml` onto +`vogt.compose.yml` — runs two. + If your deployment needs something none of these layers reach, that is worth an issue. The generic base is only generic if the customisations people actually need are supported ones. Notes about a particular host of your own diff --git a/docs/DEPLOYMENT.md b/docs/DEPLOYMENT.md index 1a3b0eed..80c50932 100644 --- a/docs/DEPLOYMENT.md +++ b/docs/DEPLOYMENT.md @@ -37,16 +37,52 @@ A core with nothing else configured is a complete, supported product. **The session engine and PWA** (optional) — a Rust server under `engine/` that embeds the web UI from `web/`, owns the terminals a work item's session runs in, hosts the voice assistant, and can act as the front door in front -of the core. It is built from `engine/Dockerfile`; there is no published -image for it, but a generic Compose overlay builds and runs one beside the -core (§3.2). Without it the core's `session.*` operations report that no -engine is configured, and nothing else changes. +of the core. It is built from `engine/Dockerfile`. No image of the engine +*alone* is published, but it does not have to be built: the all-in-one +`vogt-stack` image carries the engine, the PWA and the core together (§1.1). +A generic Compose overlay builds one beside the core instead, if you would +rather (§3.2). Without the engine, the core's `session.*` operations report +that no engine is configured, and nothing else changes. **The mobile shell** (optional) — a Capacitor wrapper under `mobile/` around the same PWA. Nothing server-side depends on it. It loads the deployed front door, so ordinary server and PWA releases reach installed phones without an APK rebuild; the production APK procedure is in §7.1. +### 1.1 Which shape do you want? + +Three supported ways to run this, and the difference between them is +containment, not features. State it before you pick, because moving between +them later means moving data. + +| | Core only | Core + engine | All-in-one (`vogt-stack`) | +|---|---|---|---| +| Files | [`vogt.compose.yml`](../deploy/vogt.compose.yml) | the above + [`engine.overlay.yml`](../deploy/engine.overlay.yml) | [`stack.compose.yml`](../deploy/stack.compose.yml) | +| Containers | 1 | 2 | 1 | +| Web UI, terminals, sessions | no | yes | yes | +| Build required | no | yes (the engine) | no | +| Core posture | hardened | hardened | inside the pod | +| Engine posture | — | development pod | development pod | +| Agent CLIs | no | if you build them in | `claude`, `codex` included | + +**Hardened** means what it says: a slim base with `git` and nothing else, any +uid with gid 0, `nologin`, `read_only`, `cap_drop: [ALL]`, +`no-new-privileges`, a `noexec` tmpfs and one writable volume. + +**Development pod** means the opposite, deliberately. The engine image exists +to run coding agents, so it carries a writable home, a fixed `sprooty` uid, +passwordless `sudo`, an SSH server, and a Docker CLI for talking to a socket +you may choose to mount. It cannot run `read_only`. That is not an oversight +to be hardened away later — an agent session needs a machine, and this is the +machine. `engine/Dockerfile`'s header records why the two images are never +collapsed into one: doing so would quietly delete Vogt's containment story +under an unchanged image name. + +So the choice is straightforward. Run the **core only** where you want an API +and the containment. Run the **AIO** where you would otherwise run a dev box. +Run **core + engine** when you want the core's containment kept intact and are +willing to build the engine to get it. + ## 2. Run from published images The supported self-hosting path is the Compose base at @@ -192,9 +228,12 @@ run it as a system service, wrap exactly that `init && serve` pair. ### 3.2 The session engine and PWA -There is no published engine image, so the engine is always built from the -checkout. The generic overlay `deploy/engine.overlay.yml` does that and wires -the engine in front of the core in one command: +No image of the engine *by itself* is published, so this path always builds +one from the checkout. (If you would rather not build, the all-in-one +`vogt-stack` image carries the engine with a core already inside it — §1.1 +compares the two, and `deploy/stack.compose.yml` runs it.) The generic +overlay `deploy/engine.overlay.yml` builds the engine and wires it in front of +the core in one command: ```console cp deploy/.env.example deploy/.env # fill in the "session engine" block @@ -229,14 +268,24 @@ overrides. The image records the resolved versions and refuses to start when a persisted home volume would shadow an image-managed CLI (set `VOGT_AGENT_SHADOW_POLICY=warn` only for a deliberate user-local override). -This build-time flag is the only way the CLIs enter a deployment, and it -defaults to `false`. No published image carries them: the release `vogt` core -image runs the register alone, and the signed release digests promoted to -production (§7) are CLI-free by design. So a deployment that runs agent -sessions is one whose engine was built with `VOGT_INSTALL_AI_CLIENTS=true` — -pinning a plain release image instead leaves the `Claude Code (protected)` and -`Codex (protected)` session templates registered but unable to start, because -the `claude` and `codex` binaries are simply not in the image. +The flag defaults to `false`, so an engine you build yourself is CLI-free +unless you say otherwise. Two published images differ here, and the difference +is the point: + +- The release **core** image (`vogt`) runs the register alone and carries no + CLIs. It is not a pod and has nowhere to run a session. +- The release **stack** image (`vogt-stack`) *does* carry them. It is the + all-in-one dev pod, and a pod that cannot run `claude` or `codex` is not the + thing it is published to be. The versions are the `engine/agent-versions.env` + pins baked at build time, and the release build runs both binaries before it + publishes the digest — a build arg nothing executes is a default waiting to + be forgotten. + +So a deployment that runs agent sessions either pins a `vogt-stack` release +digest or builds its own engine with `VOGT_INSTALL_AI_CLIENTS=true`. Pinning +the plain `vogt` core image and expecting sessions leaves the `Claude Code +(protected)` and `Codex (protected)` templates registered but unable to start, +because the binaries are simply not in that image. Be aware before you run it: the engine image is a **development pod**, not a hardened service image — it carries a writable home, `sudo`, optional agent @@ -379,14 +428,27 @@ not change production by itself. The desired state a production instance runs — which digests, which overlays, which host specifics — is owned by the operator's own deployment repository, not this one; this tree ships only the estate-neutral base and overlays, -never a turnkey production estate. Two consequences are worth stating plainly. -The signed release digests are CLI-free (§3.2), so a production engine that -runs agent sessions is one the operator built with -`VOGT_INSTALL_AI_CLIENTS=true` and published to its own registry — the release -`vogt`/`vogt-stack` digests are not that image. And the maintainer's own -production is one such private deployment, layering a private overlay on the -public base; it is not a supported drop-in scenario reproducible from this -repository alone. +never a turnkey production estate. And the maintainer's own production is one +such private deployment, layering a private overlay on the public base; it is +not a supported drop-in scenario reproducible from this repository alone. + +**Two builds share the `vogt-stack` name, and pinning the wrong one is the +mistake this table exists to prevent.** Both carry the agent CLIs; what +separates them is whether they also carry the maintainer's estate: + +| Tag family | Built by | Pod base | Carries | Meant for | +|---|---|---|---|---| +| `X.Y.Z`, `X.Y`, `sha-`, `latest` | `release.yml` (version tag) | `lean` | `claude`, `codex` | the signed public artifact — anyone | +| `dev`, `dev-`, `prod-` | `build.yml` (branch push) | `full` | the above, plus Flutter/Android SDK, Cadastre MCP, theclawbay | the maintainer's own dev/prod pods | + +The release family is the one to pin. The branch family is not a "fuller" +release — it is a private deployment's image, carrying integrations that +address one estate's infrastructure and mean nothing outside it. Pin a +`dev-` digest only if you are that estate. + +What the release digests do *not* carry is Flutter and the Android SDK: those +belong to the `full` pod base, and the signed APK is built by `release.yml`'s +own Android job rather than from inside a pod. ### 7.1 Promote `dev` to production diff --git a/engine/Dockerfile b/engine/Dockerfile index 4672222d..d64cec95 100644 --- a/engine/Dockerfile +++ b/engine/Dockerfile @@ -298,6 +298,16 @@ COPY engine/agent-versions.env /usr/local/share/vogt/agent-versions.env # One `npm install -g` for whichever packages are wanted, not one per package: # a second global install into the same prefix reconciles the whole tree for no # benefit. No `sudo`: the pod base hands this stage over as root. +# +# The install is then asked to prove itself, because `npm install -g` exiting 0 +# is not evidence that `claude` runs (#505). The published tarball ships +# `bin/claude.exe` as a ~500-byte shell stub; the real binary arrives from the +# `postinstall` script or a platform-native optional dependency, so +# `--ignore-scripts`, some pnpm configs, or a missing optional dep all yield a +# green build and a pod whose `claude` only prints an error. That is the +# packaging that broke 2.1.237 (#147/#148) and held the pin at 2.1.236 for +# months; it has not changed. One `--version` makes every future Renovate bump +# of the pin self-verifying instead of hopeful. RUN set -eu; \ . /usr/local/share/vogt/agent-versions.env; \ codex_version="${CODEX_VERSION:-${VOGT_CODEX_VERSION}}"; \ @@ -318,7 +328,9 @@ RUN set -eu; \ npm install -g --prefix=/usr/local $pkgs; \ fi \ && if [ "$INSTALL_AI_CLIENTS" = "true" ]; then \ - install -d /usr/local/libexec \ + /usr/local/bin/claude --version | grep -qF "$claude_code_version" \ + || { echo "claude is not $claude_code_version — a stub install (#147/#148, #505)" >&2; exit 1; } \ + && install -d /usr/local/libexec \ && codex_target="$(readlink -f /usr/local/bin/codex)" \ && ln -s "$codex_target" /usr/local/libexec/codex-real \ && ln -sfn /usr/local/bin/codex-full-access /usr/local/bin/codex ; \ diff --git a/engine/agent-versions.env b/engine/agent-versions.env index 16d17af2..81c9ddee 100644 --- a/engine/agent-versions.env +++ b/engine/agent-versions.env @@ -4,6 +4,12 @@ # renovate: datasource=npm depName=@openai/codex VOGT_CODEX_VERSION=0.149.1 # renovate: datasource=npm depName=@anthropic-ai/claude-code -VOGT_CLAUDE_CODE_VERSION=2.1.236 +# 2.1.258, not npm's `stable` (still 2.1.236): Claude Fable 5.1 +# (`claude-fable-5-1`) was added in 2.1.257, so an older pin cannot select the +# current default model from inside a pod — and the pin is deliberately +# unwalkable at runtime (#196), so a stale one can only be fixed by a rebuild. +# The `claude --version` check in engine/Dockerfile guards the reason 2.1.236 +# was held in the first place (#147/#148, #505). +VOGT_CLAUDE_CODE_VERSION=2.1.258 # renovate: datasource=npm depName=theclawbay VOGT_THECLAWBAY_VERSION=0.6.13 diff --git a/tests/test_deploy.py b/tests/test_deploy.py index d20c8a47..036f9c45 100644 --- a/tests/test_deploy.py +++ b/tests/test_deploy.py @@ -153,10 +153,18 @@ def test_the_dev_image_build_turns_the_ai_clients_on() -> None: "INSTALL_AI_CLIENTS must be passed to both the candidate and the " f"pushed build of engine/Dockerfile, for dev and prod; found {len(wired)}" ) + # "A pod with the clients but not the integrations is a third shape nobody + # chose" was true of `build.yml`, whose refs are the estate's own pods, and + # it stays the rule here. It is no longer true of the *release*: that shape + # has since been chosen deliberately and is what the public generic AIO is + # — clients, no estate integrations, `lean` pod base. See + # `test_the_release_stack_is_the_generic_shape` below, which pins it. + # The two files therefore disagree on purpose; this assertion is about + # `build.yml` alone and must not be generalised to both. for arg in ("INSTALL_CADASTRE_MCP", "INSTALL_THECLAWBAY"): assert len(re.findall(arg + estate, text)) == 2, ( - f"{arg} must follow the same estate rule as INSTALL_AI_CLIENTS; a " - "pod with the clients but not the integrations is a third shape " + f"{arg} must follow the same estate rule as INSTALL_AI_CLIENTS; an " + "estate pod with the clients but not the integrations is a shape " "nobody chose" ) # Flutter is no longer a build arg of the merged image (#184): it is the @@ -189,6 +197,91 @@ def test_the_dev_image_build_turns_the_ai_clients_on() -> None: ) +def test_the_release_stack_is_the_generic_shape() -> None: + """The release stack is the public AIO: agent CLIs, no estate integrations. + + The release used to be the inverse of what it is published for — no + `INSTALL_AI_CLIENTS` line at all (so `engine/Dockerfile`'s `false` default + won) and `INSTALL_CADASTRE_MCP=true`. It shipped the maintainer's estate + MCP to strangers while withholding the two CLIs the product exists to run. + Nothing was red, because nothing asserted either half. + + So this pins the shape rather than trusting the args to stay written: a + release carries `claude` and `codex`, and carries neither cadastre nor + theclawbay. `build.yml` keeps the estate rule and is asserted separately in + `test_the_dev_image_build_turns_the_ai_clients_on` — the two files differ on + purpose, which is why neither test is written over both. + """ + text = (WORKFLOWS / "release.yml").read_text(encoding="utf-8") + # Both build steps, as ever: the candidate is what the smoke test runs and + # the push is what consumers pull, and an arg on one but not the other means + # the tested image is not the published one. + for arg in ("INSTALL_AI_CLIENTS=true", "INSTALL_CADASTRE_MCP=false"): + assert text.count(arg) == 2, ( + f"release.yml must pass {arg} to both the candidate and the pushed " + f"stack build; found {text.count(arg)}" + ) + assert "INSTALL_CADASTRE_MCP=true" not in text, ( + "a release must not carry the maintainer's estate MCP: it is the " + "public artifact, and cadastre means nothing outside one estate" + ) + assert "INSTALL_THECLAWBAY" not in text, ( + "theclawbay is an estate integration; a release leaves it at the " + "Dockerfile's false default rather than naming it" + ) + # `vogt-verify-agent-clis` is not this proof and never was: it returns 0 + # when the binaries are absent (it detects a persisted-$HOME copy shadowing + # the image's, which is a different question). Only running them proves + # they are there — the #23 lesson, applied to the release stream. + smoke = text[text.index("- name: both halves run") :] + smoke = smoke[: smoke.index("- uses:")] + loop = re.search(r"for tool in ([^;]+); do", smoke) + assert loop, ( + "the release stack's smoke test must run the CLIs it now ships, not " + "merely check them for shadowing (NFR-Q7)" + ) + probed = set(loop.group(1).split()) + owed = {"claude", "codex"} + assert owed <= probed, ( + "the release image must be asked for the clients it carries; missing: " + f"{sorted(owed - probed)}" + ) + # `flutter` and `theclawbay` belong to the `full` pod base and the estate + # build. Probing for them here would fail a correct release. + assert not probed & {"flutter", "theclawbay"}, ( + "a release is the `lean` pod base with no estate integrations; probing " + f"for {sorted(probed & {'flutter', 'theclawbay'})} asks it for what it " + "deliberately does not carry" + ) + + +@pytest.mark.parametrize("workflow", ["build.yml", "release.yml"]) +def test_no_build_args_block_contains_a_comment(workflow: str) -> None: + """`#` in a `|` block scalar is content, not a comment. + + `build-args: |` is a literal block, and build-push-action parses it without + a comment option, so an explanatory note written inside the block reaches + buildx as a build arg named `#`. `build.yml` carries a comment saying so, + directly above the block it applies to — and the note was written there + because the mistake had already been made once. It was then made again + while adding the release's generic build args, which is the point at which + a comment stops being the right tool. + + Notes belong above the `build-args:` key, where YAML comments are comments. + """ + text = (WORKFLOWS / workflow).read_text(encoding="utf-8") + for block in re.finditer( + r"^(\s+)build-args: \|\n((?:\1 .*\n|\n)*)", text, re.MULTILINE + ): + offending = [ + line for line in block.group(2).splitlines() if line.strip().startswith("#") + ] + assert not offending, ( + f"{workflow}: comment inside a build-args block scalar reaches " + f"buildx as a build arg; move it above the key: {offending}" + ) + + def test_the_image_has_no_default_listen_address() -> None: """NFR-D2: the image must not silently bind anything.""" text = _without_comments(DOCKERFILE.read_text(encoding="utf-8")) @@ -1643,6 +1736,31 @@ def test_the_pinned_agent_cli_cannot_update_past_its_pin() -> None: ) +@pytest.mark.skipif( + not ENGINE_DOCKERFILE.exists(), + reason="the core-alone job (NFR-Q6) deletes engine/; this reads its Dockerfile", +) +def test_the_pinned_claude_proves_it_is_not_a_stub() -> None: + """`npm install -g` exiting 0 is not evidence that `claude` runs (#505). + + The published tarball ships `bin/claude.exe` as a ~500-byte shell stub; the + real binary arrives from the `postinstall` script or a platform-native + optional dependency. When that does not happen the install still succeeds + and the image still builds — the pod simply has a `claude` that prints an + error. That is the packaging that broke 2.1.237 (#147/#148) and held the pin + at 2.1.236 for months, and it has not changed since, so a pin is only worth + what a check that the binary answers is worth. + + Paired with the sibling test above: `DISABLE_UPDATES` stops the pin being + walked past at runtime, this stops it being hollow at build time. + """ + text = ENGINE_DOCKERFILE.read_text(encoding="utf-8") + assert re.search(r"claude --version \| grep -qF", text), ( + "engine/Dockerfile must run the claude it just installed and match it " + "against the pin, or a stub install is a green build (#505)" + ) + + def test_latest_moves_only_on_a_semver_tag() -> None: """NFR-C3: a commit build never moves `latest`; a release does. diff --git a/tests/test_public_delivery.py b/tests/test_public_delivery.py index baf3492c..a8558c36 100644 --- a/tests/test_public_delivery.py +++ b/tests/test_public_delivery.py @@ -14,6 +14,12 @@ PUBLIC_COMPOSE = REPO_ROOT / "deploy" / "vogt.compose.yml" BUILD_OVERLAY = REPO_ROOT / "deploy" / "vogt.build.yml" ENGINE_OVERLAY = REPO_ROOT / "deploy" / "engine.overlay.yml" +STACK_COMPOSE = REPO_ROOT / "deploy" / "stack.compose.yml" +# The two public files that stand a pod in front of a core. They differ in how +# the engine arrives — the overlay builds one, the AIO pulls one that already +# contains the core — and agree on everything a stranger is entitled to assume: +# loopback by default, named volumes, and not one estate name between them. +PUBLIC_DEPLOY_FILES = (ENGINE_OVERLAY, STACK_COMPOSE) PUBLIC_ENV = REPO_ROOT / "deploy" / ".env.example" DOCKERFILE = REPO_ROOT / "Dockerfile" @@ -243,25 +249,96 @@ def test_the_engine_overlay_builds_the_engine_and_fronts_the_core() -> None: assert re.search(r"ENGINE_TOKEN:\s*\"\$\{ENGINE_TOKEN:\?", overlay) -def test_the_engine_overlay_publishes_to_loopback_unless_told_otherwise() -> None: - """The host interface is an exposure decision, so the overlay refuses one. +@pytest.mark.parametrize("path", PUBLIC_DEPLOY_FILES, ids=lambda p: p.name) +def test_the_public_deploy_files_publish_to_loopback_unless_told_otherwise( + path: Path, +) -> None: + """The host interface is an exposure decision, so these files refuse one. Like the base, the engine's published port defaults to `127.0.0.1`; the engine's own in-container socket is 0.0.0.0 so the published port reaches it, which is the one place the two must differ. + + It matters more for the AIO than for the overlay, not less: that image is a + development pod carrying `sudo`, `sshd` and the agent CLIs, so a default + that put it on a network interface would be handing out a shell. """ - overlay = _without_comments(ENGINE_OVERLAY.read_text(encoding="utf-8")) + overlay = _without_comments(path.read_text(encoding="utf-8")) assert "${ENGINE_BIND:-127.0.0.1}:" in overlay assert 'ENGINE_BIND: "0.0.0.0:8910"' in overlay -def test_the_engine_overlay_carries_no_estate_addresses_or_paths() -> None: - overlay = _without_comments(ENGINE_OVERLAY.read_text(encoding="utf-8")).lower() +def test_the_stack_compose_runs_the_published_aio_and_its_own_core() -> None: + """The AIO is the other half of the parity model: one image, no build. + + Where the overlay builds an engine and proxies to a *sibling* core, this + runs a published image that already contains both and points the engine at + its own loopback. That single `VOGT_CORE_URL` is what starts the core — + the entrypoint derives the core's listen address from the proxy target, so + the pair cannot drift into a front door aimed at a port nothing serves. + + Loopback is also the containment: NFR-D11 says the engine is the only way + in, and a non-loopback value here would mean the core is somewhere else + entirely, not that it is exposed. + """ + stack = _without_comments(STACK_COMPOSE.read_text(encoding="utf-8")) + assert "build:" not in stack, ( + "the AIO is the published image; a build directive here would quietly " + "make it something other than the digest a consumer pinned" + ) + assert "${VOGT_STACK_IMAGE:-" in stack + assert 'VOGT_CORE_URL: "http://127.0.0.1:8000"' in stack, ( + "the AIO runs its own core on loopback; any other value stops the " + "entrypoint starting one (NFR-D11)" + ) + assert 'VOGT_FRONTED: "true"' in stack + assert "VOGT_CORE_TOKEN_FILE:" in stack + assert "VOGT_BOOTSTRAP_CORE_TOKEN_FILE:" in stack + # The engine's own token is the one required operator value (>=16 chars). + assert re.search(r"ENGINE_TOKEN:\s*\"\$\{ENGINE_TOKEN:\?", stack) + # One published port, and it is the engine's. Publishing 8000 would put the + # core on the network beside the door that exists to be the only way in. + published = re.findall(r"^ - \"\$\{ENGINE_BIND.*$", stack, re.MULTILINE) + assert len(published) == 1, f"expected exactly one published port: {published}" + assert ":8000" not in "".join(published) + + +def test_the_stack_compose_is_a_base_not_an_overlay() -> None: + """Layering it onto `vogt.compose.yml` would run two cores. + + The AIO image contains a core and the base compose runs one, so the two + files are alternatives. This is worth asserting because every other + deployment file in `deploy/` *is* an overlay, which makes combining them + the natural guess and a wrong one. + """ + stack = _without_comments(STACK_COMPOSE.read_text(encoding="utf-8")) + # A base declares what it runs: image, volumes and secrets of its own. + assert re.search(r"^volumes:\n", stack, re.MULTILINE) + assert re.search(r"^secrets:\n", stack, re.MULTILINE) + assert re.search(r"^services:\n", stack, re.MULTILINE) + # Scoped to the `services:` block: the top-level `volumes:` keys sit at the + # same indent, so an unscoped search finds the named volumes too. + block = re.search(r"^services:\n(.*?)(?=^\S)", stack, re.MULTILINE | re.DOTALL) + assert block, "no services block" + services = re.findall(r"^ ([a-z][a-z0-9-]*):$", block.group(1), re.MULTILINE) + assert services == ["vogt"], ( + f"the AIO is one container by definition; found services {services}" + ) + + +@pytest.mark.parametrize("path", PUBLIC_DEPLOY_FILES, ids=lambda p: p.name) +def test_the_public_deploy_files_carry_no_estate_addresses_or_paths( + path: Path, +) -> None: + overlay = _without_comments(path.read_text(encoding="utf-8")).lower() for marker in ESTATE_MARKERS: assert marker.lower() not in overlay, f"estate marker leaked: {marker}" -def test_the_engine_overlay_uses_named_volumes_not_host_binds() -> None: +@pytest.mark.parametrize("path", PUBLIC_DEPLOY_FILES, ids=lambda p: p.name) +def test_the_public_deploy_files_use_named_volumes_not_host_binds( + path: Path, +) -> None: """A fresh named volume keeps the image's ownership; a host bind would arrive root-owned and break the pod, and would tie the file to one host. @@ -269,7 +346,7 @@ def test_the_engine_overlay_uses_named_volumes_not_host_binds() -> None: never an absolute path, a relative path, a `~` home, or a `${VAR:-/path}` whose default is a path. """ - overlay = _without_comments(ENGINE_OVERLAY.read_text(encoding="utf-8")) + overlay = _without_comments(path.read_text(encoding="utf-8")) # Isolate each service's `volumes:` list; ports and secrets live under # their own keys and are not volume mounts. volume_blocks = re.findall(