From 163dcee10eea7e9b6d9d206721e5983df6ae5968 Mon Sep 17 00:00:00 2001 From: Austin Burdine Date: Mon, 14 Sep 2026 17:01:54 -0400 Subject: [PATCH 1/2] refactor: simplify installer primitives and clarify runtime plan --- .env.example | 9 +- .github/workflows/test.yml | 2 + CLAUDE.md | 11 +- README.md | 2 +- bootstrap.sh | 119 ++----------- compose.yml | 4 +- docs/configuration.md | 17 +- docs/ghost-cli-replacement.md | 284 ++++++++----------------------- docs/install.md | 29 ++-- install.sh | 96 +++++------ scripts/lib/config.sh | 5 +- scripts/lib/install.sh | 159 +++++------------ scripts/lib/preflight.sh | 8 +- tests/compose-matrix.test.mjs | 9 + tests/compose-readiness.test.mjs | 72 ++++++++ tests/helpers.mjs | 2 +- tests/install-e2e.test.mjs | 11 +- tests/install-probes.test.mjs | 45 +++++ tests/install.test.mjs | 67 ++++++-- 19 files changed, 412 insertions(+), 539 deletions(-) create mode 100644 tests/compose-readiness.test.mjs create mode 100644 tests/install-probes.test.mjs diff --git a/.env.example b/.env.example index af88e089..bc3cd00b 100644 --- a/.env.example +++ b/.env.example @@ -46,12 +46,17 @@ DOMAIN="example.com" # Optional `www.` redirect target rendered into the generated Caddy routes. # WWW_REDIRECT="www.example.com" -# Exact Ghost image pin, resolved at installation. The `next` variants install -# Ghost directly under /home/ghost rather than the older +# Requested Ghost repository/tag. Installation pins the resolved artifact below. +# The `next` variants install Ghost directly under /home/ghost rather than the older # /var/lib/ghost/versions/ layout. GHOST_IMAGE="ghost" GHOST_VERSION="6-next-alpine" +# Written by the installer as ghost@sha256:...; overrides the repository/tag +# above for both Ghost and Tinybird sync. Updates must replace this pin too. +# Without it, manually configured sites use GHOST_IMAGE:GHOST_VERSION. +# GHOST_IMAGE_REF="ghost@sha256:..." + # Paths inside the Ghost image. The defaults match the `next` variants. Pinning # a GHOST_VERSION with the older layout means setting both of these to # /var/lib/ghost/content and /var/lib/ghost/current/core/server/data/tinybird. diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 4521c431..46c09c53 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -27,6 +27,7 @@ jobs: run: | set -eu jq --version + curl --version docker version docker compose version @@ -84,6 +85,7 @@ jobs: docker --version docker compose version jq --version + curl --version - name: Run the helper tests under bash 3.2 env: diff --git a/CLAUDE.md b/CLAUDE.md index 2612637a..a07f4712 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -144,8 +144,9 @@ The repository includes comprehensive migration tools: `install.sh` is checkout-owned and installs into its own directory; `--dir` elsewhere is refused. `bootstrap.sh` selects a release by semver order (never lexically), clones it, and `exec`s that checkout's installer. Ghost versions are -resolved to an exact tag by asking the pulled image for its own `GHOST_VERSION`, -`GHOST_CONTENT` and `GHOST_INSTALL`; the digest goes into `.ghost-docker.json`. +resolved to a digest pin by asking the pulled image for its own `GHOST_VERSION`, +`GHOST_CONTENT` and `GHOST_INSTALL`; `GHOST_IMAGE_REF` pins both Ghost and +Tinybird sync, and the digest also goes into `.ghost-docker.json`. Rules that must not regress: @@ -157,8 +158,8 @@ Rules that must not regress: - Docker access is established by asking the daemon, never from `docker` group membership. Read-only probes have deadlines so a wedged daemon is reported rather than hung on. -- Host tools are `docker` + `jq` (+ `git` for the bootstrap) plus the POSIX - utilities in `GD_HOST_UTILITIES`; `tests/install-e2e.test.mjs` installs with a +- Host tools are `docker`, `jq`, `curl` (+ `git` for bootstrap), plus + the Linux/macOS utilities in `GD_HOST_UTILITIES`; `tests/install-e2e.test.mjs` installs with a `PATH` of exactly that list. - Options for steps that have not landed (`--import`, `--with supervisor`, `--image-registry`, `--ghost-channel`, `--without`) exit 3 naming the step, @@ -195,7 +196,7 @@ land as stacked pull requests in the dependency order given in §3. ## Important Notes - Runtime prerequisites: `bash`, Docker Engine 25.0.0, Docker Compose v2.24.0, - and `jq` (used by the helpers for JSON). `install.sh` verifies + `jq` (JSON) and `curl` (ingress probes). `install.sh` verifies them in preflight; `scripts/migrate.sh` already required `jq` - Node.js is a development/test requirement only, never needed to run a site. The exception is the legacy `scripts/migrate.sh`, retired by `install.sh --import` diff --git a/README.md b/README.md index 388f0af6..f0c6b20e 100644 --- a/README.md +++ b/README.md @@ -2,7 +2,7 @@ Configuration to run Ghost and its services with Docker Compose. -Requires **bash**, **Docker Engine 25.0+**, **Docker Compose v2.24+** and **jq**. +Requires **bash**, **Docker Engine 25.0+**, **Docker Compose v2.24+**, **jq** and **curl**. ## Install diff --git a/bootstrap.sh b/bootstrap.sh index fbb2eb0d..faa06fe0 100755 --- a/bootstrap.sh +++ b/bootstrap.sh @@ -16,9 +16,8 @@ # that list. # # This file contains bootstrap logic only. Installation logic belongs to the -# release, so that a site is always installed by the code it is pinned to. This -# shim carries its own version comparison rather than sourcing the repository's -# helpers, because it runs before there is a checkout to source them from. +# release, so that a site is always installed by the code it is pinned to. Release +# selection uses jq directly because there is no checkout to source yet. set -euo pipefail GD_BOOTSTRAP_REPO=${GD_BOOTSTRAP_REPO:-https://github.com/TryGhost/ghost-docker.git} @@ -69,108 +68,20 @@ _timeout() { # --- Release selection ----------------------------------------------------- -# _semver_cmp A B -> -1, 0 or 1 -# -# Full semver ordering, including prereleases: 1.2.0-beta.2 sorts before -# 1.2.0, and 1.9.0 after 1.10.0 would be wrong. Lexical sorting gets both -# backwards, which is why this is spelled out. -_semver_cmp() { - local a=${1#v} b=${2#v} ac bc ap bp i x y - ac=${a%%-*} - bc=${b%%-*} - case $a in *-*) ap=${a#*-} ;; *) ap="" ;; esac - case $b in *-*) bp=${b#*-} ;; *) bp="" ;; esac - - local -a av bv - IFS=. read -ra av <<<"$ac" - IFS=. read -ra bv <<<"$bc" - for ((i = 0; i < 3; i++)); do - x=${av[i]:-0} - y=${bv[i]:-0} - [[ $x =~ ^[0-9]+$ ]] || x=0 - [[ $y =~ ^[0-9]+$ ]] || y=0 - ((10#$x > 10#$y)) && { - printf '1\n' - return - } - ((10#$x < 10#$y)) && { - printf -- '-1\n' - return - } - done - - # A release outranks any prerelease of the same version. - [[ -z $ap && -z $bp ]] && { - printf '0\n' - return - } - [[ -z $ap ]] && { - printf '1\n' - return - } - [[ -z $bp ]] && { - printf -- '-1\n' - return - } - - local -a ai bi - IFS=. read -ra ai <<<"$ap" - IFS=. read -ra bi <<<"$bp" - for ((i = 0; i < ${#ai[@]} || i < ${#bi[@]}; i++)); do - x=${ai[i]:-} - y=${bi[i]:-} - [[ -z $x ]] && { - printf -- '-1\n' - return - } - [[ -z $y ]] && { - printf '1\n' - return - } - if [[ $x =~ ^[0-9]+$ && $y =~ ^[0-9]+$ ]]; then - ((10#$x > 10#$y)) && { - printf '1\n' - return - } - ((10#$x < 10#$y)) && { - printf -- '-1\n' - return - } - else - [[ $x > $y ]] && { - printf '1\n' - return - } - [[ $x < $y ]] && { - printf -- '-1\n' - return - } - fi - done - printf '0\n' -} - # _latest_release CHANNEL -# The newest tag on a channel. Stable is vX.Y.Z; beta also considers -# vX.Y.Z-beta.N, and still prefers a release over a prerelease of the same -# version. Selection is by semver order, never by the order the remote listed -# the tags in. +# Only the release formats we publish are accepted. Sort numeric components +# with jq (already a prerequisite), independent of locale and Git user config. +# A stable release follows every beta of the same version. _latest_release() { - local want=$1 line tag best="" - local stable='^v[0-9]+\.[0-9]+\.[0-9]+$' - local prerelease='^v[0-9]+\.[0-9]+\.[0-9]+-beta\.[0-9]+$' - - while IFS= read -r line; do - tag=${line##*refs/tags/} - tag=${tag%'^{}'} - [[ $tag =~ $stable ]] || { [[ $want == beta && $tag =~ $prerelease ]] || continue; } - if [[ -z $best ]] || [[ $(_semver_cmp "$tag" "$best") == 1 ]]; then - best=$tag - fi - done < <(git ls-remote --tags "$GD_BOOTSTRAP_REPO" 2>/dev/null) - - [[ -n $best ]] || return 1 - printf '%s\n' "$best" + local refs + refs=$(git ls-remote --tags --refs "$GD_BOOTSTRAP_REPO") || return 1 + printf '%s\n' "$refs" | jq -Rser --arg channel "$1" ' + [split("\n")[] + | capture("refs/tags/(?v(?[0-9]+)\\.(?[0-9]+)\\.(?[0-9]+)(?:-beta\\.(?[0-9]+))?)$") + | select($channel == "beta" or .beta == null) + | .order = [(.major|tonumber), (.minor|tonumber), (.patch|tonumber), + (if .beta == null then 1 else 0 end), ((.beta // "0")|tonumber)]] + | sort_by(.order) | last | .tag // empty' } # Sourcing this file with GD_BOOTSTRAP_SOURCED=1 defines the helpers above and @@ -236,7 +147,7 @@ if [[ -e $dir ]]; then fi missing="" -for cmd in git docker jq; do +for cmd in git docker jq curl; do command -v "$cmd" >/dev/null 2>&1 || missing="$missing $cmd" done [[ -z $missing ]] || die "these are required and not installed:$missing" diff --git a/compose.yml b/compose.yml index 1c5ba2fe..bf408609 100644 --- a/compose.yml +++ b/compose.yml @@ -30,7 +30,7 @@ x-site-labels: &site-labels services: ghost: # Do not alter this without updating the Tinybird Sync container as well - image: ${GHOST_IMAGE:-ghost}:${GHOST_VERSION:-6-next-alpine} + image: ${GHOST_IMAGE_REF:-${GHOST_IMAGE:-ghost}:${GHOST_VERSION:-6-next-alpine}} restart: ${RESTART_POLICY:-unless-stopped} profiles: [local, production] labels: @@ -269,7 +269,7 @@ services: tinybird-sync: # Do not alter this without updating the Ghost container as well - image: ${GHOST_IMAGE:-ghost}:${GHOST_VERSION:-6-next-alpine} + image: ${GHOST_IMAGE_REF:-${GHOST_IMAGE:-ghost}:${GHOST_VERSION:-6-next-alpine}} restart: "no" profiles: [analytics] labels: diff --git a/docs/configuration.md b/docs/configuration.md index ab818a86..c0487c13 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -252,12 +252,13 @@ Runtime, on the server: - Docker Engine 25.0.0 — for `healthcheck.start_interval` - Docker Compose v2.24.0 — for `env_file` `required` and `depends_on` `required` - `jq` — used by the helpers for JSON, including `.ghost-docker.json` +- `curl` — HTTP/HTTPS ingress probes with bounded connection and request times -`install.sh` verifies all three during preflight, and `bootstrap.sh` also needs +`install.sh` verifies these tools during preflight, and `bootstrap.sh` also needs `git`. `scripts/migrate.sh` already required `jq`, so this is not a new prerequisite for existing servers. -Every other host utility the scripts invoke is POSIX and is listed in +Other host utilities use supported Linux/macOS interfaces and are listed in `GD_HOST_UTILITIES` in `scripts/lib/preflight.sh`. That list is the tool contract: `tests/install-e2e.test.mjs` runs a complete installation with a `PATH` built from exactly it, so a GNU-only or otherwise unusual dependency @@ -294,7 +295,7 @@ migration is owned by the stack updater (S6); the changes it has to handle are: keeps the Compose and operator settings and is no longer passed into the Ghost container. - `COMPOSE_PROFILES` must gain a site mode (`production` for an existing - server), and `SITE_MODE`, `URL`, `PROJECT_DIR` and an exact `GHOST_VERSION` + server), and `SITE_MODE`, `URL`, `PROJECT_DIR` and an exact `GHOST_IMAGE_REF` pin must be added. `scripts/migrate.sh` still migrates a Ghost-CLI installation and has been @@ -316,5 +317,13 @@ That was changed during S1, deliberately, and §1 of 240 lines and bought nothing an operator can see. Node.js is **not** a runtime requirement. It is used only to run the test -suite. `install.sh` verifies `docker`, `docker compose` and `jq` during +suite. `install.sh` verifies `docker`, `docker compose`, `jq` and `curl` during preflight, and does not require Node. + +## Installed image pins + +The installer writes `GHOST_IMAGE_REF=ghost@sha256:...`. This is the authoritative +reference for Ghost and Tinybird sync, so a later pull cannot move the site to a +new image. `GHOST_IMAGE` and `GHOST_VERSION` record the requested repository/tag; +without `GHOST_IMAGE_REF`, they remain the fallback for manually configured sites. +Image-changing operations must update the pin and recorded metadata together. diff --git a/docs/ghost-cli-replacement.md b/docs/ghost-cli-replacement.md index 9772a1b2..8f4285c9 100644 --- a/docs/ghost-cli-replacement.md +++ b/docs/ghost-cli-replacement.md @@ -29,11 +29,11 @@ of those contracts. Update operator documentation and tests with each step. | ActivityPub and analytics | Per-site initially, including for future members of shared infrastructure. Each site owns its ActivityPub database/storage and Tinybird configuration/deployment lifecycle. No shared ActivityPub or analytics service in S13. | | Versions | Resolve and persist an exact Ghost image version on installation. Ghost upgrades and stack/repository updates are separate operations. Record resolved image digests for recovery. | | Distribution | Clone at a release tag. Stable tags `vX.Y.Z`; beta tags `vX.Y.Z-beta.N`. A bootstrap shim selects the release and delegates to that checkout. | -| Installation | Scriptable `install.sh`, with flags for every required prompt. Local mode uses MySQL too. Requires `bash`, `docker`, `docker compose` and `jq` on the host, verified in preflight; `scripts/migrate.sh` already required both `bash` and `jq`. Helper scripts stay bash 3.2 compatible so macOS's system bash works. No host Node requirement for a new install; Node is otherwise only used to run the test suite. The one exception is the legacy `scripts/migrate.sh`, which shells out to `scripts/config-to-env.js` and therefore still needs Node until `install.sh --import` replaces it. | +| Installation | Scriptable `install.sh`, with flags for every required prompt. Local mode uses MySQL too. Requires `bash`, `docker`, `docker compose`, `jq` and `curl` on the host, verified in preflight; `scripts/migrate.sh` already required both `bash` and `jq`. Helper scripts stay bash 3.2 compatible so macOS's system bash works. No host Node requirement for a new install; Node is otherwise only used to run the test suite. The one exception is the legacy `scripts/migrate.sh`, which shells out to `scripts/config-to-env.js` and therefore still needs Node until `install.sh --import` replaces it. | | Migration | Ghost-CLI exports a bundle; Docker imports it. Fix the encoding contract before declaring v1 frozen. `install.sh --import` replaces `scripts/migrate.sh` outright rather than living alongside it; retire the old scripts, and with them the last host Node dependency, only after replacement fidelity and recovery tests pass. | | Upgrades | Optional supervisor using a file exchange and Docker socket. Ship a tested host-driven upgrade operation first, then reuse its recovery contract in the supervisor. | | UX | Standard Compose commands for daily operation; scripts for installation, doctor/list, migration, backup/restore, and upgrades. No wrapper binary named `ghost`. | -| Where tooling runs | A thin host shell layer (bootstrap, preflight/doctor, dispatch) plus a pinned manager image that holds the stateful operations. See §2.10. Host requirements stay `bash`, `docker`, `docker compose`, `jq`; no host language runtime. | +| Where tooling runs | A thin host shell layer (bootstrap, preflight/doctor, dispatch) plus a pinned manager image that holds the stateful operations. See §2.10. Host requirements stay `bash`, `docker`, `docker compose`, `jq`, `curl`; no host language runtime. | | Configuration | `.env` contains Compose/operator settings; `ghost.env` contains only Ghost application settings. Do not pass the whole `.env` into Ghost. A mounted Ghost JSON config file was evaluated as a replacement for `ghost.env` and rejected; see §2.1. | | Ghost nightly channel | Future explicit opt-in via `--ghost-channel nightly`; published to GHCR, independently of the stack release channel. Stable remains the default. | | Service image registry | Future `--image-registry dockerhub|ghcr` selects dual-published traffic-analytics and ActivityPub images, including migrations. Preserve existing selections when updating. | @@ -124,7 +124,7 @@ Requirements: before adding infra-only mode in S13. - Keep the initial default network naming unchanged. Do not introduce an empty `name:` as a guessed equivalent of an omitted field. -- `ghost.env` is the only application `env_file`, and is transitional. Explicit +- `ghost.env` is the only application `env_file` for the initial release. Explicit Compose environment entries override container-owned keys; the importer rejects/omits those keys. @@ -174,116 +174,10 @@ values with `scripts/config.sh set`, which encodes correctly, and `env_lint`, which catches the bare-`$` case. Document that hand-editing a value containing `$` is unsafe. -Caddy is part of the `production` mode, not an optional profile. Making -bring-your-own-proxy a first-class path was considered and rejected: it would -mean owning validation of the operator's proxy configuration, and the failure it -guards against is subtle — a wrong `X-Forwarded-Proto` yields incorrect absolute -URLs and non-secure cookies, so the site half works rather than failing. That is -a poor thing to support on someone else's proxy. - -An operator who already runs nginx or Apache can still do it, as a manual -customization rather than a supported mode: Ghost publishes on -`127.0.0.1:${GHOST_PORT}` in every mode, so they point their proxy there and -edit `compose.yml` to drop the caddy service or move it off 80/443. Document -that this is unsupported and that stack updates may touch `compose.yml`. - -Profiles are additive, not mutually exclusive or conditional configuration. Validate -that exactly one site mode is selected. Optional services must not accidentally -activate unrelated modes. Explicitly targeted Compose services can run even when -their profiles are inactive; helper commands must account for dependencies. - -Example generated Compose settings (credentials omitted): - -```dotenv -# Local -COMPOSE_PROFILES=local -COMPOSE_PROJECT_NAME=ghost-local-example -PROJECT_DIR=/absolute/path/to/site -NODE_ENV=development -URL=http://localhost:2368 -GHOST_PORT=2368 -RESTART_POLICY=no -GHOST_VERSION=6.3.1-alpine -DATABASE_HOST=db -DATABASE_NAME=ghost -DATABASE_USER=ghost - -# Production uses the same variable contract with: -# COMPOSE_PROFILES=production -# NODE_ENV=production -# URL=https://example.com -# DOMAIN=example.com -# RESTART_POLICY=unless-stopped -# Optional: ADMIN_DOMAIN=admin.example.com -# Optional profiles are added only after their configuration is validated. -``` - -Requirements: - -- Ghost publishes `127.0.0.1:${GHOST_PORT:-2368}:2368`. The installer picks a free - port when none is supplied; an explicit occupied port is an error. -- Parameterize database host, name, and user now, even though single-site defaults - remain `db`/`ghost`/`ghost`. Use the same connection contract for backup and import. -- Set a unique Ghost network alias `ghost-${COMPOSE_PROJECT_NAME}` and use it in - generated proxy routes and helper clients. Never rely on `ghost` for shared-network - addressing when S13 is introduced. -- Persist the project name independently of the directory name. Moving a site still - requires updating and validating `PROJECT_DIR` and bind mounts. -- Use `restart: ${RESTART_POLICY:-unless-stopped}` only for long-running services. - Setup, migration, and deployment jobs retain `restart: "no"`. -- Initially `URL` may be required because every supported mode contains Ghost. Do - not put `:?` guards on optional-service variables such as `PROJECT_DIR` or DOMAIN. - Validate requirements by mode before provisioning or startup. Revisit URL's guard - before adding infra-only mode in S13. -- Keep the initial default network naming unchanged. Do not introduce an empty - `name:` as a guessed equivalent of an omitted field. -- `ghost.env` is the only application `env_file`, and is transitional. Explicit - Compose environment entries override container-owned keys; the importer - rejects/omits those keys. - -Application configuration is moving from `ghost.env` to a mounted JSON config -file, because dotenv cannot hold an arbitrary value safely: Compose interpolates -`env_file` values, so an SMTP password of `Pa$$w0rd!` reaches Ghost as -`Pa$w0rd!` with no error anywhere. Verified against `ghost:6-alpine` -(Ghost 6.61.0, nconf 0.13.0): - -- The image ships `config.production.json` in Ghost's install directory - (`/home/ghost` in the `next` variants, `/var/lib/ghost` in the older layout), - with `config.development.json` symlinked to it. It sets `url`, `server`, - `mail.transport: "Direct"`, `logging.transports`, `process`, `security` and - `paths.contentPath`. -- nconf is first-added-wins. `loader.js` registers `custom-env` - (`config..json`) *before* `local-env-jsonc` (`config.local.jsonc`), so - `config.local.jsonc` cannot override anything the image ships — including - `mail.transport`. It is not usable as the operator's config file. -- Compose `environment` entries still outrank every config file, so - container-owned keys stay enforced by construction. -- `localUtils.jsoncFormat` already exists and wraps `jsonc-parser`. - -The Ghost change this depends on is to register a custom-env JSONC file -*before* `custom-env` in `core/shared/config/loader.js`: - -```js -nconf.file('custom-env-jsonc', { - file: path.join(customConfigPath, 'config.' + env + '.jsonc'), - format: localUtils.jsoncFormat, -}); -nconf.file('custom-env', path.join(customConfigPath, 'config.' + env + '.json')); -``` - -Ordering is the point. Registered first, the operator's file layers *over* the -image's shipped defaults instead of replacing them, so ghost-docker never has -to keep its own copy of those defaults in sync. Comments and trailing commas -come along for free. - -Two constraints follow: +The initial release keeps `ghost.env`, including for imports of older Ghost +versions. A future JSONC format would require a Ghost loader change and an +explicit compatibility/migration design; it is not an S1-S5 dependency. -- The file is per-environment, so it mounts as `config.${NODE_ENV}.jsonc`; - local mode (`NODE_ENV=development`) needs `config.development.jsonc`. -- It sets a Ghost version floor. §2.4 imports at the *source* Ghost version, so - a site imported from a Ghost that predates this change would not read the - file at all. S5 must either detect that and fall back, or require a floor for - imported sources; it cannot assume the feature is present. - Add site/mode labels and a real Ghost readiness probe. A running container or redirect response alone does not establish readiness. - Cap container logs and make optional-service resource costs visible. @@ -665,8 +559,9 @@ this have to work when Docker is broken? **Host shell.** Small, portable, and the only thing that runs before an image exists. -- The bootstrap shim: check Docker, resolve the release, pull the manager - image, exec into it. +- The bootstrap shim: check prerequisites, resolve the release, clone it and + execute that checkout's installer. Stateful commands use the manager dispatcher + once it lands in S4. - Preflight and `doctor`. These must diagnose a host where Docker is missing, stopped, or unreachable, so they cannot depend on the manager image. They may use it for deeper checks when it is available, and must degrade to useful @@ -676,7 +571,7 @@ exists. - Anything that must survive the manager image being unpullable. **Manager image.** Pinned, published from this repo, and where the stateful -work lives: install orchestration, import, backup and restore, Ghost upgrades, +work lives: import, backup and restore, Ghost upgrades, and stack updates. It is one image with several entrypoints, not several images. @@ -715,99 +610,66 @@ interpolation. Anything writing `.env` still encodes a literal `$` as `$$` regardless of implementation language, and §2.1 records why the alternative config format was rejected. -When install.sh resolves the exact Ghost image (§1), it should write +When install.sh resolves the exact Ghost image (§1), it writes `GHOST_CONTENT_PATH` and `GHOST_TINYBIRD_PATH` into `.env` from that image's own `GHOST_CONTENT`, so the layout and the configuration cannot disagree in the first place. S1 validates the pair; S2 should set it. -Sequencing: this decision has to be made before S2, because it determines -whether `install.sh` is the installer or a bootstrap that runs one. Steps -already shipped in host shell (S1's `config.sh` and `caddy.sh`) stay where they -are; the boundary does not run through them. - -#### When the manager image lands, and what its first tenant is - -Revised 2026-09-03, after S2. Two questions kept coming up — *should the image -ship sooner?* and *should env validation and Caddy generation move into it?* — -so the boundary above is stated concretely rather than left to inference: when -the image lands, and exactly which helpers move onto it versus stay host shell. - -**The image's first tenant is the first stateful operation, at S4, not S8.** -§2.6 already publishes a privileged supervisor image from this repo and requires -it to "follow §2.5 rather than inventing a second upgrade/recovery algorithm." -If S4–S7 implement backup, restore, upgrade and recovery in host shell and S8 -then re-implements them in an image, that algorithm exists twice, and the second -copy is the one under the privileged supervisor. So the manager image is -introduced in **S4**, with backup/restore as its first entrypoint; S5 (import), -S7 (upgrade) and S8 (supervisor) are further entrypoints on the same image, not -parallel codebases. This is a scheduling clarification, not a new component: -§2.10 already defined the image and the dispatcher. It does **not** move S4's -deliverable — S4 still ships backup/restore — it fixes the language they are -written in so they are not rewritten at S8. The host dispatcher (this section's -mount, identity and exit-code rules) is written in S4 alongside its first -`docker run` target. - -**Some config helpers move into the manager CLI at S4; a specific subset must -not.** The dividing line is not "config versus stateful" — it is whether the -helper can run *before and without* a working daemon, and whether it derives its -answer by asking Docker. This was worked out concretely after S2, against the -real files, because "put the helpers in a node container so the scripts get -smaller" is a reasonable instinct that turns out to be right for two files and -wrong for two others. - -The manager image exists from S4 for backup/restore, and the dispatcher (§2.10's -mount/identity/exit-code rules) is written there. Once both exist, moving the -*pure* config logic onto them is close to free and is a real simplification, so -S4 (or S5, whichever first needs them container-side) does it: - -- **Moves into the manager CLI.** `env.sh` (the `$$` serializer/parser, ~260 - lines of bash regex state machine → ~60 lines of `JSON.parse`/stringify plus - one encoder) and `meta.sh` (`.ghost-docker.json`, ~245 lines of `jq` → native - JSON). Caddy *rendering* (`caddy_render` and `_caddy_site_block`, pure template - emission) moves with them. These are pure functions of files on disk; nothing - in them asks the daemon a question. The bash versions are deleted, not - wrapped — a straight substitution, which is the easy-to-review kind of diff. - -- **Stays host shell, permanently.** Two reasons, each disqualifying on its own: - - - *Runs before/without the image.* The bootstrap resolves the manager from the - release tag and can have it write the first `.env`, but the bootstrap shim - itself, preflight, and `site.sh check`/doctor must diagnose a host where - Docker is missing, stopped, or wedged. They cannot be `docker run`. (S2 - verified this the hard way: a wedged daemon mid-session was still diagnosed - by host-shell preflight precisely because it does not depend on the image.) - - *Derives its answer by asking Docker.* `config.sh validate` establishes the - container-owned keys by asking `docker compose config` what the container - receives (`config_ghost_environment`, the "derived, not listed" guarantee). - `caddy_validate`/`reload`/`verify` drive the running caddy container through - `compose_run`. Moving these into the manager would mean either bundling the - Docker/Compose CLI inside the manager and running compose-in-a-container over - a mounted socket, or reimplementing Compose interpolation in node — which is - exactly the drift that "ask Compose" was chosen to avoid. Neither is worth - it, so validation and caddy orchestration stay where they can call Compose - directly. - -Consequences to accept deliberately: config logic ends up split — `config -get/set/unset` in the manager CLI, `config validate` in host shell — because the -two halves have different daemon dependencies. That split is the honest cost, and -it is smaller than the cost of dragging a Docker CLI into the manager image to -avoid it. Containerizing the movable helpers also does **not** solve Compose's -`$$` interpolation: a literal `$` is encoded `$$` regardless of implementation -language, as recorded above. And the net line count of the move, counting the -Dockerfile, publish pipeline and privileged dispatcher it rides on, is roughly a -wash — the reason to do it is that env/meta stop being bash, not that the repo -gets shorter. Which is why it waits for S4: on a PR that builds the image and -dispatcher anyway, the env/meta deletion is pure upside; as a standalone change -it would stand up a privileged image and publish pipeline to save ~130 counted -lines, which does not clear the bar. - -So the dispatcher-plus-image model covers the **stateful** commands (backup, -restore, import, upgrade, supervisor) and, from S4, the **pure** config helpers -(`env`, `meta`, caddy render). Preflight, doctor, the bootstrap shim, `config -validate` and caddy orchestration stay host shell — the first three because they -must run when there is no usable image, the last two because they answer by -asking Docker. Host requirements stay `bash`, `docker`, `docker compose`, `jq`; -no host language runtime is added. +S2 installation remains host shell. S4 introduces the manager for stateful +operations without requiring an installer rewrite. + +#### When the manager image lands, and which helpers move + +The manager image lands in **S4**, with backup/restore as its first entrypoint. +S5 import, S7 upgrades and S8 supervision reuse that implementation and recovery +contract. The host dispatcher lands alongside it, following the mount, ownership +and exit-code rules above. Do not build a manager solely to shorten S2. + +Keep the existing config, metadata and Caddy helpers on the host for now. +`site.sh check`, config validation and rendering currently depend on `env_get`, +`env_lint` and metadata readers. Deleting these Bash libraries in favor of +container entrypoints would make previously offline diagnostics depend on Docker. +Pure file processing alone is therefore not a sufficient criterion for moving a +helper. S4 does not include a mandatory config-helper rewrite. + +Before moving a helper, map all callers and establish how the host reads and +validates configuration without a working daemon or manager image. Preserve one +encoding contract and its Compose round-trip fixtures. Do not introduce a second +dotenv parser or quietly weaken doctor to make the move possible. Stateful manager +operations may initially use the existing tested shell helpers shipped in the +image (with Bash and jq), rather than reimplementing them. Read-only diagnostics +remain available on the host; all mutations use the shared operation lock. + +Compose parsing (`docker compose config`) does not need a daemon. Docker-driven +operations may run in the manager if it includes the Docker/Compose client and +honors the host context and path contract; this is an explicit packaging decision, +not a reason to reimplement Compose interpolation. The S4 dispatcher/image design +must decide that interface before moving orchestration into it. + +#### S1/S2 simplification contract + +- `curl` is an explicit host prerequisite for HTTP/HTTPS checks, alongside Bash, + Docker/Compose and jq; Git is needed for bootstrap. Use deadlines, bypass local + proxy/curl configuration, preserve Host/SNI, and distinguish routing checks from + public certificate trust. A production install before DNS/certificate readiness + continues to report that condition as a warning. +- Release discovery accepts only `vX.Y.Z` and `vX.Y.Z-beta.N`. Use jq numeric keys + to order them independently of user Git configuration; do not maintain a generic + SemVer comparator. Git's configurable suffix ordering was considered but is not + needed for these two formats. +- Pull Ghost once, require a repository digest, and persist `GHOST_IMAGE_REF` as + `repository@sha256:...`. Ghost and Tinybird sync both execute this reference. + `GHOST_IMAGE`/`GHOST_VERSION` retain the requested repository/tag as provenance + and as the fallback for manually configured or pre-pin sites. Changing those + fields alone never changes an installed site's pin. Upgrades/imports/restore + must set the complete reference deliberately and keep metadata in sync. +- Generate fresh `.env` settings in one atomic write through the existing encoder; + Compose supplies optional defaults and `.env.example` documents them. Keep + general editing helpers for operator changes, migrations and updates. +- Use Compose `up --wait --wait-timeout` with existing health checks and one-shot + dependency conditions. Verify successful and failed one-shot completion at the + minimum/current Compose versions. Keep a separate ingress check; the wait timeout + applies to readiness, not the entire pull/start operation. ## 3. Implementation steps @@ -1023,17 +885,9 @@ recovery algorithm S7 and S8 reuse exists once, in the image, not in host shell awaiting a rewrite. Write the host dispatcher here — the `docker run` mount, identity and exit-code rules from §2.10 — and route backup/restore through it. -With the image and dispatcher built, migrate the **pure** config helpers onto -them: `env.sh` and `meta.sh` become manager-CLI entrypoints (`config get/set/ -unset`, `meta ...`), and `caddy_render` moves with them; the bash versions are -deleted, a straight substitution. This is optional to S4's backup/restore -deliverable and may slip to S5 if it competes for review attention, but it is -cheapest here because the image already exists. The helpers that must stay host -shell do not move: preflight, the bootstrap shim and `site.sh check`/doctor -(they run without the image), and `config validate` plus caddy -orchestration (`caddy_validate`/`reload`/`verify`, which answer by asking -`docker compose`). Expect config logic to end up split — `config get/set` in the -manager, `config validate` in host shell — as §2.10 records. +Keep config/metadata/Caddy helpers in place unless their host callers have a +verified offline replacement, as specified in §2.10. S4's deliverable is the +shared stateful operation runtime, not a second configuration implementation. Also add the shared operation lock to installation/reconfigure retroactively: §2.2 requires install to take it, and S2 deferred it to this step. diff --git a/docs/install.md b/docs/install.md index 0638bd88..75c5e4bc 100644 --- a/docs/install.md +++ b/docs/install.md @@ -96,13 +96,14 @@ the step they belong to, rather than being reported as unknown options: Nothing ships with a default credential. 4. **Exact Ghost image.** The requested version is pulled, and the image is asked for its own `GHOST_VERSION`, `GHOST_CONTENT` and `GHOST_INSTALL`. The - *exact* version tag is written to `.env` — never a moving one — and the - digest is recorded in `.ghost-docker.json` for recovery. `GHOST_CONTENT_PATH` - and `GHOST_TINYBIRD_PATH` come from the image, so the mounted content + repository digest is required and written as `GHOST_IMAGE_REF=ghost@sha256:...` + in `.env`, and recorded in `.ghost-docker.json` for recovery. Both Ghost and + Tinybird sync use that pin; the requested tag is provenance only. + `GHOST_CONTENT_PATH` and `GHOST_TINYBIRD_PATH` come from the image, so the mounted content directory and the image layout cannot disagree. -5. **Configuration.** `.env` and `ghost.env`, both mode `0600`. `.env` starts - from the tracked example so its comments survive; `ghost.env` is written - fresh, because the example's SMTP block is a placeholder and a site shipping +5. **Configuration.** `.env` and `ghost.env`, both mode `0600`. `.env` is generated + in one atomic write, with optional defaults documented in `.env.example`. + `ghost.env` is written fresh, because the example's SMTP block is a placeholder and a site shipping with `smtp.example.com` configured fails to send mail in a way that looks like a Ghost bug. 6. **Routing**, in production: routes are rendered, validated, installed and @@ -110,7 +111,8 @@ the step they belong to, rather than being reported as unknown options: `caddy/global/` are yours and are never touched. 7. **Metadata.** `.ghost-docker.json`, described in [configuration.md](configuration.md#installation-metadata). -8. **Start and verify**, unless `--no-start`: the database and Ghost must report +8. **Start and verify**, unless `--no-start`: Compose `up --wait --wait-timeout` + waits for the selected services; the database and Ghost must report *healthy* through their own health checks, and the Admin API must answer through the ingress the site actually uses. A running container is not readiness, and `up -d` returning zero is not a working site. @@ -158,13 +160,20 @@ run by the installer; the summary prints the two commands that finish it. See ## Host tools -Beyond `bash`, the installer requires **`docker`** (with Compose v2) and -**`jq`**; `bootstrap.sh` also needs **`git`**. Everything else it invokes is -POSIX and listed in `GD_HOST_UTILITIES` in `scripts/lib/preflight.sh`. That list +Beyond `bash`, the installer requires **`docker`** (with Compose v2), +**`jq`** and **`curl`**; `bootstrap.sh` also needs **`git`**. Other utilities +use the supported Linux/macOS interfaces and are listed in `GD_HOST_UTILITIES` +in `scripts/lib/preflight.sh`. That list is the tool contract, and `tests/install-e2e.test.mjs` runs an install with a `PATH` containing exactly it — so a GNU-only or unusual dependency added to a code path fails a test rather than someone's server. +Changing `GHOST_VERSION` alone does not change an installed site: `GHOST_IMAGE_REF` +is authoritative. A deliberate image change must update that pin and its metadata. + +The HTTPS probe checks routing through the published host port with the correct +Host/SNI. It accepts internal certificates and does not certify public TLS trust. + Node.js is **not** required to install or run a site. It runs the test suite. Docker access is established by **asking the daemon**, never by checking diff --git a/install.sh b/install.sh index 3a638f17..a0000d57 100755 --- a/install.sh +++ b/install.sh @@ -328,52 +328,47 @@ printf '\nWriting configuration\n' env_file="$dir/$GD_ENV_FILE_NAME" ghost_env_file="$dir/$GD_GHOST_ENV_FILE_NAME" -# Start from the tracked example so its comments — the ones that explain value -# encoding and the optional settings — survive into the installed file. -cp "$dir/$GD_ENV_FILE_NAME.example" "$env_file" -chmod 0600 "$env_file" - -set_env() { env_set "$env_file" "$1" "$2" 0600; } - -set_env COMPOSE_PROFILES "$profiles" -set_env SITE_MODE "$mode" -set_env COMPOSE_PROJECT_NAME "$project" -set_env PROJECT_DIR "$dir" -set_env NODE_ENV "$node_env" -set_env URL "$url" -set_env GHOST_IMAGE "$GD_DEFAULT_GHOST_IMAGE" -set_env GHOST_VERSION "$ghost_tag" -set_env GHOST_CONTENT_PATH "$ghost_content_path" -set_env GHOST_TINYBIRD_PATH "$ghost_tinybird_path" -set_env GHOST_PORT "$port" -set_env RESTART_POLICY "$restart_policy" -set_env DATABASE_HOST db -set_env DATABASE_PORT 3306 -set_env DATABASE_NAME ghost -set_env DATABASE_USER ghost -set_env DATABASE_PASSWORD "$db_password" -set_env DATABASE_ROOT_PASSWORD "$db_root_password" +# Fresh installs need no parsing or repeated edits of the example. Compose +# supplies defaults; the example remains the reference for optional settings. +{ + printf '# Generated site settings. See .env.example for optional settings.\n' + _gd_env_serialize COMPOSE_PROFILES "$profiles" + _gd_env_serialize SITE_MODE "$mode" + _gd_env_serialize COMPOSE_PROJECT_NAME "$project" + _gd_env_serialize PROJECT_DIR "$dir" + _gd_env_serialize NODE_ENV "$node_env" + _gd_env_serialize URL "$url" + _gd_env_serialize GHOST_IMAGE "$GD_DEFAULT_GHOST_IMAGE" + _gd_env_serialize GHOST_VERSION "$ghost_tag" + _gd_env_serialize GHOST_IMAGE_REF "$GD_DEFAULT_GHOST_IMAGE@$ghost_digest" + _gd_env_serialize GHOST_CONTENT_PATH "$ghost_content_path" + _gd_env_serialize GHOST_TINYBIRD_PATH "$ghost_tinybird_path" + _gd_env_serialize GHOST_PORT "$port" + _gd_env_serialize RESTART_POLICY "$restart_policy" + _gd_env_serialize DATABASE_HOST db + _gd_env_serialize DATABASE_PORT 3306 + _gd_env_serialize DATABASE_NAME ghost + _gd_env_serialize DATABASE_USER ghost + _gd_env_serialize DATABASE_PASSWORD "$db_password" + _gd_env_serialize DATABASE_ROOT_PASSWORD "$db_root_password" -if [[ $mode == production ]]; then - set_env DOMAIN "$domain" - set_env HTTP_PORT "$http_port" - set_env HTTPS_PORT "$https_port" - if [[ -n $admin_domain ]]; then - set_env ADMIN_DOMAIN "$admin_domain" - set_env ADMIN_URL "$admin_url" + if [[ $mode == production ]]; then + _gd_env_serialize DOMAIN "$domain" + _gd_env_serialize HTTP_PORT "$http_port" + _gd_env_serialize HTTPS_PORT "$https_port" + if [[ -n $admin_domain ]]; then + _gd_env_serialize ADMIN_DOMAIN "$admin_domain" + _gd_env_serialize ADMIN_URL "$admin_url" + fi fi -else - # A local site has no Caddy ingress, so leaving the example's DOMAIN in - # place would describe a domain nothing serves. - env_unset "$env_file" DOMAIN -fi -if ((want_analytics)); then - set_env TINYBIRD_API_URL "$tinybird_api_url" - set_env TINYBIRD_TRACKER_TOKEN "$tinybird_tracker_token" - set_env TINYBIRD_ADMIN_TOKEN "$tinybird_admin_token" - set_env TINYBIRD_WORKSPACE_ID "$tinybird_workspace_id" -fi + if ((want_analytics)); then + _gd_env_serialize TINYBIRD_API_URL "$tinybird_api_url" + _gd_env_serialize TINYBIRD_TRACKER_TOKEN "$tinybird_tracker_token" + _gd_env_serialize TINYBIRD_ADMIN_TOKEN "$tinybird_admin_token" + _gd_env_serialize TINYBIRD_WORKSPACE_ID "$tinybird_workspace_id" + fi +} | fs_atomic_write "$env_file" 0600 printf ' ok %s\n' "$GD_ENV_FILE_NAME" @@ -460,16 +455,10 @@ if ((no_start)); then printf '\nNot starting: --no-start was given.\n' else printf '\nStarting services\n' - compose_run "$dir" up -d + compose_run "$dir" up --wait --wait-timeout "$GD_READY_TIMEOUT" || + die "services did not become ready. See: docker compose ps -a; docker compose logs" started=1 - - install_wait_healthy "$dir" db "$GD_READY_TIMEOUT_DB" || - die "the database did not become ready. See: docker compose logs db" - printf ' ok database is ready\n' - - install_wait_healthy "$dir" ghost "$GD_READY_TIMEOUT_GHOST" || - die "Ghost did not become ready. See: docker compose logs ghost" - printf ' ok Ghost is ready\n' + printf ' ok services are ready\n' printf '\nVerifying ingress\n' if ! install_verify_ingress "$dir" "$mode" "$port" "$http_port" "$domain" "$admin_domain"; then @@ -489,7 +478,8 @@ Ghost is installed. Mode $mode Project $project Directory $dir - Ghost $GD_DEFAULT_GHOST_IMAGE:$ghost_tag ($ghost_exact_version) + Ghost $ghost_exact_version (requested $GD_DEFAULT_GHOST_IMAGE:$ghost_tag) + Image $GD_DEFAULT_GHOST_IMAGE@$ghost_digest Loopback 127.0.0.1:$port Profiles $profiles Content $dir/data/ghost diff --git a/scripts/lib/config.sh b/scripts/lib/config.sh index fa837777..ed57e485 100644 --- a/scripts/lib/config.sh +++ b/scripts/lib/config.sh @@ -156,11 +156,12 @@ config_validate_env() { # /home/ghost, older tags under /var/lib/ghost. Ask the image what it # expects rather than mapping tag names, which would drift. Best effort: # skipped when the image has not been pulled. - local image version declared expected + local image version declared expected image_ref image=$(env_get "$file" GHOST_IMAGE 2>/dev/null) || image= version=$(env_get "$file" GHOST_VERSION 2>/dev/null) || version= declared=$(env_get "$file" GHOST_CONTENT_PATH 2>/dev/null) || declared=/home/ghost/content - if [[ -n $version ]] && expected=$(config_image_content_path "${image:-ghost}:$version"); then + image_ref=$(env_get "$file" GHOST_IMAGE_REF 2>/dev/null) || image_ref="${image:-ghost}:$version" + if [[ -n $version ]] && expected=$(config_image_content_path "$image_ref"); then if [[ $declared != "$expected" ]]; then printf 'error: GHOST_CONTENT_PATH is %s but %s expects %s; set GHOST_CONTENT_PATH and GHOST_TINYBIRD_PATH to match the image\n' \ "$declared" "${image:-ghost}:$version" "$expected" diff --git a/scripts/lib/install.sh b/scripts/lib/install.sh index 5ff5127f..36c286ad 100644 --- a/scripts/lib/install.sh +++ b/scripts/lib/install.sh @@ -9,15 +9,14 @@ GD_INSTALL_LIB_LOADED=1 # The image and tag a new site gets when none is requested. The `next` variants -# install Ghost directly under /home/ghost; the exact tag is resolved from the -# image itself, so this is a starting point, not the pin that is written. +# install Ghost directly under /home/ghost; the digest is resolved from the +# pulled image, so this is a starting point, not the pin that is written. GD_DEFAULT_GHOST_IMAGE="ghost" GD_DEFAULT_GHOST_TAG="6-next-alpine" -# How long to wait for each service to report ready. Ghost's own health check -# allows a 180s start period on a cold boot with migrations to run. -GD_READY_TIMEOUT_DB=${GD_READY_TIMEOUT_DB:-300} -GD_READY_TIMEOUT_GHOST=${GD_READY_TIMEOUT_GHOST:-600} +# Compose enforces health checks and completed one-shot dependencies. This +# deadline covers its readiness wait; image pulls/startup precede that wait. +GD_READY_TIMEOUT=${GD_READY_TIMEOUT:-600} # install_slug STRING # A lowercase, dash separated token safe for a Compose project name. @@ -78,18 +77,6 @@ install_ghost_tag() { printf '%s\n' "$requested" } -# _gd_tag_variant TAG -# The part of a tag that is not the version: `6-next-alpine` -> `next-alpine`, -# `6.3.1-alpine` -> `alpine`, `next-alpine` -> `next-alpine`, `6` -> ``. -_gd_tag_variant() { - local tag=$1 - if [[ $tag =~ ^[0-9]+(\.[0-9]+)*(-(.*))?$ ]]; then - printf '%s\n' "${BASH_REMATCH[3]}" - return 0 - fi - printf '%s\n' "$tag" -} - # _gd_image_env IMAGE_REF NAME # One environment variable declared by an image, without running it. _gd_image_env() { @@ -144,14 +131,14 @@ _gd_image_tinybird_path() { # # TAG VERSION DIGEST CONTENT_PATH TINYBIRD_PATH # -# TAG is the exact pin written to `.env`; a moving tag is never persisted. -# DIGEST is the immutable identity, recorded in `.ghost-docker.json` so the -# image can be identified again during recovery. The paths come from the +# TAG records the requested tag; VERSION is reported by the image. +# DIGEST pins the actual Compose image reference and is also recorded in +# `.ghost-docker.json` for recovery. The paths come from the # image's own GHOST_CONTENT and GHOST_INSTALL, so the mounted content directory # and the image layout cannot disagree. install_resolve_ghost() { local image=$1 requested=$2 - local tag version content install digest variant exact ref exact_ref tinybird + local tag version content install digest ref tinybird tag=$(install_ghost_tag "$requested") ref="$image:$tag" @@ -168,25 +155,13 @@ install_resolve_ghost() { content=$(_gd_image_env "$ref" GHOST_CONTENT) || content=/home/ghost/content install=$(_gd_image_env "$ref" GHOST_INSTALL) || install=/home/ghost - # Prefer the immutable tag for that exact version, so the pin does not move - # under the site the next time the registry updates a rolling tag. It is - # accepted only when it is the same image. - variant=$(_gd_tag_variant "$tag") - if [[ -n $variant ]]; then - exact="$version-$variant" - else - exact=$version - fi - exact_ref="$image:$exact" - if [[ $exact != "$tag" ]] && docker pull --quiet "$exact_ref" >/dev/null 2>&1; then - if [[ $(docker image inspect "$exact_ref" --format '{{.Id}}' 2>/dev/null) == \ - $(docker image inspect "$ref" --format '{{.Id}}' 2>/dev/null) ]]; then - tag=$exact - ref=$exact_ref - fi - fi - - digest=$(_gd_image_digest "$ref" "$image") || digest="" + # Use the pulled artifact itself. Inferring and pulling another tag can + # race a registry update, and exact-looking tags are still mutable. + digest=$(_gd_image_digest "$ref" "$image") || { + printf 'error: %s has no repository digest; refusing an unpinned install\n' "$ref" >&2 + return 1 + } + ref="$image@$digest" tinybird=$(_gd_image_tinybird_path "$ref" "$install") printf '%s\t%s\t%s\t%s\t%s\n' "$tag" "$version" "$digest" "$content" "$tinybird" @@ -259,93 +234,37 @@ install_service_id() { compose_run "$1" ps -q "$2" 2>/dev/null | head -1 } -# install_wait_healthy DIR SERVICE TIMEOUT -# Waits for a service's own health check, not for a running container. Fails -# early when the container has exited: waiting out a timeout on a container -# that is already gone tells the operator nothing. -install_wait_healthy() { - local dir=$1 service=$2 timeout=$3 waited=0 id state health - while ((waited < timeout)); do - id=$(install_service_id "$dir" "$service") - if [[ -n $id ]]; then - state=$(docker inspect -f '{{.State.Status}}' "$id" 2>/dev/null || printf '') - health=$(docker inspect -f '{{if .State.Health}}{{.State.Health.Status}}{{else}}none{{end}}' "$id" 2>/dev/null || printf '') - case $state in - exited | dead) - printf 'error: the %s container exited before it became ready\n' "$service" >&2 - return 1 - ;; - esac - [[ $health == healthy ]] && return 0 - # A service with no health check is ready when it is running. - [[ $health == none && $state == running ]] && return 0 - fi - sleep 3 - waited=$((waited + 3)) - done - printf 'error: %s did not become ready within %ss\n' "$service" "$timeout" >&2 - return 1 -} - -# install_http_head HOST PORT PATH [HOST_HEADER] -# The status line and headers of one HTTP response, using bash's own /dev/tcp -# so that neither curl nor wget has to be installed on the host. +# HTTP probes use the host's published ports, including for HTTPS. Ignore +# user curl configuration/proxies so checks cannot accidentally probe a proxy. +# Do not follow redirects: callers distinguish redirects from an Admin 200. install_http_head() { - local host=$1 port=$2 path=$3 host_header=${4:-$1} line - - exec 3<>"/dev/tcp/$host/$port" 2>/dev/null || return 1 - printf 'GET %s HTTP/1.1\r\nHost: %s\r\nConnection: close\r\nUser-Agent: ghost-docker-install\r\nAccept: */*\r\n\r\n' \ - "$path" "$host_header" >&3 || { - exec 3<&- - exec 3>&- - return 1 - } - while IFS= read -r -t 20 line <&3; do - line=${line%$'\r'} - [[ -z $line ]] && break - printf '%s\n' "$line" - done - exec 3<&- - exec 3>&- + local host=$1 port=$2 path=$3 host_header=${4:-$1} + curl --disable --silent --show-error --noproxy '*' \ + --connect-timeout 5 --max-time 20 \ + --header "Host: $host_header" --dump-header - --output /dev/null \ + "http://$host:$port$path" } -# install_http_status HOST PORT PATH [HOST_HEADER] install_http_status() { - local head status - head=$(install_http_head "$@") || return 1 - status=$(printf '%s\n' "$head" | head -1) - [[ $status =~ ^HTTP/[0-9.]+[[:space:]]+([0-9]{3}) ]] || return 1 - printf '%s\n' "${BASH_REMATCH[1]}" + local host=$1 port=$2 path=$3 host_header=${4:-$1} + curl --disable --silent --show-error --noproxy '*' \ + --connect-timeout 5 --max-time 20 \ + --header "Host: $host_header" --output /dev/null --write-out '%{http_code}\n' \ + "http://$host:$port$path" } -# install_https_status DIR DOMAIN -# The status of an Admin API request through Caddy's HTTPS listener, made from -# inside the site network with the right SNI and Host. It runs in the Ghost -# container because TLS is beyond what bash's /dev/tcp can do, and neither curl -# nor openssl is a host requirement — while a Ghost image always has node. +# This is a routing check, including with Caddy's internal CA; it deliberately +# does not establish public certificate trust. DNS/certificate readiness is +# reported separately from successful configuration on a fresh installation. install_https_status() { - local dir=$1 domain=$2 out path script - + local dir=$1 domain=$2 path port path=$(env_get "$dir/.env" GHOST_HEALTHCHECK_PATH 2>/dev/null) || path=/ghost/api/admin/site/ - [[ -n $path ]] || path=/ghost/api/admin/site/ - - script=$( - cat <<'NODE' -const https = require('https'); -const [domain, path] = process.argv.slice(1); -const req = https.request({ - host: 'caddy', port: 443, servername: domain, path, - headers: { Host: domain }, rejectUnauthorized: false, timeout: 20000, -}, (res) => { res.resume(); process.stdout.write(String(res.statusCode)); }); -req.on('timeout', () => { req.destroy(); process.exit(1); }); -req.on('error', () => process.exit(1)); -req.end(); -NODE - ) - - out=$(compose_run "$dir" exec -T ghost node -e "$script" "$domain" "$path" 2>/dev/null) || return 1 - [[ $out =~ ^[0-9]{3}$ ]] || return 1 - printf '%s\n' "$out" + port=$(env_get "$dir/.env" HTTPS_PORT 2>/dev/null) || port=443 + curl --disable --silent --show-error --noproxy '*' \ + --connect-timeout 5 --max-time 20 --insecure \ + --resolve "$domain:$port:127.0.0.1" \ + --output /dev/null --write-out '%{http_code}\n' \ + "https://$domain:$port${path:-/ghost/api/admin/site/}" } # install_verify_ingress DIR MODE GHOST_PORT HTTP_PORT DOMAIN [ADMIN_DOMAIN] diff --git a/scripts/lib/preflight.sh b/scripts/lib/preflight.sh index 80fa73cc..447fd685 100644 --- a/scripts/lib/preflight.sh +++ b/scripts/lib/preflight.sh @@ -12,7 +12,7 @@ # decides the exit status. That keeps the checks free of presentation and makes # them straightforward to assert on. # -# Portability: no GNU-only utilities, no `sort -V`, no `ss`/`lsof`/`curl` +# Portability: no GNU-only utilities, no `sort -V`, no `ss`/`lsof` # requirement, and daemon access is established by using the daemon rather than # by inspecting group membership. @@ -22,10 +22,10 @@ GD_PREFLIGHT_LIB_LOADED=1 # Host commands the installer and helpers actually invoke. Anything not on this # list must not appear in a code path an operator can reach; the minimum-tools # test runs an install with a PATH containing only these. -readonly GD_REQUIRED_COMMANDS=(docker jq) +readonly GD_REQUIRED_COMMANDS=(docker jq curl) -# Every other host utility the installer and helpers invoke. These are POSIX -# and present on any supported host, so they are not preflight checks; they are +# Other host utilities use the supported Linux/macOS interfaces. They are not +# individual preflight checks, but are # recorded here as the tool contract, and tests/install-e2e.test.mjs runs an # install with a PATH built from exactly this list plus GD_REQUIRED_COMMANDS. # Adding a utility to a code path without adding it here fails that test, which diff --git a/tests/compose-matrix.test.mjs b/tests/compose-matrix.test.mjs index b1ccd632..b857d10c 100644 --- a/tests/compose-matrix.test.mjs +++ b/tests/compose-matrix.test.mjs @@ -77,6 +77,15 @@ describe('compose mode matrix', { skip: dockerAvailable() ? false : 'docker is n for (const { label, bin } of composeBinaries()) { describe(label, () => { + test('the digest pin overrides tags for Ghost and Tinybird together', () => { + setup(MATRIX[1]); + const pin = `ghost@sha256:${'a'.repeat(64)}`; + shOk(`env_set ${q(join(site, '.env'))} GHOST_IMAGE_REF ${q(pin)}`); + const config = composeConfig(site, { bin }); + assert.equal(config.services.ghost.image, pin); + assert.equal(config.services['tinybird-sync'].image, pin); + }); + for (const entry of MATRIX) { describe(entry.profiles, () => { let config; diff --git a/tests/compose-readiness.test.mjs b/tests/compose-readiness.test.mjs new file mode 100644 index 00000000..271ffee9 --- /dev/null +++ b/tests/compose-readiness.test.mjs @@ -0,0 +1,72 @@ +// Exercise the real optional-service dependency graph with disposable commands, +// without provisioning Tinybird credentials or running remote schema changes. +import { describe, test } from 'node:test'; +import assert from 'node:assert/strict'; +import { writeFileSync } from 'node:fs'; +import { join } from 'node:path'; +import { + tempDir, cleanup, makeSite, writeEnv, compose, composeConfig, + composeBinaries, dockerAvailable, +} from './helpers.mjs'; + +const jobs = ['activitypub-migrate', 'tinybird-login', 'tinybird-sync', 'tinybird-deploy']; + +describe('Compose readiness contract', { skip: !dockerAvailable() }, () => { + for (const { label, bin } of composeBinaries()) { + for (const failure of [null, 'activitypub-migrate', 'ghost']) { + test(`${label}: ${failure ? `rejects failed ${failure}` : 'waits for health and completed jobs'}`, () => { + const dir = tempDir('gd-readiness'); + const site = makeSite(dir); + try { + writeEnv(join(site, '.env'), { + COMPOSE_PROFILES: 'local,analytics,activitypub', + COMPOSE_PROJECT_NAME: `gd-ready-${process.pid}-${failure || 'success'}`, + URL: 'http://localhost:2368', DATABASE_PASSWORD: 'test', DATABASE_ROOT_PASSWORD: 'test', + }); + const original = composeConfig(site, { bin }); + const services = Object.fromEntries(Object.entries(original.services).map(([name, service]) => { + const job = jobs.includes(name); + return [name, { + image: 'alpine:3.20', + restart: 'no', + stop_grace_period: '1s', + depends_on: service.depends_on, + command: ['sh', '-c', job + ? `exit ${name === failure ? 1 : 0}` + : 'sleep 1; touch /tmp/ready; exec sleep 300'], + ...(!job && { + healthcheck: { + test: ['CMD-SHELL', name === failure ? 'exit 1' : 'test -f /tmp/ready'], + interval: '1s', timeout: '1s', retries: 3, + }, + }), + }]; + })); + writeFileSync(join(site, 'compose.yml'), JSON.stringify({ services })); + const result = compose(site, ['up', '--wait', '--wait-timeout', '15'], { bin }); + if (failure) { + assert.notEqual(result.status, 0, result.stdout + result.stderr); + assert.match(result.stderr, new RegExp(`${failure}|unhealthy|failed`)); + } else { + assert.equal(result.status, 0, result.stderr); + const ps = compose(site, ['ps', '-a', '--format', 'json'], { bin }); + assert.equal(ps.status, 0, ps.stderr); + const rows = ps.stdout.trim().startsWith('[') + ? JSON.parse(ps.stdout) : ps.stdout.trim().split('\n').map(JSON.parse); + for (const name of jobs) { + const row = rows.find((r) => r.Service === name); + assert.equal(row?.State, 'exited', name); + assert.equal(row?.ExitCode, 0, name); + } + for (const name of ['ghost', 'db']) { + assert.equal(rows.find((r) => r.Service === name)?.Health, 'healthy', name); + } + } + } finally { + compose(site, ['down', '-v', '--remove-orphans', '--timeout', '1'], { bin }); + cleanup(dir); + } + }); + } + } +}); diff --git a/tests/helpers.mjs b/tests/helpers.mjs index 768207ad..58849be7 100644 --- a/tests/helpers.mjs +++ b/tests/helpers.mjs @@ -218,7 +218,7 @@ const GIT_IDENTITY = { }; export function git(repo, args) { - return execFileSync('git', ['-C', repo, ...args], { + return execFileSync('git', ['-C', repo, '-c', 'commit.gpgsign=false', '-c', 'tag.gpgsign=false', ...args], { encoding: 'utf8', env: { ...process.env, ...GIT_IDENTITY }, stdio: ['pipe', 'pipe', 'pipe'], diff --git a/tests/install-e2e.test.mjs b/tests/install-e2e.test.mjs index cb3fe61a..0ece8dd0 100644 --- a/tests/install-e2e.test.mjs +++ b/tests/install-e2e.test.mjs @@ -125,8 +125,13 @@ describe('installing from a candidate release', { skip, concurrency: 1 }, () => // A moving tag would let the site change Ghost version under the operator // on the next `docker compose pull`. The pin has to be exact. test('the Ghost version is pinned exactly, and the paths come from the image', () => { - assert.match(env(site, 'GHOST_VERSION'), /^\d+\.\d+\.\d+(-.+)?$/); - assert.equal(env(site, 'GHOST_VERSION').startsWith(meta(site).ghost.version), true); + const pin = `ghost@${meta(site).ghost.digest}`; + assert.equal(env(site, 'GHOST_IMAGE_REF'), pin); + const config = JSON.parse(compose(site, ['config', '--format', 'json']).stdout); + assert.equal(config.services.ghost.image, pin); + const id = compose(site, ['ps', '-q', 'ghost']).stdout.trim(); + assert.equal(execFileSync('docker', ['inspect', '-f', '{{.Config.Image}}', id], + { encoding: 'utf8' }).trim(), pin); const content = env(site, 'GHOST_CONTENT_PATH'); assert.ok(content.endsWith('/content'), content); assert.match(env(site, 'GHOST_TINYBIRD_PATH'), /\/core\/server\/data\/tinybird$/); @@ -287,7 +292,7 @@ describe('installing from a candidate release', { skip, concurrency: 1 }, () => // Node's fetch forbids overriding the Host header, so Caddy would see // 127.0.0.1 and redirect there; the request has to carry Host: ghost.test. - // The installer's own /dev/tcp helper sets it, which is what it verifies + // The installer's own curl helper sets it, which is what it verifies // with too, so drive the check through that rather than fetch. test('HTTP on the ingress port redirects to HTTPS for the site domain', () => { const head = run('bash', ['-c', diff --git a/tests/install-probes.test.mjs b/tests/install-probes.test.mjs new file mode 100644 index 00000000..840f7662 --- /dev/null +++ b/tests/install-probes.test.mjs @@ -0,0 +1,45 @@ +import { test } from 'node:test'; +import assert from 'node:assert/strict'; +import { createServer } from 'node:http'; +import { execFile } from 'node:child_process'; +import { promisify } from 'node:util'; +import { writeFileSync } from 'node:fs'; +import { join } from 'node:path'; +import { tempDir, cleanup, REPO_DIR, q } from './helpers.mjs'; + +const exec = promisify(execFile); + +test('HTTP probes preserve Host and status, ignoring curl config and proxies', async () => { + const dir = tempDir('gd-probe'); + const server = createServer((req, res) => { + if (req.headers.host !== 'ghost.test') { + res.writeHead(400).end(); + } else if (req.url === '/redirect') { + res.writeHead(302, { Location: '/ok' }).end(); + } else { + res.writeHead(req.url === '/failure' ? 503 : 200).end('response body'); + } + }); + await new Promise((resolve) => server.listen(0, '127.0.0.1', resolve)); + const port = server.address().port; + // These settings would change the probe's result if curl loaded them. + writeFileSync(join(dir, '.curlrc'), 'location\nfail\n'); + const probe = async (fn, path) => (await exec(process.env.GD_TEST_BASH || 'bash', ['-c', + `. ${q(join(REPO_DIR, 'scripts/lib/common.sh'))}\n${fn} 127.0.0.1 ${port} ${q(path)} ghost.test`, + ], { + env: { ...process.env, CURL_HOME: dir, http_proxy: 'http://127.0.0.1:1', ALL_PROXY: 'http://127.0.0.1:1' }, + timeout: 25_000, + })).stdout; + try { + assert.equal((await probe('install_http_status', '/ok')).trim(), '200'); + assert.equal((await probe('install_http_status', '/redirect')).trim(), '302'); + assert.equal((await probe('install_http_status', '/failure')).trim(), '503'); + const headers = await probe('install_http_head', '/redirect'); + assert.match(headers, /^HTTP\/1\.1 302/); + assert.match(headers, /location: \/ok/i); + assert.doesNotMatch(headers, /response body/); + } finally { + await new Promise((resolve) => server.close(resolve)); + cleanup(dir); + } +}); diff --git a/tests/install.test.mjs b/tests/install.test.mjs index 6ba943b3..35f8523e 100644 --- a/tests/install.test.mjs +++ b/tests/install.test.mjs @@ -250,11 +250,39 @@ describe('Ghost version resolution', () => { assert.equal(shOk('install_ghost_tag 6.3.1-alpine').trim(), '6.3.1-alpine'); }); - test('the variant is separated from the version so an exact pin can be built', () => { - assert.equal(shOk('_gd_tag_variant 6-next-alpine').trim(), 'next-alpine'); - assert.equal(shOk('_gd_tag_variant 6.3.1-alpine').trim(), 'alpine'); - assert.equal(shOk('_gd_tag_variant next-alpine').trim(), 'next-alpine'); - assert.equal(shOk('_gd_tag_variant 6').trim(), ''); + test('resolution pulls once and requires a digest instead of guessing a version tag', () => { + const dir = tempDir('gd-resolve'); + const pulls = join(dir, 'pulls'); + const digest = `sha256:${'a'.repeat(64)}`; + const fakeDocker = ` + docker() { + case "$1" in + pull) printf '%s\\n' "$*" >> ${q(pulls)} ;; + image) + case "$*" in + *RepoDigests*) [[ -z $MOCK_DIGEST ]] || printf 'ghost@%s\\n' "$MOCK_DIGEST" ;; + *) printf '%s\\n' GHOST_VERSION=6.3.1 GHOST_CONTENT=/var/lib/ghost/content GHOST_INSTALL=/var/lib/ghost ;; + esac ;; + run) printf '%s\\n' /var/lib/ghost/current/core/server/data/tinybird ;; + *) return 1 ;; + esac + } + install_resolve_ghost ghost 6-alpine`; + try { + const result = sh(fakeDocker, { env: { MOCK_DIGEST: digest } }); + assert.equal(result.status, 0, result.stderr.toString()); + assert.deepEqual(result.stdout.toString().trim().split('\t'), [ + '6-alpine', '6.3.1', digest, '/var/lib/ghost/content', + '/var/lib/ghost/current/core/server/data/tinybird', + ]); + assert.equal(readFileSync(pulls, 'utf8').trim().split('\n').length, 1); + const missing = sh(fakeDocker, { env: { MOCK_DIGEST: '' } }); + assert.notEqual(missing.status, 0); + assert.match(missing.stderr.toString(), /refusing an unpinned install/); + assert.equal(missing.stdout.toString(), ''); + } finally { + cleanup(dir); + } }); }); @@ -276,14 +304,6 @@ describe('release selection', () => { }); after(() => cleanup(dir)); - test('semver ordering, including prereleases', () => { - const cmp = (a, b) => bootstrap(`_semver_cmp ${a} ${b}`).stdout.trim(); - assert.equal(cmp('v1.10.0', 'v1.9.0'), '1', '1.10.0 must be newer than 1.9.0'); - assert.equal(cmp('v1.2.0-beta.1', 'v1.2.0'), '-1', 'a prerelease precedes its release'); - assert.equal(cmp('v1.2.0-beta.10', 'v1.2.0-beta.2'), '1', 'beta.10 must be newer than beta.2'); - assert.equal(cmp('v1.2.3', 'v1.2.3'), '0'); - }); - test('stable selects the newest release and ignores prereleases', () => { const result = bootstrap('_latest_release stable', { GD_BOOTSTRAP_REPO: repo }); assert.equal(result.stdout.trim(), 'v1.10.0'); @@ -294,6 +314,27 @@ describe('release selection', () => { assert.equal(result.stdout.trim(), 'v1.11.0-beta.10'); }); + test('a stable release outranks its betas, independent of Git sorting settings', () => { + git(repo, ['tag', 'v1.11.0']); + try { + const config = join(dir, 'gitconfig'); + writeFileSync(config, '[versionsort]\n suffix = -other\n suffix = \n suffix = -beta.\n'); + const result = bootstrap('_latest_release beta', { + GD_BOOTSTRAP_REPO: repo, GIT_CONFIG_GLOBAL: config, + }); + assert.equal(result.status, 0, result.output); + assert.equal(result.stdout.trim(), 'v1.11.0'); + } finally { + git(repo, ['tag', '-d', 'v1.11.0']); + } + }); + + test('an inaccessible remote fails instead of selecting a release', () => { + const result = bootstrap('_latest_release stable', { GD_BOOTSTRAP_REPO: join(dir, 'missing') }); + assert.notEqual(result.status, 0); + assert.equal(result.stdout, ''); + }); + test('a repository with no releases fails rather than guessing', () => { const empty = join(dir, 'empty'); copyWorktree(empty); From 634f6933df16acd2272ea1f4ad4b646fd8948033 Mon Sep 17 00:00:00 2001 From: Austin Burdine Date: Mon, 14 Sep 2026 17:10:31 -0400 Subject: [PATCH 2/2] ci: run tests and shellcheck on next pushes --- .github/workflows/shellcheck.yml | 1 + .github/workflows/test.yml | 1 + 2 files changed, 2 insertions(+) diff --git a/.github/workflows/shellcheck.yml b/.github/workflows/shellcheck.yml index 1f89f2ea..90ef2d46 100644 --- a/.github/workflows/shellcheck.yml +++ b/.github/workflows/shellcheck.yml @@ -5,6 +5,7 @@ on: push: branches: - main + - next - renovate/* # The shellcheck version is pinned so CI and a developer's machine agree. diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 46c09c53..a0674390 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -5,6 +5,7 @@ on: push: branches: - main + - next - renovate/* env: