From a66085fbf05dc4265e6c684c68f3282593ce8218 Mon Sep 17 00:00:00 2001 From: Austin Burdine Date: Wed, 2 Sep 2026 22:06:34 -0400 Subject: [PATCH 1/2] Implemented phase 1 of cli replacement plan ref https://linear.app/ghost/issue/PLA-413/phase-1-initial-scripts-library-testing-infrastructure - add intitial scripts library + entrypoints - split env into compose-level and ghost-level configuration - switch to next ghost variant by default - add node tests for e2e testing in ci - add docs for ghost-cli replacement, configuration, and bundle format --- .env.example | 194 ++-- .github/workflows/shellcheck.yml | 2 +- .github/workflows/test.yml | 64 ++ .gitignore | 20 + .shellcheckrc | 4 + CLAUDE.md | 125 ++- README.md | 116 ++- TINYBIRD.md | 11 +- caddy/Caddyfile | 17 + caddy/Caddyfile.example | 89 +- caddy/custom/.gitignore | 4 + caddy/custom/README.md | 18 + caddy/global/.gitignore | 4 + caddy/global/README.md | 18 + caddy/sites/.gitignore | 3 + caddy/snippets/ActivityPub | 12 +- caddy/snippets/SecurityHeaders | 7 +- caddy/snippets/TrafficAnalytics | 9 +- compose.yml | 278 ++++-- docs/bundle-v1.md | 131 +++ docs/caddy.md | 89 ++ docs/configuration.md | 283 ++++++ docs/ghost-cli-replacement.md | 1599 ++++++++++++++++++++++++++++++ ghost.env.example | 35 + help | 55 +- scripts/caddy.sh | 45 + scripts/config-to-env.js | 29 +- scripts/config.sh | 54 + scripts/lib/caddy.sh | 280 ++++++ scripts/lib/common.sh | 38 + scripts/lib/compose.sh | 97 ++ scripts/lib/config.sh | 282 ++++++ scripts/lib/env.sh | 257 +++++ scripts/lib/fs.sh | 66 ++ scripts/migrate.sh | 18 +- tests/caddy.test.mjs | 157 +++ tests/compose-matrix.test.mjs | 217 ++++ tests/config.test.mjs | 245 +++++ tests/env-compose.test.mjs | 120 +++ tests/env.test.mjs | 221 +++++ tests/helpers.mjs | 177 ++++ tests/ingress.test.mjs | 234 +++++ tests/legacy-migrate.test.mjs | 68 ++ 43 files changed, 5539 insertions(+), 253 deletions(-) create mode 100644 .github/workflows/test.yml create mode 100644 .shellcheckrc create mode 100644 caddy/Caddyfile create mode 100644 caddy/custom/.gitignore create mode 100644 caddy/custom/README.md create mode 100644 caddy/global/.gitignore create mode 100644 caddy/global/README.md create mode 100644 caddy/sites/.gitignore create mode 100644 docs/bundle-v1.md create mode 100644 docs/caddy.md create mode 100644 docs/configuration.md create mode 100644 docs/ghost-cli-replacement.md create mode 100644 ghost.env.example create mode 100755 scripts/caddy.sh create mode 100755 scripts/config.sh create mode 100644 scripts/lib/caddy.sh create mode 100644 scripts/lib/common.sh create mode 100644 scripts/lib/compose.sh create mode 100644 scripts/lib/config.sh create mode 100644 scripts/lib/env.sh create mode 100644 scripts/lib/fs.sh create mode 100644 tests/caddy.test.mjs create mode 100644 tests/compose-matrix.test.mjs create mode 100644 tests/config.test.mjs create mode 100644 tests/env-compose.test.mjs create mode 100644 tests/env.test.mjs create mode 100644 tests/helpers.mjs create mode 100644 tests/ingress.test.mjs create mode 100644 tests/legacy-migrate.test.mjs diff --git a/.env.example b/.env.example index c6337440..af88e089 100644 --- a/.env.example +++ b/.env.example @@ -1,66 +1,128 @@ -# Use the below flags to enable the Analytics or ActivityPub containers as well -# COMPOSE_PROFILES=analytics,activitypub - -# Ghost domain -# Custom public domain Ghost will run on -DOMAIN=example.com - -# Ghost Admin domain -# If you have Ghost Admin setup on a separate domain uncomment the line below and add the domain -# You also need to uncomment the corresponding block in your Caddyfile -# ADMIN_DOMAIN= - -# Ghost ports -# Ports where Ghost will listen for HTTP traffic. -# Change these if the default ports are in use, or if Ghost is behind a reverse proxy. -HTTP_PORT=80 -HTTPS_PORT=443 - -# Database settings -# All database settings must not be changed once the database is initialised -DATABASE_ROOT_PASSWORD=reallysecurerootpassword -# DATABASE_USER=optionalusername -DATABASE_PASSWORD=ghostpassword - -# ActivityPub -# If you'd prefer to self-host ActivityPub yourself uncomment the line below -# ACTIVITYPUB_TARGET=activitypub:8080 - -# Tinybird configuration -# If you want to run Analytics, paste the output from `docker compose run --rm tinybird-login get-tokens` below -# TINYBIRD_API_URL=https://api.tinybird.co -# TINYBIRD_TRACKER_TOKEN=p.eyJxxxxx -# TINYBIRD_ADMIN_TOKEN=p.eyJxxxxx -# TINYBIRD_WORKSPACE_ID=xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx - -# Ghost configuration (https://ghost.org/docs/config/) - -# SMTP Email (https://ghost.org/docs/config/#mail) -# Transactional email is required for logins, account creation (staff invites), password resets and other features -# This is not related to bulk mail / newsletter sending -mail__transport=SMTP -mail__options__host=smtp.example.com -mail__options__port=465 -mail__options__secure=true -mail__options__auth__user=support@example.com -mail__options__auth__pass=1234567890 -mail__from="'Acme Support' " - -# Advanced customizations - -# Force Ghost version -# You should only do this if you need to pin a specific version -# The update commands won't work -# GHOST_VERSION=6-alpine - -# Port Ghost should listen on -# You should only need to edit this if you want to host -# multiple sites on the same server -# GHOST_PORT=2368 - -# Data locations -# Location to store uploaded data -UPLOAD_LOCATION=./data/ghost - -# Location for database data -MYSQL_DATA_LOCATION=./data/mysql +# Compose and operator settings. +# +# This file is read by Docker Compose for `${...}` interpolation. It is NEVER +# passed into the Ghost container: it holds infrastructure credentials that +# Ghost must not receive. Ghost's own configuration lives in `ghost.env` +# (see `ghost.env.example`). +# +# Values are written by the tooling in double-quoted form. If you edit by hand, +# remember that Compose interpolates unquoted and double-quoted values: write a +# literal dollar sign as `$$`. `scripts/config.sh set .env KEY VALUE` does this +# for you, and `scripts/config.sh validate` reports values that would be +# interpolated by accident. + +# --- Site mode ------------------------------------------------------------- +# Exactly one site mode must be selected: `local` or `production`. +# Optional per-site profiles are added to the same list: `analytics`, +# `activitypub`. Profiles are additive; adding an optional profile never +# changes the site mode. +COMPOSE_PROFILES="production" +SITE_MODE="production" + +# Stable project identity. Kept independent of the directory name so a site can +# be moved. Also the suffix of the unique service aliases +# (`ghost-${COMPOSE_PROJECT_NAME}` and friends) used by generated proxy routes. +COMPOSE_PROJECT_NAME="ghost-example-com" + +# Absolute path of this site directory. Moving a site requires updating this +# and re-validating the bind mounts. +PROJECT_DIR="/opt/ghost/example.com" + +# --- Ghost ---------------------------------------------------------------- +NODE_ENV="production" + +# Public URL of the site, without a trailing slash. +# production: https://example.com +# local: http://localhost:2368 +URL="https://example.com" + +# Public domain served by Caddy. Production only. +DOMAIN="example.com" + +# Optional separate Ghost Admin domain. Leave unset when there is none. +# ADMIN_DOMAIN="admin.example.com" +# ADMIN_URL="https://admin.example.com" + +# Optional `www.` redirect target rendered into the generated Caddy routes. +# WWW_REDIRECT="www.example.com" + +# Exact Ghost image pin, resolved at installation. The `next` variants install +# Ghost directly under /home/ghost rather than the older +# /var/lib/ghost/versions/ layout. +GHOST_IMAGE="ghost" +GHOST_VERSION="6-next-alpine" + +# Paths inside the Ghost image. The defaults match the `next` variants. Pinning +# a GHOST_VERSION with the older layout means setting both of these to +# /var/lib/ghost/content and /var/lib/ghost/current/core/server/data/tinybird. +# GHOST_CONTENT_PATH="/home/ghost/content" +# GHOST_TINYBIRD_PATH="/home/ghost/core/server/data/tinybird" + +# Ghost is always published on the loopback interface only, so a +# bring-your-own reverse proxy can reach it without exposing it publicly. +GHOST_PORT="2368" + +# Readiness probe path. Change this only for a subdirectory install. +# GHOST_HEALTHCHECK_PATH="/ghost/api/admin/site/" + +# --- Ingress (production) -------------------------------------------------- +HTTP_PORT="80" +HTTPS_PORT="443" + +# --- Lifecycle ------------------------------------------------------------ +# Applied to long-running services only. One-shot jobs keep `restart: "no"`. +# production: unless-stopped +# local: no +RESTART_POLICY="unless-stopped" + +# --- Database ------------------------------------------------------------- +# Parameterized now so backup, restore and import all share one connection +# contract. The defaults are correct for a single-site installation. +DATABASE_HOST="db" +DATABASE_PORT="3306" +DATABASE_NAME="ghost" +DATABASE_USER="ghost" +DATABASE_PASSWORD="change-me-application-password" + +# Infrastructure credential. Ghost never receives this. +DATABASE_ROOT_PASSWORD="change-me-root-password" + +# Extra databases created on first initialisation. +DATABASE_EXTRA_DATABASES="activitypub" + +# --- Data locations ------------------------------------------------------- +UPLOAD_LOCATION="./data/ghost" +MYSQL_DATA_LOCATION="./data/mysql" + +# --- Container logs ------------------------------------------------------- +# Container logs are capped so a long-running site cannot fill the disk. +LOG_DRIVER="json-file" +LOG_MAX_SIZE="10m" +LOG_MAX_FILE="3" + +# --- ActivityPub (optional, per-site) ------------------------------------- +# Add `activitypub` to COMPOSE_PROFILES to run this site's own ActivityPub +# service, its migration job and its database. Each site owns its ActivityPub +# database, storage and serving URL. +# +# Resource cost: one long-running Node service (~150-250 MB RSS), one one-shot +# migration job per start, and one extra MySQL database. +# +# ACTIVITYPUB_DATABASE_NAME="activitypub" +# Used only when the `activitypub` profile is NOT enabled, to point the +# generated routes at the hosted service. +# ACTIVITYPUB_TARGET="https://ap.ghost.org" + +# --- Analytics (optional, per-site) --------------------------------------- +# Add `analytics` to COMPOSE_PROFILES. Tinybird credentials, workspace +# selection and schema deployment belong to this site; see TINYBIRD.md. +# +# Resource cost: one long-running proxy service (~100-200 MB RSS) plus the +# one-shot Tinybird login/sync/deploy jobs, and a Tinybird workspace. +# +# TINYBIRD_API_URL="https://api.tinybird.co" +# TINYBIRD_TRACKER_TOKEN="p.eyJxxxxx" +# TINYBIRD_ADMIN_TOKEN="p.eyJxxxxx" +# TINYBIRD_WORKSPACE_ID="xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" +# SALT_STORE_TYPE="file" +# TRAFFIC_ANALYTICS_LOG_LEVEL="info" diff --git a/.github/workflows/shellcheck.yml b/.github/workflows/shellcheck.yml index b6077032..b61de763 100644 --- a/.github/workflows/shellcheck.yml +++ b/.github/workflows/shellcheck.yml @@ -14,4 +14,4 @@ jobs: steps: - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 - name: Run ShellCheck - run: find . -type f -name "*.sh" -exec shellcheck {} + + run: find . -type f -name "*.sh" -not -path "./.git/*" -exec shellcheck {} + diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml new file mode 100644 index 00000000..2f67935a --- /dev/null +++ b/.github/workflows/test.yml @@ -0,0 +1,64 @@ +--- +name: "Tests" +on: + pull_request: + push: + branches: + - main + - renovate/* + +env: + # Declared minimum, kept in sync with GD_MIN_COMPOSE_VERSION in + # scripts/lib/compose.sh. + MIN_COMPOSE_VERSION: "2.24.0" + +jobs: + test: + name: Helpers and mode matrix + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + + - uses: actions/setup-node@2028fbc5c25fe9cf00d9f06a71cc4710d4507903 # v6.0.0 + with: + node-version: "22" + + - name: Check the runtime prerequisites + run: | + set -eu + jq --version + docker version + docker compose version + + - name: Install the minimum supported Docker Compose + run: | + set -eu + mkdir -p "$RUNNER_TEMP/min-compose" + curl -fsSL -o "$RUNNER_TEMP/min-compose/docker-compose" \ + "https://github.com/docker/compose/releases/download/v${MIN_COMPOSE_VERSION}/docker-compose-linux-x86_64" + chmod +x "$RUNNER_TEMP/min-compose/docker-compose" + "$RUNNER_TEMP/min-compose/docker-compose" version + + - name: Run the test suite + env: + GD_TEST_MIN_COMPOSE: ${{ runner.temp }}/min-compose/docker-compose + run: node --test --test-timeout=120000 tests/*.test.mjs + + ingress: + name: Ingress smoke tests + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + + - uses: actions/setup-node@2028fbc5c25fe9cf00d9f06a71cc4710d4507903 # v6.0.0 + with: + node-version: "22" + + - name: Run the local and production ingress smoke tests + env: + GD_TEST_INGRESS: "1" + run: node --test --test-timeout=900000 tests/ingress.test.mjs + + - name: Show service logs on failure + if: failure() + run: docker ps -a && docker compose ls || true diff --git a/.gitignore b/.gitignore index d005b221..48466d73 100644 --- a/.gitignore +++ b/.gitignore @@ -73,3 +73,23 @@ typings/ # Where we store docker data by default data + +# Ghost Docker generated/operator files +# Application configuration (credentials) +ghost.env +# Installation metadata +.ghost-docker.json +# Generated and operator-managed Caddy routes, and the validation staging tree +caddy/sites/* +!caddy/sites/.gitignore +caddy/custom/* +!caddy/custom/.gitignore +!caddy/custom/README.md +caddy/global/* +!caddy/global/.gitignore +!caddy/global/README.md +caddy/.staging/ +# Operator's own Caddyfile from pre-1.0 installations (see the S6 migration) +caddy/Caddyfile.local +# Backups written by the helpers +*.bak.* diff --git a/.shellcheckrc b/.shellcheckrc new file mode 100644 index 00000000..90b08082 --- /dev/null +++ b/.shellcheckrc @@ -0,0 +1,4 @@ +# Follow `.` / `source` directives so the shared libraries in scripts/lib are +# analysed together with their callers. +external-sources=true +source-path=SCRIPTDIR diff --git a/CLAUDE.md b/CLAUDE.md index 3f23002e..f60b0d74 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -17,13 +17,28 @@ The project uses Docker Compose to orchestrate these services: 5. **ActivityPub** (optional profile) - Federated social networking support 6. **Supporting services** - Tinybird setup tools and ActivityPub migrations -Services communicate internally via Docker networks. Caddy handles all external traffic routing including special paths for analytics (`/_tinybird`) and ActivityPub (`/.well-known/`, `/activitypub/`). +Services communicate internally via Docker networks, addressed through unique +per-site aliases (`ghost-${COMPOSE_PROJECT_NAME}`, `db-…`, `activitypub-…`, +`traffic-analytics-…`) rather than bare service names. Caddy handles all +external traffic routing, including analytics (`/.ghost/analytics/`) and +ActivityPub (`/.ghost/activitypub/`, `/.well-known/webfinger`, +`/.well-known/nodeinfo`). + +Site mode is selected through `COMPOSE_PROFILES` and exactly one of `local` or +`production` must be present. Optional per-site profiles (`analytics`, +`activitypub`) are additive and never change the mode. Caddy is part of +`production`, not optional; bring-your-own-proxy is an unsupported manual edit +of `compose.yml`. `supervisor` is a +reserved profile name with no service yet. + +Long-running services use `restart: ${RESTART_POLICY:-unless-stopped}`; +one-shot jobs (`activitypub-migrate`, `tinybird-*`) keep `restart: "no"`. ## Common Commands ```bash # Core operations -docker compose up -d # Start Ghost + MySQL + Caddy +docker compose up -d # Start the services for the selected mode docker compose down # Stop all services docker compose logs -f [service] # View logs (e.g., ghost, mysql, caddy) docker compose ps # Check service status @@ -42,23 +57,55 @@ docker compose --profile=analytics up tinybird-deploy # Deploy configuration # Development & debugging docker compose exec ghost sh # Access Ghost container shell -docker compose exec mysql mysql -u root -p # Access MySQL CLI -``` +docker compose exec db mysql -u root -p # Access MySQL CLI -## Configuration +# Configuration helpers (never source an env file; always atomic writes) +scripts/config.sh validate # Validate .env and ghost.env by mode +scripts/config.sh set ghost.env KEY VAL # Write one value safely +scripts/config.sh unset ghost.env KEY -All configuration is done via environment variables. Key patterns: +# Caddy routes +scripts/caddy.sh apply # Render, validate, install, reload, verify + +# Tests (Node 20+ built-in runner, no dependencies and no package.json; +# docker tests skip without a daemon) +node --test --test-timeout=120000 tests/*.test.mjs +GD_TEST_INGRESS=1 node --test --test-timeout=900000 tests/ingress.test.mjs +``` -- **Required variables**: `DOMAIN`, `DATABASE_PASSWORD`, `DATABASE_ROOT_PASSWORD` -- **Ghost config pattern**: `section__subsection__key=value` (e.g., `mail__options__service=Mailgun`) -- **Developer experiments**: Must set `labs__publicAPI=true` for analytics/ActivityPub features -- **Data persistence**: Volumes stored in `./data/ghost` and `./data/mysql` +## Configuration -### Key configuration files: -- `.env` - Main environment configuration (create from `.env.example`) -- `compose.yml` - Docker Compose service definitions -- `Caddyfile` - Reverse proxy routing configuration -- `mysql-init/create-multiple-databases.sh` - MySQL multi-database initialization +Configuration is split by audience. See `docs/configuration.md` for the full +contract. + +- `.env` — Compose and operator settings: project identity, site mode, ports, + data locations, restart policy, and infrastructure credentials such as + `DATABASE_ROOT_PASSWORD`. **Never passed into the Ghost container.** +- `ghost.env` — Ghost application settings only, in `section__subsection__key` + form. The sole `env_file` of the ghost service. Container-owned keys (`url`, + `admin__url`, `NODE_ENV`, `server__*`, `paths__*`, `database__*`) are set as + Compose `environment` entries and are rejected here. That rejection is + derived by asking `docker compose config` what the container receives, not + from a hardcoded list, so it cannot drift from `compose.yml`. + +Compose interpolates dotenv values, including inside double quotes. Write a +literal dollar sign as `$$`, or use `scripts/config.sh set`, which serializes +safely. Never source an env file; use `scripts/lib/env.sh`. + +- **Developer experiments**: set `labs__publicAPI=true` in `ghost.env` for the + analytics/ActivityPub features +- **Data persistence**: `UPLOAD_LOCATION` and `MYSQL_DATA_LOCATION` + +### Key files +- `.env` / `.env.example` — operator configuration +- `ghost.env` / `ghost.env.example` — application configuration +- `.ghost-docker.json` — generated installation metadata (schema v1, written from S2) +- `compose.yml` — service definitions +- `caddy/Caddyfile` — tracked generic entry point; site routes are generated + into `caddy/sites/`, operator routes live in `caddy/custom/`, global options + in `caddy/global/` +- `scripts/lib/*.sh` — shared helpers (env, fs, compose, config, caddy) +- `mysql-init/create-multiple-databases.sh` — MySQL multi-database initialization ## Migration from Ghost CLI @@ -74,21 +121,55 @@ The repository includes comprehensive migration tools: - Creates recovery script with clear restoration instructions - Sets up Docker Compose environment -- `scripts/config-to-env.js` - Converts Ghost JSON config to .env format +- `scripts/config-to-env.js` - Converts Ghost JSON config to ghost.env format. + CommonJS; there is no package.json in this repository, so `.js` is CommonJS + by default. This is the only host Node dependency, and `install.sh --import` + removes it ## Development Workflow -1. Clone repository and copy `.env.example` to `.env` -2. Configure required environment variables (domain, passwords) -3. Run `docker compose up -d` to start services -4. Access Ghost at `https://DOMAIN` (Caddy handles SSL automatically) -5. Monitor logs with `docker compose logs -f ghost` +1. Copy `.env.example` to `.env` and `ghost.env.example` to `ghost.env` + (both mode `0600`) +2. Configure the required variables for the mode; run `scripts/config.sh validate` +3. Production only: `scripts/caddy.sh apply` +4. Run `docker compose up -d` +5. Access Ghost at `URL`; monitor with `docker compose logs -f ghost` + +All shell code is `bash`, kept compatible with bash 3.2 so it runs on macOS's +system bash: no `declare -A`, `mapfile`, `${v,,}` or namerefs. It must pass +`shellcheck`, which picks the dialect from the shebang. Tests are `.mjs` files +in `tests/` run by Node's built-in runner: the shell libraries are exercised +through a real shell via `tests/helpers.mjs`, while fixtures, structured-output +parsing and assertions are JavaScript. For analytics setup, see `TINYBIRD.md` for detailed instructions. +## Implementation plan + +`docs/ghost-cli-replacement.md` is the plan this repository is being built +against, including the architecture contracts (§2) and the step breakdown (§3). +Read the relevant step and its dependencies before implementing one, and amend +the affected contract in the same pull request when a decision changes. Steps +land as stacked pull requests in the dependency order given in §3. + ## Important Notes -- Ghost runs internally on port 2368; Caddy exposes it on 80/443 +- Runtime prerequisites: `bash`, Docker Engine 25.0.0, Docker Compose v2.24.0, + and `jq` (used by the helpers for JSON). `install.sh` verifies + them in preflight; `scripts/migrate.sh` already required `jq` +- Node.js is a development/test requirement only, never needed to run a site. + The exception is the legacy `scripts/migrate.sh`, retired by `install.sh --import` +- Ghost runs internally on port 2368 and is published on `127.0.0.1` only; + Caddy exposes 80/443 in production +- The default image is a `next` variant, which installs Ghost directly under + `/home/ghost` instead of the older `/var/lib/ghost/versions/` + `current` + layout. `GHOST_CONTENT_PATH` and `GHOST_TINYBIRD_PATH` exist so pinning an + older `GHOST_VERSION` stays possible +- Helpers use one Compose contract: `docker compose --project-directory "$DIR" + -f "$DIR/compose.yml"`, never `-C`, never `COMPOSE_FILE` +- `ghost` and `db` have real readiness health checks; a running container is + not readiness +- Container logs are capped via `LOG_MAX_SIZE` / `LOG_MAX_FILE` - Email configuration is critical even without newsletter features (used for admin notifications) - MySQL health checks ensure database is ready before Ghost starts - The compose file uses yaml-language-server schema for IDE completion support diff --git a/README.md b/README.md index 1f8cb923..464f9498 100644 --- a/README.md +++ b/README.md @@ -1,15 +1,125 @@ # Ghost Docker -Configuration to run Ghost and its services with Docker Compose +Configuration to run Ghost and its services with Docker Compose. + +Requires **bash**, **Docker Engine 25.0+**, **Docker Compose v2.24+** and **jq**. + +## Configuration + +Two files, deliberately separate: + +- `.env` — Compose and operator settings, including infrastructure credentials. + Never passed into the Ghost container. Start from [`.env.example`](.env.example). +- `ghost.env` — Ghost application settings only, the sole `env_file` of the + `ghost` service. Start from [`ghost.env.example`](ghost.env.example). + +```sh +cp .env.example .env && chmod 0600 .env +cp ghost.env.example ghost.env && chmod 0600 ghost.env +scripts/config.sh validate +``` + +See [docs/configuration.md](docs/configuration.md) for the full contract: +value encoding, site modes, profiles, lifecycle, service aliases and metadata. + +## Site modes + +Exactly one site mode is selected through `COMPOSE_PROFILES`: + +```sh +# Local: Ghost + MySQL, published on 127.0.0.1:${GHOST_PORT} +COMPOSE_PROFILES=local docker compose up -d + +# Production: Ghost + MySQL + Caddy with automatic HTTPS +COMPOSE_PROFILES=production docker compose up -d +``` + +Optional per-site profiles are additive: + +```sh +COMPOSE_PROFILES=production,analytics,activitypub docker compose up -d +``` + +## Routing + +Production routes are generated, validated and installed by: + +```sh +scripts/caddy.sh apply +``` + +Add your own routes as `.caddy` files in `caddy/custom/` and global options in +`caddy/global/`; neither is ever overwritten. See [docs/caddy.md](docs/caddy.md). + +## Day-to-day + +```sh +docker compose ps +docker compose logs -f ghost +docker compose pull && docker compose up -d +docker compose exec ghost sh +``` + +Run `./help` for a longer list. + +## Analytics + +See [TINYBIRD.md](TINYBIRD.md). + +## Implementation plan + +[docs/ghost-cli-replacement.md](docs/ghost-cli-replacement.md) is the plan this +repository is being built against: the architecture contracts, the step +breakdown, and the decisions behind them. Each step lands as its own pull +request, stacked on the ones it depends on. + +## Migrating from Ghost-CLI + +`scripts/migrate.sh` remains the supported path today; it writes Ghost +configuration into `ghost.env`. The migration bundle format that replaces it is +specified in [docs/bundle-v1.md](docs/bundle-v1.md). + +## Upgrading an existing checkout + +This layout is a breaking change for installations made before it landed — +`caddy/Caddyfile` is now tracked, the Caddy snippets take import arguments, and +Ghost configuration moves out of `.env`. See +[Existing installations](docs/configuration.md#existing-installations) before +pulling. ## IPv6 networking -IPv6 networking is opt-in because it requires newer Docker and Docker Compose versions than the base setup. Enable it by including the IPv6 override file: +IPv6 networking is opt-in because it requires newer Docker and Docker Compose +versions than the base setup. Enable it by including the IPv6 override file: ```sh docker compose -f compose.yml -f compose.ipv6.yml up -d ``` -# Copyright & License +The helper scripts take the same override through `GD_COMPOSE_OVERRIDES`: + +```sh +GD_COMPOSE_OVERRIDES=compose.ipv6.yml scripts/caddy.sh validate +``` + +## Tests + +The test suite runs on Node.js 20+ using its built-in test runner. There are no +dependencies and no `package.json`: Node is a development requirement only, and +the repository is not a Node package. + +```sh +# helpers, mode matrix, Caddy routes +node --test --test-timeout=120000 tests/*.test.mjs + +# local and production ingress, against real containers +GD_TEST_INGRESS=1 node --test --test-timeout=900000 tests/ingress.test.mjs +``` + +Docker-dependent tests are skipped when no daemon is reachable. Set +`GD_TEST_MIN_COMPOSE=/path/to/docker-compose` to also run the mode matrix +against the declared minimum Compose. + +# Copyright & License Copyright (c) 2013-2026 Ghost Foundation - Released under the [MIT license](LICENSE). diff --git a/TINYBIRD.md b/TINYBIRD.md index 874e27f4..a1fdb33b 100644 --- a/TINYBIRD.md +++ b/TINYBIRD.md @@ -7,7 +7,14 @@ Note: Currently Traffic Analytics features are behind a feature flag. For now, y 1. Run `docker compose run --rm tinybird-sync`. This will copy the Tinybird files from the Ghost container into a shared volume. The service should log "Tinybird files synced into shared volume.", then exit. 1. Run `docker compose run --rm tinybird-deploy` and wait for the service to exit successfully. This will create your Tinybird datasources, pipes and API endpoints. It may take a minute or two to complete the first time. You should see "Deployment #1 is live!" in your terminal before the service exits. 1. Run `docker compose run --rm tinybird-login get-tokens` -1. Copy and paste the values from the previous step into your `.env` file +1. Copy and paste the values from the previous step into your `.env` file (Tinybird credentials are operator settings, not Ghost application settings) 1. Run `docker compose --profile=analytics up -d` to start all services in the background -1. Add `analytics` to `COMPOSE_PROFILES=` in the top of your `.env` file to automatically include the `analytics` profile when running `docker compose` commands +1. Add `analytics` to `COMPOSE_PROFILES` in your `.env` file, alongside the site mode, to include the `analytics` profile automatically when running `docker compose` commands. Profiles are additive: adding `analytics` does not change the site mode. +1. Run `scripts/caddy.sh apply` so the generated routes proxy `/.ghost/analytics/` to this site's analytics service (production only) 1. At this point, everything should be working. You can test it's working by visiting your site's homepage, then checking the Stats page in Ghost Admin — you should see a view recorded. + +Tinybird credentials, workspace selection and schema deployment belong to this +site. `tinybird-sync` and `tinybird-deploy` are distinct steps and are one-shot +jobs: they keep `restart: "no"` and stay stopped after they complete. Copying +new Tinybird files into the shared volume is not a deployment — a Ghost upgrade +must run `tinybird-deploy` after `tinybird-sync`. diff --git a/caddy/Caddyfile b/caddy/Caddyfile new file mode 100644 index 00000000..8571650f --- /dev/null +++ b/caddy/Caddyfile @@ -0,0 +1,17 @@ +# Generic entry point. This file is tracked and should not be edited. +# +# Site routes are generated by `scripts/caddy.sh apply` into caddy/sites/. +# Add your own routes as .caddy files in caddy/custom/, and global options +# (ACME account email, DNS provider, internal CA, ...) as .caddy files in +# caddy/global/. Neither is generated or overwritten. +# +# The directories are placeholders so that a candidate configuration can be +# validated with this exact file before it is installed. + +{ + import {$CADDY_GLOBAL_DIR:/etc/caddy/global}/*.caddy +} + +import {$CADDY_SITES_DIR:/etc/caddy/sites}/*.caddy + +import {$CADDY_CUSTOM_DIR:/etc/caddy/custom}/*.caddy diff --git a/caddy/Caddyfile.example b/caddy/Caddyfile.example index 17b6e027..9cf38f3a 100644 --- a/caddy/Caddyfile.example +++ b/caddy/Caddyfile.example @@ -1,60 +1,55 @@ -{$DOMAIN} { - import snippets/Logging +# Reference example, not used by the running stack. +# +# The stack serves `caddy/Caddyfile`, which imports generated routes from +# `caddy/sites/`, your own routes from `caddy/custom/`, and global options +# from `caddy/global/`. Generate the site routes with: +# +# scripts/caddy.sh apply +# +# This file shows what those generated routes look like, for a +# bring-your-own-proxy setup or to compare against a hand-written Caddyfile +# from a pre-1.0 installation. Snippets are imported by absolute path and take +# their upstreams and domains as arguments: a missing argument is only a Caddy +# warning and fails at runtime. +# +# Substitute your own values for `example.com` and `my-site` +# (your COMPOSE_PROJECT_NAME). - # Traffic Analytics service - import snippets/TrafficAnalytics +example.com { + import /etc/caddy/snippets/Logging - # ActivityPub Service - import snippets/ActivityPub + # Traffic Analytics service (only when the `analytics` profile is enabled) + import /etc/caddy/snippets/TrafficAnalytics traffic-analytics-my-site:3000 - # Default proxy everything else to Ghost + # ActivityPub service. Use `activitypub-my-site:8080` when the + # `activitypub` profile is enabled, or the hosted service otherwise. + import /etc/caddy/snippets/ActivityPub https://ap.ghost.org + + # Default: proxy everything else to Ghost, through its unique alias. handle { - reverse_proxy ghost:2368 + reverse_proxy ghost-my-site:2368 } - # Optional: Enable gzip compression + # Optional: gzip compression encode gzip - # Optional: Add security headers - import snippets/SecurityHeaders + # Optional: security headers. The argument is the admin domain allowed to + # frame this site, or "" when there is none. + import /etc/caddy/snippets/SecurityHeaders "" } -# Separate admin domains -# To use a separate domain for Ghost Admin uncomment the block below (recommended) -# {$ADMIN_DOMAIN} { -# import snippets/Logging -# -# # Traffic Analytics service -# import snippets/TrafficAnalytics -# -# # ActivityPub Service -# import snippets/ActivityPub -# -# # Default proxy everything else to Ghost -# handle { -# reverse_proxy ghost:2368 -# } -# -# # Optional: Enable gzip compression -# encode gzip -# -# # Optional: Add security headers -# import snippets/SecurityHeaders -# } +# Separate admin domain +# Set ADMIN_DOMAIN in .env and `scripts/caddy.sh apply` renders this block for +# you. It is identical to the block above, served on the admin domain, with the +# admin domain passed to SecurityHeaders. # Redirect www -> root domain -# To redirect the www variant of your domain to the non-www variant uncomment the 4 lines below -# Note: You must have DNS setup correctly for both domains for this to work -# www.{$DOMAIN} { -# import snippets/Logging -# redir https://{$DOMAIN}{uri} -# } - -# Redirect root -> www domain -# To redirect the non-www variant of your domain to the www variant uncomment the 4 lines below and change CHANGE_ME to your root domain -# Note: You must have DNS setup correctly for both domains for this to work -# When using ActivityPub with a www. domain, you must enable this redirect for ActivityPub to work correctly -# CHANGE_ME { -# import snippets/Logging -# redir https://{$DOMAIN}{uri} +# Set WWW_REDIRECT=www.example.com in .env and `scripts/caddy.sh apply` renders: +# +# www.example.com { +# import /etc/caddy/snippets/Logging +# redir https://example.com{uri} # } +# +# Note: DNS must be set up for both names. When using ActivityPub with a `www.` +# domain, this redirect is required for ActivityPub to work correctly. diff --git a/caddy/custom/.gitignore b/caddy/custom/.gitignore new file mode 100644 index 00000000..cd8b9b82 --- /dev/null +++ b/caddy/custom/.gitignore @@ -0,0 +1,4 @@ +# Operator-managed routes. Never generated, never overwritten. +* +!.gitignore +!README.md diff --git a/caddy/custom/README.md b/caddy/custom/README.md new file mode 100644 index 00000000..2b105512 --- /dev/null +++ b/caddy/custom/README.md @@ -0,0 +1,18 @@ +# Custom Caddy routes + +Files matching `*.caddy` in this directory are imported after the generated +site routes in `caddy/sites/`. They are never generated and never overwritten +by `scripts/caddy.sh`, and they are validated together with the generated +routes before any configuration is installed or reloaded. + +Snippets live in `caddy/snippets/` and are imported by absolute path, for +example: + +```caddyfile +status.example.com { + import /etc/caddy/snippets/Logging + reverse_proxy 172.17.0.1:9000 +} +``` + +Ghost itself is reachable on the Compose network as `ghost-$COMPOSE_PROJECT_NAME:2368`. diff --git a/caddy/global/.gitignore b/caddy/global/.gitignore new file mode 100644 index 00000000..6e67c29c --- /dev/null +++ b/caddy/global/.gitignore @@ -0,0 +1,4 @@ +# Operator-managed global options. Never generated, never overwritten. +* +!.gitignore +!README.md diff --git a/caddy/global/README.md b/caddy/global/README.md new file mode 100644 index 00000000..f461e91c --- /dev/null +++ b/caddy/global/README.md @@ -0,0 +1,18 @@ +# Caddy global options + +Files matching `*.caddy` in this directory are imported into Caddy's global +options block. Use them for settings that apply to the whole server rather than +to one site, for example: + +```caddyfile +email ops@example.com +``` + +```caddyfile +# Issue certificates from Caddy's internal CA instead of a public ACME CA. +# Useful for staging hosts and test domains. +local_certs +``` + +They are validated together with the generated site routes before any +configuration is installed or reloaded. diff --git a/caddy/sites/.gitignore b/caddy/sites/.gitignore new file mode 100644 index 00000000..f4b944f3 --- /dev/null +++ b/caddy/sites/.gitignore @@ -0,0 +1,3 @@ +# Generated site routes. Managed by scripts/caddy.sh. +* +!.gitignore diff --git a/caddy/snippets/ActivityPub b/caddy/snippets/ActivityPub index e62d651e..93090dad 100644 --- a/caddy/snippets/ActivityPub +++ b/caddy/snippets/ActivityPub @@ -1,13 +1,15 @@ -# ActivityPub -# Proxy activitypub requests /.ghost/activitypub/ +# ActivityPub. +# +# args[0] — ActivityPub upstream. Either the site's own service +# (activitypub-:8080) or the hosted service. handle /.ghost/activitypub/* { - reverse_proxy {$ACTIVITYPUB_TARGET} + reverse_proxy {args[0]} } handle /.well-known/webfinger { - reverse_proxy {$ACTIVITYPUB_TARGET} + reverse_proxy {args[0]} } handle /.well-known/nodeinfo { - reverse_proxy {$ACTIVITYPUB_TARGET} + reverse_proxy {args[0]} } diff --git a/caddy/snippets/SecurityHeaders b/caddy/snippets/SecurityHeaders index a0b0d4ef..90705a78 100644 --- a/caddy/snippets/SecurityHeaders +++ b/caddy/snippets/SecurityHeaders @@ -1,3 +1,8 @@ +# Security headers. +# +# args[0] — admin domain allowed to frame this site, or "" when there is none. +# The argument is mandatory: an omitted argument only produces a Caddy warning +# and would silently ship a broken frame-ancestors policy. header { # Enable HSTS Strict-Transport-Security max-age=31536000; @@ -8,5 +13,5 @@ header { # Referrer policy Referrer-Policy strict-origin-when-cross-origin # Prevent embedding in external iframes - Content-Security-Policy "frame-ancestors 'self' {$ADMIN_DOMAIN:}" + Content-Security-Policy "frame-ancestors 'self' {args[0]}" } diff --git a/caddy/snippets/TrafficAnalytics b/caddy/snippets/TrafficAnalytics index 53a81f2c..611557e8 100644 --- a/caddy/snippets/TrafficAnalytics +++ b/caddy/snippets/TrafficAnalytics @@ -1,6 +1,11 @@ -# Proxy analytics requests with any prefix (e.g. /.ghost/analytics/ or /blog/.ghost/analytics/) +# Traffic Analytics. +# +# args[0] — analytics upstream, for example traffic-analytics-:3000. +# +# Proxy analytics requests with any prefix (e.g. /.ghost/analytics/ or +# /blog/.ghost/analytics/). @analytics_paths path_regexp analytics_match ^(.*)/\.ghost/analytics(.*)$ handle @analytics_paths { rewrite * {re.analytics_match.2} - reverse_proxy traffic-analytics:3000 + reverse_proxy {args[0]} } diff --git a/compose.yml b/compose.yml index 2831ade3..1c5ba2fe 100644 --- a/compose.yml +++ b/compose.yml @@ -1,48 +1,87 @@ --- # yaml-language-server: $schema=https://raw.githubusercontent.com/compose-spec/compose-spec/main/schema/compose-spec.json -services: - caddy: - image: caddy:2.10.2-alpine@sha256:953131cfea8e12bfe1c631a36308e9660e4389f0c3dfb3be957044d3ac92d446 - restart: always - ports: - - "${HTTP_PORT:-80}:80" - - "${HTTPS_PORT:-443}:443" - environment: - DOMAIN: ${DOMAIN:?DOMAIN environment variable is required} - ADMIN_DOMAIN: ${ADMIN_DOMAIN:-} - ACTIVITYPUB_TARGET: ${ACTIVITYPUB_TARGET:-https://ap.ghost.org} - volumes: - - ./caddy:/etc/caddy - - caddy_data:/data - - caddy_config:/config - depends_on: - - ghost - networks: - - ghost_network +# +# Ghost Docker — single-site Compose foundation. +# +# Requires Docker Engine >= 25.0.0 and Docker Compose >= v2.24.0. +# +# Operator/Compose settings live in `.env` (see `.env.example`). +# Ghost application settings live in `ghost.env` (see `ghost.env.example`). +# `.env` is NEVER passed into the Ghost container: it holds infrastructure +# credentials (MySQL root) that Ghost must not receive. +# +# Site modes are selected through COMPOSE_PROFILES and are additive: +# local ghost + db, Ghost published on 127.0.0.1:${GHOST_PORT} +# production ghost + db + caddy (HTTPS ingress) +# Optional, per-site profiles: analytics, activitypub. +# `supervisor` is reserved for the upgrade supervisor and defines no service yet. + +x-logging: &default-logging + driver: ${LOG_DRIVER:-json-file} + options: + max-size: ${LOG_MAX_SIZE:-10m} + max-file: "${LOG_MAX_FILE:-3}" +x-site-labels: &site-labels + org.ghost.docker.managed: "true" + org.ghost.docker.site: ${COMPOSE_PROJECT_NAME:-ghost} + org.ghost.docker.mode: ${SITE_MODE:-production} + +services: ghost: # Do not alter this without updating the Tinybird Sync container as well - image: ghost:${GHOST_VERSION:-6-alpine} - restart: always - # This is required to import current config when migrating + image: ${GHOST_IMAGE:-ghost}:${GHOST_VERSION:-6-next-alpine} + restart: ${RESTART_POLICY:-unless-stopped} + profiles: [local, production] + labels: + <<: *site-labels + org.ghost.docker.role: ghost + org.ghost.docker.lifecycle: long-running + logging: *default-logging + # Loopback only: production ingress is Caddy, and a bring-your-own proxy + # connects over the host loopback interface. + ports: + - "127.0.0.1:${GHOST_PORT:-2368}:2368" + # Application configuration only. Never `.env`. env_file: - - .env + - path: ghost.env + required: false + # Container-owned keys. These override anything set in ghost.env. environment: - NODE_ENV: production - url: https://${DOMAIN:?DOMAIN environment variable is required} - admin__url: ${ADMIN_DOMAIN:+https://${ADMIN_DOMAIN}} + NODE_ENV: ${NODE_ENV:-production} + url: ${URL:?URL is required (for example https://example.com or http://localhost:2368)} + admin__url: ${ADMIN_URL:-} + server__host: 0.0.0.0 + server__port: "2368" + paths__contentPath: ${GHOST_CONTENT_PATH:-/home/ghost/content} database__client: mysql - database__connection__host: db + database__connection__host: ${DATABASE_HOST:-db} + database__connection__port: ${DATABASE_PORT:-3306} database__connection__user: ${DATABASE_USER:-ghost} - database__connection__password: ${DATABASE_PASSWORD:?DATABASE_PASSWORD environment variable is required} - database__connection__database: ghost - tinybird__tracker__endpoint: https://${DOMAIN:?DOMAIN environment variable is required}/.ghost/analytics/api/v1/page_hit + database__connection__password: ${DATABASE_PASSWORD:?DATABASE_PASSWORD is required} + database__connection__database: ${DATABASE_NAME:-ghost} + tinybird__tracker__endpoint: ${URL}/.ghost/analytics/api/v1/page_hit + tinybird__tracker__datasource: analytics_events tinybird__adminToken: ${TINYBIRD_ADMIN_TOKEN:-} tinybird__workspaceId: ${TINYBIRD_WORKSPACE_ID:-} - tinybird__tracker__datasource: analytics_events tinybird__stats__endpoint: ${TINYBIRD_API_URL:-https://api.tinybird.co} volumes: - - ${UPLOAD_LOCATION:-./data/ghost}:/var/lib/ghost/content + - ${UPLOAD_LOCATION:-./data/ghost}:${GHOST_CONTENT_PATH:-/home/ghost/content} + # A real readiness probe: Ghost has booted and is serving its Admin API. + # A running container or a redirect is not readiness. + healthcheck: + test: + - CMD + - node + - -e + - >- + require('http').get('http://127.0.0.1:2368${GHOST_HEALTHCHECK_PATH:-/ghost/api/admin/site/}', + r => process.exit(r.statusCode < 400 ? 0 : 1)).on('error', () => process.exit(1)) + interval: 30s + timeout: 10s + start_period: 180s + start_interval: 5s + retries: 5 depends_on: db: condition: service_healthy @@ -56,102 +95,197 @@ services: condition: service_started required: false networks: - - ghost_network + ghost_network: + aliases: + # Unique alias. Generated proxy routes and helper clients address Ghost + # through this name, never through the bare `ghost` service name. + - ghost-${COMPOSE_PROJECT_NAME:-ghost} db: image: mysql:8.0.44@sha256:f37951fc3753a6a22d6c7bf6978c5e5fefcf6f31814d98c582524f98eae52b21 - restart: always + restart: ${RESTART_POLICY:-unless-stopped} + profiles: [local, production] + labels: + <<: *site-labels + org.ghost.docker.role: db + org.ghost.docker.lifecycle: long-running + logging: *default-logging expose: - "3306" environment: - MYSQL_ROOT_PASSWORD: ${DATABASE_ROOT_PASSWORD:?DATABASE_ROOT_PASSWORD environment variable is required} + MYSQL_ROOT_PASSWORD: ${DATABASE_ROOT_PASSWORD:?DATABASE_ROOT_PASSWORD is required} MYSQL_USER: ${DATABASE_USER:-ghost} - MYSQL_PASSWORD: ${DATABASE_PASSWORD:?DATABASE_PASSWORD environment variable is required} - MYSQL_DATABASE: ghost - MYSQL_MULTIPLE_DATABASES: activitypub + MYSQL_PASSWORD: ${DATABASE_PASSWORD:?DATABASE_PASSWORD is required} + MYSQL_DATABASE: ${DATABASE_NAME:-ghost} + MYSQL_MULTIPLE_DATABASES: ${DATABASE_EXTRA_DATABASES:-activitypub} volumes: - ${MYSQL_DATA_LOCATION:-./data/mysql}:/var/lib/mysql - ./mysql-init:/docker-entrypoint-initdb.d + # Verifies real client connectivity to the application database, not just + # that a server process answers. healthcheck: - test: mysqladmin ping -p$$MYSQL_ROOT_PASSWORD -h 127.0.0.1 - interval: 1s - start_period: 30s - start_interval: 10s - retries: 120 + test: + - CMD-SHELL + - >- + mysql -h 127.0.0.1 -u"$$MYSQL_USER" -p"$$MYSQL_PASSWORD" + -e 'SELECT 1' "$$MYSQL_DATABASE" >/dev/null 2>&1 + interval: 30s + timeout: 10s + start_period: 120s + start_interval: 2s + retries: 5 + networks: + ghost_network: + aliases: + - db-${COMPOSE_PROJECT_NAME:-ghost} + + caddy: + image: caddy:2.10.2-alpine@sha256:953131cfea8e12bfe1c631a36308e9660e4389f0c3dfb3be957044d3ac92d446 + restart: ${RESTART_POLICY:-unless-stopped} + profiles: [production] + labels: + <<: *site-labels + org.ghost.docker.role: caddy + org.ghost.docker.lifecycle: long-running + logging: *default-logging + ports: + - "${HTTP_PORT:-80}:80" + - "${HTTPS_PORT:-443}:443" + environment: + # Consumed by caddy/Caddyfile so a candidate tree can be validated with the + # exact tracked Caddyfile before it is installed. + CADDY_SITES_DIR: ${CADDY_SITES_DIR:-/etc/caddy/sites} + CADDY_CUSTOM_DIR: ${CADDY_CUSTOM_DIR:-/etc/caddy/custom} + CADDY_GLOBAL_DIR: ${CADDY_GLOBAL_DIR:-/etc/caddy/global} + volumes: + - ./caddy:/etc/caddy + - caddy_data:/data + - caddy_config:/config + depends_on: + ghost: + condition: service_started networks: - ghost_network traffic-analytics: image: ghost/traffic-analytics:1.0.368@sha256:f44abf4379f53d349c91a9a6298c252d0d390d0f24f94087bcdb2b627b4c1d4f - restart: always + restart: ${RESTART_POLICY:-unless-stopped} + profiles: [analytics] + labels: + <<: *site-labels + org.ghost.docker.role: traffic-analytics + org.ghost.docker.lifecycle: long-running + logging: *default-logging expose: - "3000" volumes: - traffic_analytics_data:/data environment: - NODE_ENV: production + NODE_ENV: ${NODE_ENV:-production} PROXY_TARGET: ${TINYBIRD_API_URL:-https://api.tinybird.co}/v0/events SALT_STORE_TYPE: ${SALT_STORE_TYPE:-file} SALT_STORE_FILE_PATH: /data/salts.json TINYBIRD_TRACKER_TOKEN: ${TINYBIRD_TRACKER_TOKEN:-} - LOG_LEVEL: debug - profiles: [analytics] + LOG_LEVEL: ${TRAFFIC_ANALYTICS_LOG_LEVEL:-info} networks: - - ghost_network + ghost_network: + aliases: + - traffic-analytics-${COMPOSE_PROJECT_NAME:-ghost} activitypub: image: ghcr.io/tryghost/activitypub:1.2.9@sha256:f950017169c778f90bc1d4097c0c83735dd88ee26a7dfacda56fb6325d4053a0 - restart: always + restart: ${RESTART_POLICY:-unless-stopped} + profiles: [activitypub] + labels: + <<: *site-labels + org.ghost.docker.role: activitypub + org.ghost.docker.lifecycle: long-running + logging: *default-logging expose: - "8080" volumes: - ${UPLOAD_LOCATION:-./data/ghost}:/opt/activitypub/content environment: # See https://github.com/TryGhost/ActivityPub/blob/main/docs/env-vars.md - NODE_ENV: production - MYSQL_HOST: db + NODE_ENV: ${NODE_ENV:-production} + MYSQL_HOST: ${DATABASE_HOST:-db} + MYSQL_PORT: ${DATABASE_PORT:-3306} MYSQL_USER: ${DATABASE_USER:-ghost} - MYSQL_PASSWORD: ${DATABASE_PASSWORD:?DATABASE_PASSWORD environment variable is required} - MYSQL_DATABASE: activitypub + MYSQL_PASSWORD: ${DATABASE_PASSWORD:?DATABASE_PASSWORD is required} + MYSQL_DATABASE: ${ACTIVITYPUB_DATABASE_NAME:-activitypub} LOCAL_STORAGE_PATH: /opt/activitypub/content/images/activitypub - LOCAL_STORAGE_HOSTING_URL: https://${DOMAIN}/content/images/activitypub + LOCAL_STORAGE_HOSTING_URL: ${URL}/content/images/activitypub depends_on: db: condition: service_healthy activitypub-migrate: condition: service_completed_successfully + networks: + ghost_network: + aliases: + - activitypub-${COMPOSE_PROJECT_NAME:-ghost} + + # Supporting one-shot jobs. + # + # These are not long-running services: they keep `restart: "no"` so a completed + # or failed job stays stopped instead of being restarted by the daemon. + + activitypub-migrate: + image: ghcr.io/tryghost/activitypub-migrations:1.2.9@sha256:f8a376e83187cc927fd6286a9e825b71056b2ac7dd8afcd1af1c8c7dc4657a8e + restart: "no" profiles: [activitypub] + labels: + <<: *site-labels + org.ghost.docker.role: activitypub-migrate + org.ghost.docker.lifecycle: one-shot + logging: *default-logging + environment: + MYSQL_DB: mysql://${DATABASE_USER:-ghost}:${DATABASE_PASSWORD:?DATABASE_PASSWORD is required}@tcp(${DATABASE_HOST:-db}:${DATABASE_PORT:-3306})/${ACTIVITYPUB_DATABASE_NAME:-activitypub} + depends_on: + db: + condition: service_healthy networks: - ghost_network - # Supporting Services - tinybird-login: build: context: ./tinybird dockerfile: Dockerfile + restart: "no" + profiles: [analytics] + labels: + <<: *site-labels + org.ghost.docker.role: tinybird-login + org.ghost.docker.lifecycle: one-shot + logging: *default-logging working_dir: /home/tinybird command: /usr/local/bin/tinybird-login volumes: - tinybird_home:/home/tinybird - tinybird_files:/data/tinybird - profiles: [analytics] networks: - ghost_network tty: false - restart: no tinybird-sync: # Do not alter this without updating the Ghost container as well - image: ghost:${GHOST_VERSION:-6-alpine} + image: ${GHOST_IMAGE:-ghost}:${GHOST_VERSION:-6-next-alpine} + restart: "no" + profiles: [analytics] + labels: + <<: *site-labels + org.ghost.docker.role: tinybird-sync + org.ghost.docker.lifecycle: one-shot + logging: *default-logging command: > sh -c " - if [ -d /var/lib/ghost/current/core/server/data/tinybird ]; then + if [ -d ${GHOST_TINYBIRD_PATH:-/home/ghost/core/server/data/tinybird} ]; then rm -rf /data/tinybird/*; - cp -rf /var/lib/ghost/current/core/server/data/tinybird/* /data/tinybird/; + cp -rf ${GHOST_TINYBIRD_PATH:-/home/ghost/core/server/data/tinybird}/* /data/tinybird/; echo 'Tinybird files synced into shared volume.'; else echo 'Tinybird source directory not found.'; + exit 1; fi " volumes: @@ -161,13 +295,18 @@ services: condition: service_completed_successfully networks: - ghost_network - profiles: [analytics] - restart: no tinybird-deploy: build: context: ./tinybird dockerfile: Dockerfile + restart: "no" + profiles: [analytics] + labels: + <<: *site-labels + org.ghost.docker.role: tinybird-deploy + org.ghost.docker.lifecycle: one-shot + logging: *default-logging working_dir: /data/tinybird command: > sh -c " @@ -179,23 +318,10 @@ services: depends_on: tinybird-sync: condition: service_completed_successfully - profiles: [analytics] networks: - ghost_network tty: true - activitypub-migrate: - image: ghcr.io/tryghost/activitypub-migrations:1.2.9@sha256:f8a376e83187cc927fd6286a9e825b71056b2ac7dd8afcd1af1c8c7dc4657a8e - environment: - MYSQL_DB: mysql://${DATABASE_USER:-ghost}:${DATABASE_PASSWORD:?DATABASE_PASSWORD environment variable is required}@tcp(db:3306)/activitypub - networks: - - ghost_network - depends_on: - db: - condition: service_healthy - profiles: [activitypub] - restart: no - volumes: caddy_data: caddy_config: diff --git a/docs/bundle-v1.md b/docs/bundle-v1.md new file mode 100644 index 00000000..acdf1b8e --- /dev/null +++ b/docs/bundle-v1.md @@ -0,0 +1,131 @@ +# Migration bundle v1 — encoding contract + +Ghost-CLI exports a migration bundle; ghost-docker imports it. This document +fixes the parts of the format that the importer depends on. It is the +authority for the encoding and required metadata; the exporter +(Ghost-CLI PR #2333 and its `docs/migration-bundle.md`) and its fixtures are +updated to match. + +**Bundle v1 is unpublished.** There is no draft-format compatibility path: a +bundle that does not meet this contract is rejected with an actionable error, +not silently adapted. Exporter, importer, documentation and fixtures change +together, and only then is v1 frozen. + +Steps referenced below are defined in +[the implementation plan](ghost-cli-replacement.md); each lands as its own +pull request. + +Status of the work: + +- This document and the importer-side contract: **S1** (this step). +- Exporter implementation, fixtures and cutover support: **S3**. +- Importer implementation and the cutover workflow: **S5**. + +The importer's target is `ghost.env`. Replacing it with a mounted Ghost JSON +config file was evaluated and rejected; see §2.1 of +[the plan](ghost-cli-replacement.md). The serialization rules below therefore +stand as written. +- Migration of existing Ghost-CLI installations: **S6**. + +## Manifest + +```json +{ + "bundleVersion": 1, + "bundleCreatedAt": "2026-09-02T12:00:00Z", + "sourceInstallType": "production", + "kind": "mysql-dump", + "ghost": { + "version": "5.130.3" + }, + "config": { + "url": "https://example.com", + "mail__options__auth__pass": "p$ssword", + "mail__from": "'Acme Support' " + } +} +``` + +### Required metadata + +| Field | Requirement | +| --- | --- | +| `bundleVersion` | Must be `1`. | +| `bundleCreatedAt` | **Required.** RFC 3339 timestamp in UTC. | +| `sourceInstallType` | **Required.** Exactly `local` or `production`. The importer infers the installation mode from this field. | +| `kind` | Bundle kind. Validated against the kinds the importer supports. | +| `ghost.version` | Exact source Ghost version. Validated as supported; the import happens *at* this version, and upgrading is a separate operation. | + +A bundle missing `bundleCreatedAt` or `sourceInstallType`, or carrying a +`sourceInstallType` outside that set, is rejected. There is no inference +fallback and no default. + +Every path in the manifest is validated. Path traversal, absolute member +paths, and symlinks or hardlinks that escape the bundle are rejected, +including in directory bundles. + +### `config` + +`config` is a **flat map of Ghost configuration keys to raw string values**. + +- Keys are the flattened `section__subsection__key` form. +- Values are the **raw strings**, exactly as Ghost would receive them. They + carry **no dotenv quoting and no dotenv escaping** of any kind: no + surrounding quotes added by the exporter, no `$$` for a dollar sign, no + backslash escapes. +- Serializing those values safely for Docker Compose is the **importer's** + job. The importer writes them into `ghost.env` using the encoding described + in [configuration.md](configuration.md) and at the top of + [scripts/lib/env.sh](../scripts/lib/env.sh). + +So a mail password of `p$ssword` appears in the manifest as the four-character +JSON string `"p$ssword"`, and reaches `ghost.env` as `mail__options__auth__pass="p$$ssword"`. +An exporter that pre-quotes or pre-escapes a value produces a corrupted +import, and the round trip is tested through real Docker Compose containers +rather than by comparing exporter strings. + +### Keys the importer omits + +The container owns these keys, so the importer drops them from `config` +rather than writing them into `ghost.env`, where they would be silently +overridden: + +- `url`, `admin__url` +- `database__*` +- `server__*` +- `paths__*` +- `process`, `logging__transports`, `logging__path` +- upgrade-adapter controls + +Public and admin URLs are mapped deliberately into `.env` (`URL`, `DOMAIN`, +`ADMIN_DOMAIN`, `ADMIN_URL`), preserving supported path and port semantics or +rejecting an unsupported URL with a clear message. Operator overrides of +URL and mode given at import time are retained. + +The authoritative list is `GD_CONTAINER_OWNED_KEYS` and +`GD_CONTAINER_OWNED_PREFIXES` in [scripts/lib/config.sh](../scripts/lib/config.sh); +`scripts/config.sh validate` enforces it. + +## Reading the manifest + +The manifest is read and validated using a pinned helper container, with no +Compose dependencies, no published ports, and read-only access to the bundle. +It cannot depend on an already-valid site `.env` or on a running Ghost, +because neither exists yet at that point in the import. + +The container is not about JSON parsing convenience — `jq` is available on the +host and is used freely for the site's own `.ghost-docker.json` and operation +journals. It is about isolating untrusted bundle content: path traversal, +absolute member paths, escaping links, and unbounded expansion are all +properties of a file someone else produced. + +## Not settled in this step + +The following are defined by their own steps and are **not** promised here: + +- Exporter behaviour, including the documented final-export mode that leaves + the source stopped for cutover, and the preserved restart behaviour for + ordinary exports (S3). +- The portable-import fidelity matrix and the explicit list of losses, which + is established from fixtures and real exporter/importer behaviour (S3/S5). +- The import sequence, isolated destination, verification and cutover (S5). diff --git a/docs/caddy.md b/docs/caddy.md new file mode 100644 index 00000000..e7c62e5b --- /dev/null +++ b/docs/caddy.md @@ -0,0 +1,89 @@ +# Caddy routing + +## Layout + +| Path | Tracked | Owner | +| --- | --- | --- | +| `caddy/Caddyfile` | yes | generic entry point, do not edit | +| `caddy/snippets/*` | yes | reusable route fragments, imported with arguments | +| `caddy/sites/*.caddy` | no | **generated** by `scripts/caddy.sh` | +| `caddy/custom/*.caddy` | no | operator owned, never generated or overwritten | +| `caddy/global/*.caddy` | no | operator owned global options, never overwritten | +| `caddy/.staging/` | no | candidate tree, validated before installation | + +The tracked `Caddyfile` imports the three directories through Caddy +placeholders, so a candidate configuration can be validated with the exact +file that will be installed: + +```caddyfile +{ + import {$CADDY_GLOBAL_DIR:/etc/caddy/global}/*.caddy +} + +import {$CADDY_SITES_DIR:/etc/caddy/sites}/*.caddy +import {$CADDY_CUSTOM_DIR:/etc/caddy/custom}/*.caddy +``` + +## Applying routes + +```bash +scripts/caddy.sh render # render the candidate into caddy/.staging +scripts/caddy.sh apply # render, validate, install, reload, verify +scripts/caddy.sh validate # validate the installed configuration +scripts/caddy.sh reload # explicit reload of the running Caddy +``` + +`apply` is transactional: + +1. Render the candidate into `caddy/.staging/sites/`. +2. Validate the tracked `Caddyfile` against the staged sites *and* the + operator's `custom/` and `global/` files. +3. Install the candidate atomically into `caddy/sites/`, keeping a backup. +4. Validate the installed configuration. +5. Reload the running Caddy explicitly. +6. Verify that Caddy's *loaded* configuration actually routes the expected + hostnames — a zero exit status from `reload` is not verification. + +If validation, reload or verification fails, the previous on-disk +configuration is restored and reloaded, and the command fails. + +Production uses an explicit reload. Caddy documents `--watch` as a local +development feature, so it is not used. + +## Import arguments + +Snippets take their upstreams and domains as import arguments: + +```caddyfile +import /etc/caddy/snippets/TrafficAnalytics traffic-analytics-my-site:3000 +import /etc/caddy/snippets/ActivityPub activitypub-my-site:8080 +import /etc/caddy/snippets/SecurityHeaders "admin.example.com" +``` + +A missing argument is only a *warning* during Caddy's adaptation — the +resulting server starts and misbehaves at runtime. `scripts/caddy.sh` promotes +that warning to an error, and the renderer refuses to install a file with an +unresolved template placeholder. + +## Custom routes + +Put your own server blocks in `caddy/custom/*.caddy`. They are imported after +the generated routes, validated together with them, and never rewritten. Ghost +is reachable on the Compose network as `ghost-$COMPOSE_PROJECT_NAME:2368`. + +Global options — an ACME account email, a DNS provider, or Caddy's internal CA +for a staging host — go in `caddy/global/*.caddy`: + +```caddyfile +email ops@example.com +``` + +## Optional services + +Analytics routes are rendered only when the `analytics` profile is enabled. + +ActivityPub routes are always rendered. With the `activitypub` profile enabled +they point at this site's own service; otherwise they point at +`ACTIVITYPUB_TARGET` (the hosted service by default). ActivityPub, its +migration job, database grants, storage and serving URL all belong to the +site. diff --git a/docs/configuration.md b/docs/configuration.md new file mode 100644 index 00000000..5f9270b9 --- /dev/null +++ b/docs/configuration.md @@ -0,0 +1,283 @@ +# Configuration + +## Two files, two audiences + +| File | Contents | Read by | +| --- | --- | --- | +| `.env` | Compose and operator settings: project identity, site mode, ports, data locations, restart policy, and infrastructure credentials such as `DATABASE_ROOT_PASSWORD`. | Docker Compose, for `${...}` interpolation | +| `ghost.env` | Ghost application settings only, in Ghost's `section__subsection__key` form. | The `ghost` container, as its only `env_file` | + +`.env` is **never** passed into the Ghost container. It holds the MySQL root +password and other infrastructure controls that Ghost has no reason to see. + +Keys the container owns — `url`, `admin__url`, `NODE_ENV`, `server__*`, +`paths__*` and `database__*` — are set as explicit Compose `environment` +entries. Compose `environment` overrides `env_file`, so setting them in +`ghost.env` has no effect; `scripts/config.sh validate` rejects them rather +than letting them look effective. + +That check is **derived, not listed**. Validation asks Compose what the +container actually receives (`docker compose config`, which is pure parsing and +needs no daemon) and reports any `ghost.env` key whose effective value differs. +An entry added to `compose.yml` is therefore caught the moment it is added, +with nothing to keep in sync. The same applies to operator settings in the +wrong file: the set of keys that belong in `.env` is derived from the variables +`compose.yml` interpolates, plus `COMPOSE_*`, plus whatever `.env` already +defines. + +Both files hold credentials and should be mode `0600`. The helpers write them +atomically with a restrictive umask and preserve the mode of an existing file. + +## Value encoding + +Compose interpolates dotenv values, **including inside double quotes**, and +`env_file` values are no exception. A literal dollar sign must be written `$$`. +There is no quoting a person naturally reaches for that avoids this, and the +only symptom is a wrong value at runtime: + +```dotenv +mail__options__auth__pass=s3cr$t! # Ghost receives: s3cr! +mail__options__auth__pass=Pa$$w0rd! # Ghost receives: Pa$w0rd! +``` + +Let the helper encode it instead of guessing: + +```bash +scripts/config.sh set ghost.env mail__options__auth__pass 'Pa$$w0rd!' +``` + +`scripts/config.sh validate` reports values Compose would interpolate by +accident. It cannot catch every case: a hand-written `$$` is indistinguishable +from a correctly escaped single `$`, so writing values through `config.sh set` +is the only way to be sure. + +### A limit worth knowing + +Nothing above makes a hand-written `Pa$$w0rd!` safe: `$$` is indistinguishable +from a correctly escaped single `$`, so no linter can catch it. **Do not +hand-edit a value containing `$`** — use `scripts/config.sh set`, which encodes +it correctly. + +Mounting Ghost's own JSON config file instead of `ghost.env` would remove the +interpolation layer entirely. It was evaluated and rejected; §2.1 of +[the plan](ghost-cli-replacement.md) records the findings and the reasons. + +### The rules + +These are what Docker Compose (compose-go/dotenv) actually implements, verified +by round-tripping real containers in `tests/env-compose.test.mjs`: + +| form | interpolated | escapes | +| --- | --- | --- | +| `KEY="value"` | yes — write `$$` for a literal `$` | `\\` → `\`, `\"` → `"`, `\$` → `$`, `\n` → LF, `\r` → CR, `\t` → TAB | +| `KEY='value'` | no | `\'` → `'` only, and a backslash immediately before a quote is not representable | +| `KEY=value` | yes | trailing whitespace trimmed, ` #` starts a comment | + +Double quotes are therefore the only form that can represent every value, and +are what the helpers write. JSON string escaping is identical to the +double-quoted dotenv escaping apart from `$`, so `jq` does the character-level +decoding, and bash substitution does the encoding. + +One limitation: a value whose quotes span several lines is valid dotenv but is +not editable through these helpers. Such a key is skipped when listing keys and +when linting, and reading or writing it fails with a message telling you to +edit it by hand. Nothing these helpers write ever produces one — newlines are +encoded as `\n`. + +Values are read back through `scripts/lib/env.sh`, which parses the file as +data and never sources or evaluates it. Helpers log key names only, never +values — there is no list of "sensitive" keys to keep in sync, because any +value in either file may be a credential. + +## Site modes and profiles + +Site mode is selected through `COMPOSE_PROFILES`, and exactly one of `local` +or `production` must be present: + +| Profile | Services | Ingress | +| --- | --- | --- | +| `local` | `ghost`, `db` | Ghost on `127.0.0.1:${GHOST_PORT}` | +| `production` | `ghost`, `db`, `caddy` | Caddy on `${HTTP_PORT}` / `${HTTPS_PORT}` | + +Optional per-site profiles are added to the same list and are purely additive. +Adding one never changes the site mode: + +| Profile | Services | Cost | +| --- | --- | --- | +| `analytics` | `traffic-analytics` plus the Tinybird one-shot jobs | one long-running Node service (~100-200 MB RSS) and a Tinybird workspace | +| `activitypub` | `activitypub`, `activitypub-migrate` | one long-running Node service (~150-250 MB RSS), one migration job per start, one extra MySQL database | + +`supervisor` is reserved for the upgrade supervisor and currently defines no +service. Any other profile name is rejected by validation. + +ActivityPub and analytics are **per-site**: each site owns its ActivityPub +database, storage and serving URL, and its own Tinybird credentials, workspace +selection and schema deployment. + +Ghost is always published on the loopback interface only, in both modes, so it +is never exposed publicly except through Caddy. + +Caddy is part of `production` rather than an optional profile, deliberately: a +supported bring-your-own-proxy path would mean validating the operator's +forwarded-header configuration, and getting `X-Forwarded-Proto` wrong yields +incorrect absolute URLs and non-secure cookies — a site that half works. If you +already run nginx or Apache, you can still point it at +`127.0.0.1:${GHOST_PORT}` and edit `compose.yml` to drop the caddy service or +move it off 80/443, but that is an unsupported manual customization and stack +updates may touch `compose.yml`. + +## Lifecycle + +Long-running services (`ghost`, `db`, `caddy`, `traffic-analytics`, +`activitypub`) use `restart: ${RESTART_POLICY:-unless-stopped}`. + +One-shot jobs (`activitypub-migrate`, `tinybird-login`, `tinybird-sync`, +`tinybird-deploy`) keep `restart: "no"`. A completed or failed job stays +stopped; the restart policy is not migration orchestration and is not +readiness. + +`ghost` and `db` have real health checks: Ghost's probe requires the Admin API +to answer, and MySQL's probe requires a real client connection to the +application database. A running container or a redirect is not readiness. + +Container logs are capped (`LOG_MAX_SIZE`, `LOG_MAX_FILE`) so a long-running +site cannot fill the disk. + +## Service names on the network + +Each service has a unique alias suffixed with the project name: + +- `ghost-${COMPOSE_PROJECT_NAME}:2368` +- `db-${COMPOSE_PROJECT_NAME}:3306` +- `traffic-analytics-${COMPOSE_PROJECT_NAME}:3000` +- `activitypub-${COMPOSE_PROJECT_NAME}:8080` + +Generated proxy routes and helper clients use these, never the bare service +name. `COMPOSE_PROJECT_NAME` is the site's stable identity and is kept +independent of the directory name. + +## Database connection + +`DATABASE_HOST`, `DATABASE_PORT`, `DATABASE_NAME` and `DATABASE_USER` are +parameterized even though the single-site defaults (`db`, `3306`, `ghost`, +`ghost`) do not change. Backup, restore and import all use the same connection +contract. + +## Installation metadata + +`.ghost-docker.json` records the schema version, installation time, mode, +release channel, installed stack version/commit, project identity, resolved +Ghost image and digest, and completed migrations. Its schema is specified in +§2.2 of [the plan](ghost-cli-replacement.md). It is gitignored and machine +generated — do not hand-edit it. Durable operation journals for in-progress +work and recovery state are separate files. + +An installation that predates this file is supported explicitly: readers must +treat a missing file as "unknown, pre-metadata install", not as a broken site. + +**Not implemented yet.** `install.sh` is the first thing that writes this file, +so the reader and writer land with it in S2 rather than sitting unused here. + +## The Compose invocation contract + +Every helper uses one contract: + +```bash +docker compose --project-directory "$DIR" -f "$DIR/compose.yml" ... +``` + +`--project-directory`, never `-C`. `COMPOSE_FILE` is unset by the helpers, +because it changes override auto-loading, and an explicit `-f` replaces the +selected list. Opt into an override file with `GD_COMPOSE_OVERRIDES`: + +```bash +GD_COMPOSE_OVERRIDES=compose.ipv6.yml scripts/caddy.sh validate +``` + +## Ghost image layout + +The default `GHOST_VERSION` is a `next` variant, which installs Ghost directly +under `/home/ghost`. The older variants use `/var/lib/ghost/versions/` with a +`current` symlink. Two variables carry the difference so that pinning an older +image still works: + +| Variable | Default (`next`) | Older layout | +| --- | --- | --- | +| `GHOST_CONTENT_PATH` | `/home/ghost/content` | `/var/lib/ghost/content` | +| `GHOST_TINYBIRD_PATH` | `/home/ghost/core/server/data/tinybird` | `/var/lib/ghost/current/core/server/data/tinybird` | + +Set both if you pin a `GHOST_VERSION` from the older layout; the content mount +and the Tinybird sync job read them. + +`scripts/config.sh validate` checks that `GHOST_CONTENT_PATH` matches the image +by reading the image's own `GHOST_CONTENT` variable, rather than inferring it +from the tag name — so a future layout change is caught without updating a +mapping. The check is skipped when the image has not been pulled yet. + +## Prerequisites + +Runtime, on the server: + +- `bash` — the helper scripts and `scripts/migrate.sh` are bash. Kept + compatible with bash 3.2 so macOS's system bash works for local development. +- Docker Engine 25.0.0 — for `healthcheck.start_interval` +- Docker Compose v2.24.0 — for `env_file` `required` and `depends_on` `required` +- `jq` — used by the helpers for JSON, including `.ghost-docker.json` (S2) + +`install.sh` verifies all three during preflight (S2). `scripts/migrate.sh` +already required `jq`, so this is not a new prerequisite for existing servers. + +Development only, not needed on a server: + +- Node.js 20+, to run the test suite with its built-in runner + (`node --test tests/*.test.mjs`). There is no `package.json` and nothing to + install; the repository is not a Node package. + +The one exception is the legacy `scripts/migrate.sh`, which shells out to +`scripts/config-to-env.js` and so needs Node on the host. `install.sh --import` +replaces that script outright, and the dependency goes with it. + +Both the declared minimum and a current Compose are exercised by +`tests/compose-matrix.test.mjs`. + +## Existing installations + +This layout is a breaking change for checkouts made before it landed. The +migration is owned by the stack updater (S6); the changes it has to handle are: + +- `caddy/Caddyfile` is now **tracked**. An existing installation has an + untracked file at that exact path, and Git refuses to overwrite an untracked + file with a tracked one. The updater moves the operator's file aside — into + `caddy/custom/` where its routes keep working — *before* the checkout. +- The snippets in `caddy/snippets/` now take their upstreams and domains as + import arguments instead of reading `{$DOMAIN}` / `{$ACTIVITYPUB_TARGET}` + from the Caddy container's environment, which the `caddy` service no longer + sets. A hand-written Caddyfile that imports them needs the arguments added. +- Ghost application configuration moves from `.env` to `ghost.env`. `.env` + keeps the Compose and operator settings and is no longer passed into the + Ghost container. +- `COMPOSE_PROFILES` must gain a site mode (`production` for an existing + server), and `SITE_MODE`, `URL`, `PROJECT_DIR` and an exact `GHOST_VERSION` + pin must be added. + +`scripts/migrate.sh` still migrates a Ghost-CLI installation and has been +updated to write Ghost configuration into `ghost.env`. It is kept until the +bundle import in [docs/bundle-v1.md](bundle-v1.md) has passed fidelity and +recovery testing. + +## Why `jq` is a prerequisite + +The helpers use `jq` for JSON, starting with `.ghost-docker.json` in S2. + +The implementation plan originally recorded "No host Node or jq requirement". +That was changed during S1, deliberately, and §1 of +[the plan](ghost-cli-replacement.md) now records the decision: + +- `scripts/migrate.sh` already declared `jq` in its `required_commands` + preflight, so servers running the supported migration path already have it. +- Hand-rolling a JSON writer and reader in shell to avoid it cost about + 240 lines and bought nothing an operator can see. + +Node.js is **not** a runtime requirement. It is used only to run the test +suite. `install.sh` (S2) must verify `docker`, `docker compose` and `jq` during +preflight, and must not require Node. diff --git a/docs/ghost-cli-replacement.md b/docs/ghost-cli-replacement.md new file mode 100644 index 00000000..5c61142b --- /dev/null +++ b/docs/ghost-cli-replacement.md @@ -0,0 +1,1599 @@ +# Plan: ghost-docker as a Ghost-CLI replacement + +Status: revised 2026-09-02 after code and Compose review. This is an implementation +plan, not authorization to execute its steps. Single-site installations are the +initial release target. Shared infrastructure, image distribution/channel extensions, +and default Redis follow in the final phases. + +This document lives in the repository so that every branch in the stack carries +the contracts it is implementing against. Amend it in the PR that changes a +decision, rather than letting the code and the plan drift apart. + +Repos involved: + +- `TryGhost/ghost-docker`: installation, configuration, import, upgrades, operations. +- `TryGhost/Ghost-CLI`: migration bundle export, PR #2333. +- `TryGhost/Ghost`: upgrade adapter, Admin API, and Admin UI. + +Each step below is a separate work package. Read its dependencies and the contracts +in this document before implementation. Do not treat step prompts as independent +of those contracts. Update operator documentation and tests with each step. + +## 1. Scope and decisions + +| Topic | Decision | +| --- | --- | +| Initial audience | Local and single-site production installations; most servers run one site. | +| Site model | One checkout = one site. One base `compose.yml`, with `local` and `production` modes selected through `COMPOSE_PROFILES`. | +| Shared infrastructure | Deferred to S13, after the single-site replacement is complete. It is not a dependency of the initial release. | +| ActivityPub and analytics | Per-site initially, including for future members of shared infrastructure. Each site owns its ActivityPub database/storage and Tinybird configuration/deployment lifecycle. No shared ActivityPub or analytics service in S13. | +| Versions | Resolve and persist an exact Ghost image version on installation. Ghost upgrades and stack/repository updates are separate operations. Record resolved image digests for recovery. | +| Distribution | Clone at a release tag. Stable tags `vX.Y.Z`; beta tags `vX.Y.Z-beta.N`. A bootstrap shim selects the release and delegates to that checkout. | +| Installation | Scriptable `install.sh`, with flags for every required prompt. Local mode uses MySQL too. Requires `bash`, `docker`, `docker compose` and `jq` on the host, verified in preflight; `scripts/migrate.sh` already required both `bash` and `jq`. Helper scripts stay bash 3.2 compatible so macOS's system bash works. No host Node requirement for a new install; Node is otherwise only used to run the test suite. The one exception is the legacy `scripts/migrate.sh`, which shells out to `scripts/config-to-env.js` and therefore still needs Node until `install.sh --import` replaces it. | +| Migration | Ghost-CLI exports a bundle; Docker imports it. Fix the encoding contract before declaring v1 frozen. `install.sh --import` replaces `scripts/migrate.sh` outright rather than living alongside it; retire the old scripts, and with them the last host Node dependency, only after replacement fidelity and recovery tests pass. | +| Upgrades | Optional supervisor using a file exchange and Docker socket. Ship a tested host-driven upgrade operation first, then reuse its recovery contract in the supervisor. | +| UX | Standard Compose commands for daily operation; scripts for installation, doctor/list, migration, backup/restore, and upgrades. No wrapper binary named `ghost`. | +| Where tooling runs | A thin host shell layer (bootstrap, preflight/doctor, dispatch) plus a pinned manager image that holds the stateful operations. See §2.10. Host requirements stay `bash`, `docker`, `docker compose`, `jq`; no host language runtime. | +| Configuration | `.env` contains Compose/operator settings; `ghost.env` contains only Ghost application settings. Do not pass the whole `.env` into Ghost. A mounted Ghost JSON config file was evaluated as a replacement for `ghost.env` and rejected; see §2.1. | +| Ghost nightly channel | Future explicit opt-in via `--ghost-channel nightly`; published to GHCR, independently of the stack release channel. Stable remains the default. | +| Service image registry | Future `--image-registry dockerhub|ghcr` selects dual-published traffic-analytics and ActivityPub images, including migrations. Preserve existing selections when updating. | +| Redis | Default on new installations once S16 ships, with explicit `--without redis` opt-out. Existing sites adopt through a documented migration. Per-site caching first; future durable uses require their own state policy. | +| Multi-site prototype | Retire the old 1800-line single-project generator on `feat/multisite`. Salvage tested helpers and DB provisioning logic only after review. | + +Explicitly document initial limitations: no shared-infra provisioning, no automatic +major Ghost/MySQL upgrades, no arbitrary downgrade support, and no claim of lossless +SQLite-to-MySQL migration through the portable API export. + +## 2. Architecture and contracts + +### 2.1 Compose modes and configuration + +Initial service profiles: + +| Service | Profiles | Lifecycle | +| --- | --- | --- | +| `ghost` | `local`, `production` | Long-running | +| `db` | `local`, `production` | Long-running | +| `caddy` | `production` | Long-running | +| `traffic-analytics` | `analytics` | Long-running, per-site | +| `activitypub` | `activitypub` | Long-running, per-site | +| `activitypub-migrate` | `activitypub` | One-shot | +| `tinybird-*` | `analytics` | Setup/deployment jobs, per-site | +| `upgrade-supervisor` | `supervisor` | Long-running, optional | + +Caddy is part of the `production` mode, not an optional profile. Making +bring-your-own-proxy a first-class path was considered and rejected: it would +mean owning validation of the operator's proxy configuration, and the failure it +guards against is subtle — a wrong `X-Forwarded-Proto` yields incorrect absolute +URLs and non-secure cookies, so the site half works rather than failing. That is +a poor thing to support on someone else's proxy. + +An operator who already runs nginx or Apache can still do it, as a manual +customization rather than a supported mode: Ghost publishes on +`127.0.0.1:${GHOST_PORT}` in every mode, so they point their proxy there and +edit `compose.yml` to drop the caddy service or move it off 80/443. Document +that this is unsupported and that stack updates may touch `compose.yml`. + +Profiles are additive, not mutually exclusive or conditional configuration. Validate +that exactly one site mode is selected. Optional services must not accidentally +activate unrelated modes. Explicitly targeted Compose services can run even when +their profiles are inactive; helper commands must account for dependencies. + +Example generated Compose settings (credentials omitted): + +```dotenv +# Local +COMPOSE_PROFILES=local +COMPOSE_PROJECT_NAME=ghost-local-example +PROJECT_DIR=/absolute/path/to/site +NODE_ENV=development +URL=http://localhost:2368 +GHOST_PORT=2368 +RESTART_POLICY=no +GHOST_VERSION=6.3.1-alpine +DATABASE_HOST=db +DATABASE_NAME=ghost +DATABASE_USER=ghost + +# Production uses the same variable contract with: +# COMPOSE_PROFILES=production +# NODE_ENV=production +# URL=https://example.com +# DOMAIN=example.com +# RESTART_POLICY=unless-stopped +# Optional: ADMIN_DOMAIN=admin.example.com +# Optional profiles are added only after their configuration is validated. +``` + +Requirements: + +- Ghost publishes `127.0.0.1:${GHOST_PORT:-2368}:2368`. The installer picks a free + port when none is supplied; an explicit occupied port is an error. +- Parameterize database host, name, and user now, even though single-site defaults + remain `db`/`ghost`/`ghost`. Use the same connection contract for backup and import. +- Set a unique Ghost network alias `ghost-${COMPOSE_PROJECT_NAME}` and use it in + generated proxy routes and helper clients. Never rely on `ghost` for shared-network + addressing when S13 is introduced. +- Persist the project name independently of the directory name. Moving a site still + requires updating and validating `PROJECT_DIR` and bind mounts. +- Use `restart: ${RESTART_POLICY:-unless-stopped}` only for long-running services. + Setup, migration, and deployment jobs retain `restart: "no"`. +- Initially `URL` may be required because every supported mode contains Ghost. Do + not put `:?` guards on optional-service variables such as `PROJECT_DIR` or DOMAIN. + Validate requirements by mode before provisioning or startup. Revisit URL's guard + before adding infra-only mode in S13. +- Keep the initial default network naming unchanged. Do not introduce an empty + `name:` as a guessed equivalent of an omitted field. +- `ghost.env` is the only application `env_file`, and is transitional. Explicit + Compose environment entries override container-owned keys; the importer + rejects/omits those keys. + +Application configuration stays in `ghost.env`. Replacing it with a mounted +Ghost JSON config file was evaluated, because dotenv cannot hold an arbitrary +value safely: Compose interpolates `env_file` values, so an SMTP password of +`Pa$$w0rd!` reaches Ghost as `Pa$w0rd!`, and `s3cr$t!` reaches it as `s3cr!`, +with no error anywhere. It was rejected — the findings are recorded here so the +question is not reopened from scratch. + +Verified against `ghost:6-alpine` (Ghost 6.61.0, nconf 0.13.0): + +- The image ships `config.production.json` in Ghost's install directory + (`/home/ghost` in the `next` variants, `/var/lib/ghost` in the older layout), + with `config.development.json` symlinked to it. It sets `url`, `server`, + `mail.transport: "Direct"`, `logging.transports`, `process`, `security` and + `paths.contentPath`. +- nconf is first-added-wins, and `loader.js` registers `custom-env` + (`config..json`) *before* `local-env-jsonc` (`config.local.jsonc`). So + `config.local.jsonc` cannot override anything the image ships, including + `mail.transport`, and is unusable as the operator's config file. +- A file mounted over `config..json` is read, and all value shapes survive: + strings, numbers, booleans, nested objects, storage adapters, labs flags. A + `$` in a value survives verbatim. +- Compose `environment` entries outrank every config file, so container-owned + keys stay enforced by construction either way. +- nconf coerces env var types too (`port=465` arrives as a number), so the env + form loses nothing on typing. + +Why it was rejected: + +- Comments are the main reason to prefer a config file over dotenv for + hand-editing, and they require JSONC. `jq` cannot parse JSONC at all, so + validation and every programmatic read would fail on a commented file, and + writes would strip the comments. The format would fight the tooling. +- Strict JSON keeps `jq` working but has no comments, and because the image + already ships `config.production.json`, our file would replace it rather than + layer over it — pinning a hand-maintained copy of the image's defaults that + silently drifts when the image changes them. Layering would need a Ghost + loader change registering a custom-env JSONC file *before* `custom-env`. +- Multi-line values are awkward in both directions: JSON requires `\n` escapes, + dotenv allows literal newlines that the helpers refuse to edit. + +The residual risk is accepted: a hand-written `$$` is indistinguishable from a +correctly escaped single `$`, so no linter can catch it. Mitigation is to write +values with `scripts/config.sh set`, which encodes correctly, and `env_lint`, +which catches the bare-`$` case. Document that hand-editing a value containing +`$` is unsafe. + +Caddy is part of the `production` mode, not an optional profile. Making +bring-your-own-proxy a first-class path was considered and rejected: it would +mean owning validation of the operator's proxy configuration, and the failure it +guards against is subtle — a wrong `X-Forwarded-Proto` yields incorrect absolute +URLs and non-secure cookies, so the site half works rather than failing. That is +a poor thing to support on someone else's proxy. + +An operator who already runs nginx or Apache can still do it, as a manual +customization rather than a supported mode: Ghost publishes on +`127.0.0.1:${GHOST_PORT}` in every mode, so they point their proxy there and +edit `compose.yml` to drop the caddy service or move it off 80/443. Document +that this is unsupported and that stack updates may touch `compose.yml`. + +Profiles are additive, not mutually exclusive or conditional configuration. Validate +that exactly one site mode is selected. Optional services must not accidentally +activate unrelated modes. Explicitly targeted Compose services can run even when +their profiles are inactive; helper commands must account for dependencies. + +Example generated Compose settings (credentials omitted): + +```dotenv +# Local +COMPOSE_PROFILES=local +COMPOSE_PROJECT_NAME=ghost-local-example +PROJECT_DIR=/absolute/path/to/site +NODE_ENV=development +URL=http://localhost:2368 +GHOST_PORT=2368 +RESTART_POLICY=no +GHOST_VERSION=6.3.1-alpine +DATABASE_HOST=db +DATABASE_NAME=ghost +DATABASE_USER=ghost + +# Production uses the same variable contract with: +# COMPOSE_PROFILES=production +# NODE_ENV=production +# URL=https://example.com +# DOMAIN=example.com +# RESTART_POLICY=unless-stopped +# Optional: ADMIN_DOMAIN=admin.example.com +# Optional profiles are added only after their configuration is validated. +``` + +Requirements: + +- Ghost publishes `127.0.0.1:${GHOST_PORT:-2368}:2368`. The installer picks a free + port when none is supplied; an explicit occupied port is an error. +- Parameterize database host, name, and user now, even though single-site defaults + remain `db`/`ghost`/`ghost`. Use the same connection contract for backup and import. +- Set a unique Ghost network alias `ghost-${COMPOSE_PROJECT_NAME}` and use it in + generated proxy routes and helper clients. Never rely on `ghost` for shared-network + addressing when S13 is introduced. +- Persist the project name independently of the directory name. Moving a site still + requires updating and validating `PROJECT_DIR` and bind mounts. +- Use `restart: ${RESTART_POLICY:-unless-stopped}` only for long-running services. + Setup, migration, and deployment jobs retain `restart: "no"`. +- Initially `URL` may be required because every supported mode contains Ghost. Do + not put `:?` guards on optional-service variables such as `PROJECT_DIR` or DOMAIN. + Validate requirements by mode before provisioning or startup. Revisit URL's guard + before adding infra-only mode in S13. +- Keep the initial default network naming unchanged. Do not introduce an empty + `name:` as a guessed equivalent of an omitted field. +- `ghost.env` is the only application `env_file`, and is transitional. Explicit + Compose environment entries override container-owned keys; the importer + rejects/omits those keys. + +Application configuration is moving from `ghost.env` to a mounted JSON config +file, because dotenv cannot hold an arbitrary value safely: Compose interpolates +`env_file` values, so an SMTP password of `Pa$$w0rd!` reaches Ghost as +`Pa$w0rd!` with no error anywhere. Verified against `ghost:6-alpine` +(Ghost 6.61.0, nconf 0.13.0): + +- The image ships `config.production.json` in Ghost's install directory + (`/home/ghost` in the `next` variants, `/var/lib/ghost` in the older layout), + with `config.development.json` symlinked to it. It sets `url`, `server`, + `mail.transport: "Direct"`, `logging.transports`, `process`, `security` and + `paths.contentPath`. +- nconf is first-added-wins. `loader.js` registers `custom-env` + (`config..json`) *before* `local-env-jsonc` (`config.local.jsonc`), so + `config.local.jsonc` cannot override anything the image ships — including + `mail.transport`. It is not usable as the operator's config file. +- Compose `environment` entries still outrank every config file, so + container-owned keys stay enforced by construction. +- `localUtils.jsoncFormat` already exists and wraps `jsonc-parser`. + +The Ghost change this depends on is to register a custom-env JSONC file +*before* `custom-env` in `core/shared/config/loader.js`: + +```js +nconf.file('custom-env-jsonc', { + file: path.join(customConfigPath, 'config.' + env + '.jsonc'), + format: localUtils.jsoncFormat, +}); +nconf.file('custom-env', path.join(customConfigPath, 'config.' + env + '.json')); +``` + +Ordering is the point. Registered first, the operator's file layers *over* the +image's shipped defaults instead of replacing them, so ghost-docker never has +to keep its own copy of those defaults in sync. Comments and trailing commas +come along for free. + +Two constraints follow: + +- The file is per-environment, so it mounts as `config.${NODE_ENV}.jsonc`; + local mode (`NODE_ENV=development`) needs `config.development.jsonc`. +- It sets a Ghost version floor. §2.4 imports at the *source* Ghost version, so + a site imported from a Ghost that predates this change would not read the + file at all. S5 must either detect that and fall back, or require a floor for + imported sources; it cannot assume the feature is present. +- Add site/mode labels and a real Ghost readiness probe. A running container or + redirect response alone does not establish readiness. +- Cap container logs and make optional-service resource costs visible. + +Before publishing the minimum Docker/Compose versions, run the mode matrix against +that exact minimum and a current version. The installed Compose v5.1.2 accepted +interpolated restart `no` and network `external=true`; that is not verification of +older versions. Include `start_interval` and any env-file features in compatibility +checks. Avoid dependencies on undeclared host utilities or GNU-only shell behavior. + +### 2.2 Environment values, metadata, and permissions + +`scripts/lib/env.sh` must never source or evaluate an env file. Define a serializer +and parser with round-trip tests through Docker Compose itself, including `$VAR`, +`${VAR}`, `$$`, spaces, quotes, backslashes, newlines, empty strings, and JSON arrays. +Do not assume double-quoted values are literal: Compose interpolates them. + +Keep arbitrary application configuration in `ghost.env`; root DB credentials, +project paths, network settings, and other operator controls stay out of Ghost's +environment. Write credential-bearing files privately, with restrictive umask and +atomic replacement preserving intended ownership/mode. Logs list sensitive key names, +never their values. Add file-based credentials later only for supported Ghost images. + +`.ghost-docker.json` is gitignored and contains a schema version, installation time, +mode, release channel, installed stack version/commit, project identity, and completed +migrations. Separate durable operation journals record in-progress work and recovery +state. An installation that predates metadata must be supported explicitly. + +All mutating operations on a site acquire the same host-visible operation lock: +install/reconfigure, import, restore, Ghost upgrade, and stack update. Define stale +lock recovery after a crashed process; never discard a lock solely due to elapsed +time. S13 adds an infra-wide registration lock. + +### 2.3 Caddy and optional services + +- Track a generic Caddyfile importing generated `sites/*.caddy` and operator-managed + `custom/*.caddy`. Ignore generated/operator files in Git. +- Render sites from a template with explicit upstream, public/admin domains, and + optional-service targets. Preserve every import argument; missing arguments may + survive adaptation and fail at runtime. +- Validate a candidate configuration, atomically install it, reload Caddy, and verify + routing. Restore the previous on-disk configuration if validation/reload fails. +- Use explicit reload in production, not `--watch`. Caddy documents watch as a local + development feature. Use `docker compose --project-directory "$DIR" ...`, not `-C`. +- Migrations must preserve custom routes. Do not silently replace a customized + Caddyfile with a generated approximation after printing a warning. +- ActivityPub, its migration job, database grants, storage, and serving URL belong + to the site. Test retrieval of uploaded ActivityPub assets through the site's URL. +- Tinybird credentials/workspace selection and schema deployment belong to the site. + Treat sync and deploy as distinct steps. A Ghost upgrade must run the required + deployment after sync, not merely copy new files into a volume. +- Specify schema compatibility and recovery for analytics before automated upgrades + with analytics are supported. Do not imply a Ghost DB restore undoes remote schema + changes. Unsupported combinations must fail preflight with an actionable message. + +### 2.4 Bundle format and migration + +Reference: Ghost-CLI PR #2333 and its `docs/migration-bundle.md`. Review the local +`claude/ghost-cli-migration-export-c00253` branch without assuming it is merged. + +Before freezing the contract: + +- Define `config` as a map of flattened keys to raw string values, without embedded + dotenv quoting. The importer serializes these values safely for Docker Compose. +- Require `bundleCreatedAt` and `sourceInstallType: local|production`. Infer installation + mode from `sourceInstallType`; validate kind, supported Ghost version, and all paths. +- Update exporter, importer, documentation, and fixtures together before freezing + bundle v1. The unpublished draft format does not need backward compatibility. +- Test actual Compose round trips rather than only comparing exporter strings. +- Define portable-import losses explicitly, including identity/authentication and + integration/subscription relationships as applicable. Establish the supported + fidelity matrix from fixtures and real exporter/importer behavior. +- Record the consistency/cutover behavior: the exporter currently restarts Ghost. + Add a documented final-export mode that leaves the source stopped, with explicit + operator selection, and preserve the current restart behavior for ordinary exports. + Portable exports require a running source; final export must arrange a write freeze + before taking API/content snapshots, then leave it stopped for cutover. + +Import sequence: + +1. Acquire the site lock. Validate target state and available space. Default to a + fresh target; refuse merging into an existing live database/content tree. +2. Inspect/extract into private staging. Reject path traversal, absolute member paths, + and escaping symlinks/hardlinks, including in directory bundles. Bound expansion + and check space for extracted content, database restore, and recovery copies; + compressed archive size times 1.5 is not a sufficient estimate. +3. Read/validate the manifest using a pinned helper container, with no Compose + dependencies, no public ports, and read-only access to the bundle. This cannot + depend on an already-valid site `.env` or already-running Ghost. +4. Resolve the exact source Ghost image and check architecture/availability before + changing the target. Import at the source version; upgrading is a separate step. +5. Generate `.env` and `ghost.env`; retain operator URL/mode overrides. Omit + container-owned config including `url`, `admin__url`, database, server, paths, + process, logging, and upgrade-adapter controls. Map public/admin URLs deliberately, + preserving supported path/port semantics or rejecting unsupported URLs clearly. +6. Stage content including hidden files and establish ownership appropriate to the + selected Docker mode. Do not blindly apply host uid 1000 under rootless/userns. +7. Restore the selected database only, with explicit connection and database name. + `mysql-dump` does not contain CREATE DATABASE/users/grants. Provision first, then + restore before Ghost is started; propagate pipeline failures. +8. For portable data, start an isolated destination with no public ingress, set up + the owner, authenticate, and perform multipart content/member imports. Explicitly + mount the helper script. Use the unique service alias, correct Host/Origin/proxy + semantics, and handle separate admin URLs, HTTPS, sessions, and supported auth. + Owner credentials may come from a prompt or private file, not only command args. +9. Verify the expected content/member records, active theme/assets, redirects, URLs, + and supported configuration. Preserve a journal so a retry cannot duplicate a + partially completed portable import. Prefer recreating the isolated target from + the bundle when resumability cannot be proved. +10. Switch ingress only after verification and an explicit final-source write freeze. + Explain DNS/proxy cutover for cross-host moves. Keep the old installation and + recovery instructions intact until the operator accepts the destination. + +`install.sh --import` must not start normal production ingress before this workflow. +For rehearsal/local imports, provide a safe documented way to suppress outbound +email, newsletters, payments/webhooks, and federation activity. Do not silently send +real production traffic from a copied database. + +### 2.5 Backup, upgrade, and recovery + +Implement backup and restore before promising automated rollback. Backups include +the Ghost database, content, site configuration, versions/digests, and a manifest. +Document optional-service state and which remote changes cannot be restored locally. +Use restrictive permissions, retention controls, space checks, and a restore drill. +Neither `docker compose down -v` nor a database-only export is a complete backup. + +Ghost upgrade contract: + +1. Acquire the operation lock; validate current state, target, compatibility, disk + space, and backup capability. Reject unsupported majors/downgrades. Resolve + `latest` to one exact supported same-major version and immutable image identity. +2. Pull/verify the target before downtime. Record the previous image digest and + configuration; retain the previous image for recovery. +3. Enable maintenance ingress and stop application writes/background writers. + Create and verify a consistent recovery checkpoint of all affected local state. + Production automated upgrades require this checkpoint; request input cannot waive it. +4. Persist the journal before mutation. Apply the exact image configuration, run + migrations/startup and the supported optional-service deployment sequence. +5. Verify database/application readiness and proxy routing while external writes + remain blocked. Resume traffic only after verification, then mark the job done. +6. If verification fails, restore the checkpoint and previous image/configuration + using the tested recovery procedure. Report `rolled-back` only after verifying + the restored system. If restore fails or affected external state cannot safely + be reconciled, retain maintenance mode and report `recovery-required`. + +Do not confuse switching images with reversing schema migrations. A later rollback +after traffic has resumed needs a separate deliberate recovery workflow because +restoring the pre-upgrade snapshot would discard newer writes. A generic request +for an older version must not bypass this rule. + +On restart, reconcile an interrupted operation from its journal and actual container/ +database state; never blindly replay destructive steps. Inject failures at pull, +backup, config write, migration, readiness, restore, and process interruption. + +### 2.6 Supervisor and Ghost integration + +Publish an optional supervisor image from this repo. The Docker socket is +host-privileged; non-root process execution does not remove that authority. The host +operator explicitly enables self-upgrades and controls policy. Ghost owner/admin +authorization permits requests only within that host-defined policy. + +The supervisor sees the checkout at the same absolute host path for Compose bind +resolution. Define supported local Docker contexts and socket paths; reject remote +daemons or unsupported rootless setups clearly. Avoid broad writable mounts beyond +what execution requires. Supervisor behavior follows §2.5 rather than inventing a +second upgrade/recovery algorithm. + +Write the protocol document before implementing either side. It must include: + +- Versioned JSON schemas for status, requests, and jobs; exact version/image fields; + timestamps/heartbeat; supported capabilities; bounded error details. +- Request states including queued, backing-up, pulling, restarting, verifying, done, + failed, restoring, rolled-back, and recovery-required, plus legal transitions. +- UUID validation, bounded file sizes, no symlink following, exclusive request + claiming, deduplication, restart recovery, retention, and polling backoff. +- Separate request-write and status/job-read permissions for Ghost. A shared writable + parent directory must not let Ghost replace supervisor-owned status or job files. + Define initialization/uid ownership and mount layout explicitly. +- Atomic publication and durable journaling around side effects. A POST can return + an accepted job ID before the supervisor claims it; distinguish pending, unknown, + expired, stale supervisor, and protocol mismatch rather than treating all 404s as + a restart indefinitely. +- Strict target allowlisting, argument-array subprocess invocation, no shell input, + host-enforced major/backup policy, and the shared site operation lock. + +Ghost adapter type: `upgrade`. Canonical implementation names: +`NoopUpgradeAdapter` and `FileDropUpgradeAdapter`; use these exact names in config, +code, docs, and tests. Enable FileDrop only for a compatible Ghost version and an +initialized exchange. Noop remains the default. Absence of a supervisor is a +supported state, not a Ghost startup failure. + +Admin API: status, create request, and fetch job. Owner/admin only, rate-limited POST, +with the normal Admin auth/permission conventions. Admin feature-detects both absent +endpoints on older cores and `supported: false`. Polling survives restart with a +bounded reconnect period and useful stalled/recovery-required states. UI backup +promises must match the actual enforced policy. + +Version discovery must handle registry pagination, rate limits, stale cache, semver +ordering, image architecture, and compatibility requirements. A valid Docker tag +alone is not evidence that an upgrade path is supported. + +### 2.7 Stack versions and repository updates + +Release-please manages releases. Configure dependency changes explicitly as +releasable patches; do not assume `chore(deps)` does that by default. Specify beta +release mechanics and test version selection rather than using lexical sorting. + +`scripts/update.sh [--check] [--channel stable|beta] [--to vX.Y.Z]` updates the stack, +not Ghost. Preserve the exact Ghost pin; if a stack release requires a newer Ghost, +stop with the required upgrade sequence. Initially reject stack downgrades unless +the relevant migrations explicitly support them. + +Transactional flow: + +1. Acquire the site lock; reject a dirty tracked tree and unresolved operations. +2. Fetch/resolve the release, check compatibility, show changes, and record the + previous commit SHA (the source may not have been installed from a tag). +3. Run a stable updater/helper outside files being replaced. Back up `.env`, + `ghost.env`, generated/custom Caddy files, metadata, and migration journal. +4. Obtain pre-checkout hooks from the target release. Journal their phases and run + them before checkout where necessary; then checkout and run post-checkout hooks. + Idempotency does not substitute for recovery. Record completion only after the + corresponding stage succeeds. +5. Validate Compose and Caddy, pull images, apply the release, and verify readiness. + Define safe handling of DB/ActivityPub schema-affecting stack changes using the + backup/recovery contract; do not blindly roll back a migrated service image. +6. On failure, restore the previous code and configuration where safe. If stateful + service changes already occurred, use their recovery procedure or report + recovery-required. Never report success merely because `up -d` returned zero. + +Migration `0001-compose-profiles` must handle both an absent profile setting and +existing `analytics,activitypub` values, adding `production` in either case. Split +application config into `ghost.env`, preserve credentials and project identity, add +URL/PROJECT_DIR and an exact pin for the currently running Ghost version. + +Move an existing untracked Caddyfile before checkout introduces the tracked file. +Preserve customizations automatically when supported; otherwise stop before changing +the live setup and present the required configuration resolution. Test custom routes, +legacy Compose overrides, skipped releases, repeat invocation, and failed hooks. + +Publish a bootstrap path for existing installations that do not yet have update.sh; +do not tell them to execute a script absent from their checkout or use raw git pull +to cross the breaking change. + +### 2.8 Installer and day-to-day operations + +```text +install.sh [--local | --domain example.com [--admin-domain admin.example.com]] + [--dir PATH] [--port 2368] [--version 6.3.1] + [--channel stable|beta] [--ref vX.Y.Z] + [--with analytics,activitypub,supervisor] + [--import BUNDLE] [--no-prompt] [--no-start] +``` + +The curl-able shim contains only bootstrap logic; installation logic lives at the +selected release. Pin the selected release before executing its helpers. Unknown or +not-yet-supported flags fail clearly. Use `/dev/tty` for interactive input; no-prompt +must not silently accept destructive choices. + +Preflight covers supported OS/architecture, Docker and Compose versions, selected +daemon access/context, required tools, writable directories, memory/disk, ports, +optional-service credentials, URL/DNS, and compose/caddy validation. Test daemon +access instead of requiring root/docker-group membership. Rootless support requires +verified socket, port, ownership, and boot behavior; do not infer support from linger +alone. Handle systems without systemd or explicitly narrow the supported platforms. + +Check daemon startup on Linux; document Docker Desktop/OrbStack startup for local +macOS. Bind-mounted paths must be available at daemon startup. Keep nginx/apache +running until cutover; a server may proxy other applications, so replacing its whole +service requires an explicit operator choice. Installation must not stop or +reconfigure an existing proxy. Bringing your own proxy is a documented manual +edit of `compose.yml`, not a supported mode; see §2.1. + +Installation writes configuration, renders routing, initializes permissions and +metadata, then verifies readiness before publishing the Admin URL. `--no-start` +must not start application services. Imported sites follow the isolated flow in §2.4. + +`site.sh list` includes stopped containers (`docker ps -a` with labels) and, where +possible, known installations with no current container. State what cannot be +discovered without a registry. `site.sh check`/doctor works for single-site installs +and validates actual DB connectivity, configuration, readiness, and recovery state. + +### 2.9 Final-phase image distribution, nightly builds, and Redis + +These extensions follow single-site qualification and do not block the initial +release. Their numbering is a delivery sequence, not a requirement to ship shared +infra before single-site registry selection or Redis. Each extends the acceptance +matrix and operator documentation when it ships. + +Future installer interface (add only as the relevant steps land): + +```text +--image-registry dockerhub|ghcr +--ghost-channel stable|nightly +--without redis +``` + +Keep `--channel stable|beta` for the ghost-docker stack release. Persist stack channel, +Ghost channel, selected service registry, and resolved image identities separately. +`--image-registry` applies to traffic-analytics, ActivityPub, and its migration image; +it does not promise mirrors of MySQL, Caddy, Redis, stable Ghost, or every other +third-party image. Ghost nightly is GHCR-only regardless of this registry selection. +Help and installation summaries must make that scope clear. + +Dual publishing must use a single tested release build for both registries, include +matching architecture manifests and version metadata, and define complete-publication +checks before advertising a release. Verify registry-specific digests; do not assume +references in two registries have interchangeable digests. Persist full resolved image +references and provenance. Registry selection must survive stack updates, upgrades, +backup/restore, and supervisor execution. Changing registries must preserve service +versions and data, and must not silently run database migrations or newer code. + +Before stable releases are dual-published, preserve the existing mixed registry +locations. On new installs after S14, default eligible services to Docker Hub; existing +installs keep their recorded locations until explicitly switched. Confirm actual +repository ownership/names in each publishing repo rather than inventing GHCR or +Docker Hub paths. Registry outages or missing architecture artifacts fail clearly; +no silent cross-registry fallback. Public image pulls should work without credentials. + +Nightly is an explicitly selected Ghost channel, not a stack prerelease channel and +not an automatic-update schedule. Publish to the agreed Ghost GHCR repository from +an identified source commit, with immutable build tags/metadata and an optional moving +nightly discovery tag. Installation/upgrade resolves discovery to one exact build and +digest; never persist only a moving tag. Use the same Ghost image for tinybird-sync. +Record the Ghost-reported version plus commit/build identity: two nightly builds may +report the same semver. Build discovery and job schemas must handle that deliberately, +without relaxing trusted image allowlists to arbitrary user-supplied references. + +Only nightly sites discover nightly updates. Enabling the channel does not bypass +host major-version policy, backups, write freezing, compatibility checks, or recovery. +Stable-to-nightly and nightly-to-stable are explicit compatibility-checked transitions; +returning to stable may require waiting for a compatible release or restoring a +checkpoint because schema migrations cannot be undone by changing a tag. Retain exact +recovery images/build metadata even if registry retention removes old nightly tags. +Show the channel and build identity in CLI/status/Admin where applicable. + +Redis becomes the default per-site service for fresh local and production installs +when S16 lands. Add `redis` to the generated profiles by default and support +`--without redis` to retain compatible in-memory Ghost caching. The original §2.1 +profile table describes the initial release; S16 extends it as follows: + +| Service | Profiles | Lifecycle | Installation default | +| --- | --- | --- | --- | +| `redis` | `redis` | Long-running, per-site | Enabled on new sites unless explicitly excluded | + +Use a pinned supported Redis image, private site networking with no published host +port, healthchecks, bounded memory, defined eviction/persistence settings, and +credentials handled through the established secret/config interfaces. Configure the +actual cache features and adapter schema supported by the selected Ghost image; +review `docs/codebase/internal-caching.md`, the built-in Redis adapter, and +[Ghost's cache documentation](https://docs.ghost.org/config#cache-adapters). Do not +assume every older supported image accepts the same cache configuration. Incompatible +images must use a documented supported configuration or fail preflight with the opt-out +path, rather than producing a broken default install. + +For existing sites, provide a documented enablement migration that preserves operator +cache overrides and exact image pins. Routine stack updates must not unexpectedly +switch an existing cache backend. Disabling Redis must also remove/revert generated +cache configuration; do not stop it while leaving Ghost pointed at it. Define/test +startup ordering, runtime outage behavior, reconnects, cache invalidation after +upgrade/restore, and the effect of opting out. Do not promise automatic runtime +fallback unless the selected Ghost implementation actually provides it. + +Initial Redis use is rebuildable Ghost cache data. Explicitly document whether it is +persisted for warm restarts and whether backups exclude/rebuild it. Potential later +traffic-analytics/ActivityPub use is an extension point, not a claim of current Redis +support. Before wiring any such consumer, verify its released configuration contract +and distinguish disposable cache from durable queues/counters/salts/other state. +Durable state needs appropriate persistence, eviction, isolation, backup/restore, and +upgrade policy; separate instances when policies differ. Key prefixes or Redis logical +DBs alone do not isolate memory/eviction/durability policies. Redis remains per-site +if shared Caddy/MySQL infra is enabled. + +### 2.10 Where the tooling runs + +Operations split across two layers. The boundary is set by one question: does +this have to work when Docker is broken? + +**Host shell.** Small, portable, and the only thing that runs before an image +exists. + +- The bootstrap shim: check Docker, resolve the release, pull the manager + image, exec into it. +- Preflight and `doctor`. These must diagnose a host where Docker is missing, + stopped, or unreachable, so they cannot depend on the manager image. They may + use it for deeper checks when it is available, and must degrade to useful + host-level output when it is not. +- A dispatcher that maps a command to a `docker run` of the manager image, with + the mount and identity rules below. +- Anything that must survive the manager image being unpullable. + +**Manager image.** Pinned, published from this repo, and where the stateful +work lives: install orchestration, import, backup and restore, Ghost upgrades, +and stack updates. It is one image with several entrypoints, not several +images. + +Rationale, not preference: §2.6 already publishes a privileged supervisor image +from this repo and requires that it "follows §2.5 rather than inventing a second +upgrade/recovery algorithm". If S4-S7 implement backup, upgrade and recovery in +host shell while S8 implements a supervisor in an image, that algorithm exists +twice. Putting the stateful operations in the image the supervisor already needs +keeps one implementation, and the supervisor becomes an entrypoint on it rather +than a parallel codebase. + +Contract for every manager invocation: + +- **The site directory is mounted at its own absolute host path.** Compose bind + sources are resolved by the daemon against the host filesystem, so a site + mounted anywhere else silently binds a different host path. Verified: a site + mounted at `/site` renders `source: /site/data/ghost`, which the daemon then + creates on the host. Refuse to run when `PROJECT_DIR` and the mount point + disagree. +- The Docker socket is host-privileged. Running the manager as a non-root user + does not reduce that authority. Mount nothing writable beyond what the + operation needs. +- Define uid/gid ownership for every file the manager writes into the site + directory. Do not assume host uid 1000, and account for rootless and userns + remapping. +- Interactive prompts go through `/dev/tty` into the container; every prompt has + a flag equivalent so `--no-prompt` is fully scriptable. +- The manager version is pinned with the stack release, resolved by the + bootstrap, and recorded in `.ghost-docker.json` so an operation can be + reproduced later. +- Exit codes and structured errors propagate through the dispatcher unchanged. A + wrapper that collapses failures into "docker run failed" is not acceptable. + +What this does **not** solve, and should not be claimed to: Compose's dotenv +interpolation. Anything writing `.env` still encodes a literal `$` as `$$` +regardless of implementation language, and §2.1 records why the alternative +config format was rejected. + +When install.sh resolves the exact Ghost image (§1), it should write +`GHOST_CONTENT_PATH` and `GHOST_TINYBIRD_PATH` into `.env` from that image's own +`GHOST_CONTENT`, so the layout and the configuration cannot disagree in the +first place. S1 validates the pair; S2 should set it. + +Sequencing: this decision has to be made before S2, because it determines +whether `install.sh` is the installer or a bootstrap that runs one. Steps +already shipped in host shell (S1's `config.sh` and `caddy.sh`) stay where they +are; the boundary does not run through them. + +## 3. Implementation steps + +The numbering is revised from the original plan; use names as well as numbers when +referring to older discussions. These are work packages, not instructions to create +parallel agents or separate tasks automatically. + +Each step includes a copyable implementation prompt. Give the implementation session +this plan (attach it or make the path below accessible), then paste the prompt for +that step. The full step requirements and acceptance criteria remain part of its +scope. A prompt authorizes only that work package, not execution of later steps. +Dependency checks should inspect the actual implementation, not rely on step numbers +being marked complete in a document. + +```text +S1 contracts/config + Compose foundation + ├─ S2 installer + ├─ S3 migration exporter (can begin after the bundle contract is settled) + └─ S4 backup/restore and operation journal +S5 importer/cutover needs S2, S3, S4 +S6 stack releases/updater needs S1, S2, S4 +S7 host Ghost upgrade needs S4, S6 compatibility rules +S8 supervisor protocol/app needs S7 +S9 Ghost adapter/API needs S8 protocol contract +S10 Admin UI needs S9 +S11 file-based secrets needs S1, S4-S8 credential consumers +S12 release qualification needs S1-S10; qualify S11 if included +S13 shared infrastructure needs S12; optional +S14 service image registries needs S12; independent of S13 +S15 Ghost nightly channel needs S14 image resolution; explicit opt-in +S16 default Redis needs S12; independent of S13-S15 +``` + +That graph is also the branch order. Each step is one pull request stacked on +the ones it depends on, so a reviewer sees only that step's diff, and the +contracts in §2 stay reviewable in the branch that changes them. Steps with a +shared dependency and no dependency on each other (S2, S3 and S4 on S1) can be +separate stacks off the same base. Mark a step's status in its section when its +PR lands, and amend the affected contract in §2 in the same PR rather than +afterwards. + +### S1 — Contracts, configuration, and Compose foundation + +Status: implemented, with one deliberate deferral. The `.ghost-docker.json` +schema is specified in §2.2 but its reader/writer moved to S2, alongside +`install.sh`, which is the first thing that writes the file. See +`docs/configuration.md`, `docs/caddy.md` and `docs/bundle-v1.md` for the +contracts as built, and the note in §1 on the `jq` prerequisite, which this +step resolved. + +Repo: ghost-docker, coordinating the bundle schema with Ghost-CLI. Implement §2.1-2.3 +and define the bundle encoding contract in §2.4. Establish `.env`/`ghost.env`, safe +env helpers, atomic writes, metadata schema, unique aliases, DB variables, Caddy +templates, readiness, log limits, and lifecycle-specific restart policies. Keep +infra/member modes out of the supported initial matrix. Preserve existing installs +through the S6 migration path; do not delete migrate.sh yet. + +Acceptance: all supported mode/optional-service configurations validate on minimum +and current Compose; local and production HTTPS smoke tests pass; one-shot jobs stay +stopped after completion; actual container configuration preserves edge-case values +and excludes root credentials. Shellcheck and focused helper tests pass. + +**Implementation prompt** + +```text +Read docs/ghost-cli-replacement.md in this repository first. Treat the +referenced step, its dependencies, and the architecture contracts as the +requirements for this implementation. + +Implement S1 — Contracts, configuration, and Compose foundation. Read the +repository instructions and inspect the existing Compose, Caddy, migration +helpers, and optional-service setup first. + +Follow sections 2.1–2.4 and S1 of the plan. Establish the application/operator +config split, safe env serialization, atomic writes, metadata schema, +parameterized DB connection, unique service aliases, readiness checks, capped +logs, and local/production profiles. Keep one-shot restart policies distinct +from long-running services. Generate and validate Caddy routes with explicit +production reload. ActivityPub and analytics remain per-site; do not add +shared-infra modes or remove the existing migration scripts. + +Document the unpublished bundle v1 contract: raw string values in config and +required bundleCreatedAt/sourceInstallType metadata, with no draft-format +compatibility path. Leave exporter implementation to S3 and legacy-install +migration to S6. + +Verify all supported profile combinations on the declared minimum and current +Compose, local and production ingress, one-shot completion, and real Compose +round trips for special characters. Confirm Ghost receives no infrastructure +root credentials. + +Complete this step only. Update the relevant documentation and focused tests +as part of the implementation. Finish with a summary of changed +behavior/files, verification results, and any unmet acceptance criteria or +blockers for dependent steps. +``` + +### S2 — Local and single-site production installer + +Repo: ghost-docker. Deps: S1. Implement §2.8 for fresh installs, exact version +resolution, stable identity, scriptable prompts, custom proxy use, and optional +service setup. `--import`/supervisor behavior may initially fail as unimplemented +until their steps land; do not advertise working support prematurely. Do not add +infra flags yet. + +Acceptance: CI installs local and production from a candidate release source, checks +Admin readiness through the intended ingress, validates no-prompt/no-start behavior, +and runs two independent local sites. Cover explicit port conflicts and a server +with an existing proxy: installation must fail clearly on the port conflict +rather than stopping the operator's proxy or silently taking its ports. + +**Implementation prompt** + +```text +Read docs/ghost-cli-replacement.md in this repository first. Treat the +referenced step, its dependencies, and the architecture contracts as the +requirements for this implementation. + +Implement S2 — Local and single-site production installer. Read repository +instructions and verify the S1 configuration/helpers are present before +building on them. S1 deferred the `.ghost-docker.json` reader/writer to this step: +implement it against the schema in §2.2, since install.sh is its first writer. + +Follow section 2.8 and S2. Implement the release-selecting bootstrap and +checkout-owned installer with exact Ghost version resolution, stable project +identity, private config files, generated passwords, mode-aware preflight, +free-port selection, optional-service setup, Caddy rendering, readiness +verification, and a useful final summary. Never stop or reconfigure an existing +proxy; fail clearly on a port conflict instead. Prompts must work through /dev/tty; +every required input must be scriptable. Honor --no-prompt and --no-start. Do +not add infra flags; reject import/supervisor options clearly if their +implementation has not landed. + +Add candidate-release CI for local and production installs, two separate local +sites, explicit port conflicts, an existing proxy, and no-prompt/no-start +behavior. Exercise the selected minimum tools and avoid assuming GNU-only +utilities or docker-group access. + +Complete this step only. Update the relevant documentation and focused tests +as part of the implementation. Finish with a summary of changed +behavior/files, verification results, and any unmet acceptance criteria or +blockers for dependent steps. +``` + +### S3 — Complete the Ghost-CLI export contract and cutover support + +Repo: Ghost-CLI, PR #2333 branch. Deps: agreed S1/§2.4 contract. Review the current +export implementation/tests, use raw values in `config`, require the v1 metadata, +test plain-tar extraction, and implement/document deliberate final-export behavior. Review portable snapshot consistency and supported losses. +Use the actual `lib/tasks/import/` implementation as the API reference, not a +nonexistent `lib/tasks/import.js`. Keep the beta warning until qualification. + +Acceptance: fixture exports cover MySQL/portable, ordinary restart-on-failure, +intentional leave-stopped cutover, permissions, and configuration round trips. +Run the Ghost-CLI suite and lint; update bundle documentation and command reference. + +**Implementation prompt** + +```text +Read docs/ghost-cli-replacement.md in this repository first. Treat the +referenced step, its dependencies, and the architecture contracts as the +requirements for this implementation. + +Implement S3 — Complete the Ghost-CLI export contract and cutover support in +the TryGhost/Ghost-CLI repository. Read repository instructions and inspect PR +#2333's claude/ghost-cli-migration-export-c00253 branch and its existing +changes. Preserve unrelated work. Verify the S1 bundle contract is settled +before changing the exporter. + +Follow section 2.4 and S3. Make bundle v1 config contain raw flattened string +values, require bundleCreatedAt and sourceInstallType, and update exporter, +bundle docs, and fixtures together. Do not add configValues, legacy +quoted-config decoding, or a sourceEnvironment fallback for mode selection: +this format has not shipped. + +Implement deliberate final-export/cutover behavior while preserving ordinary +export restart semantics. Address write-freeze and snapshot consistency for +portable exports; document precisely which data/relationships the portable +format cannot preserve. Use lib/tasks/import/ as the API reference. Keep the +beta warning until qualification. + +Test MySQL and portable exports, normal restart-on-failure, intentional +leave-stopped cutover, private bundle permissions, plain-tar extraction on +another host, and actual Compose configuration round trips. Run the Ghost-CLI +test suite and lint. + +Complete this step only. Update the relevant documentation and focused tests +as part of the implementation. Finish with a summary of changed +behavior/files, verification results, and any unmet acceptance criteria or +blockers for dependent steps. +``` + +### S4 — Backup, restore, locks, and recovery journal + +Repo: ghost-docker. Deps: S1. Implement the reusable §2.5 checkpoint/restore contract, +explicit DB connection abstraction, operation lock, maintenance handling, retention, +and journals. Backups include required local application/configuration state and +describe optional-service limitations. Define supported backup formats independently +of migration bundles; a portable export is not a lossless recovery checkpoint. + +Acceptance: restore a representative site to a fresh destination and verify database, +assets/theme/configuration; inject interrupted backup/restore, full disk, stale lock, +and SQL pipeline failure. Recover without exposing an incomplete destination. + +**Implementation prompt** + +```text +Read docs/ghost-cli-replacement.md in this repository first. Treat the +referenced step, its dependencies, and the architecture contracts as the +requirements for this implementation. + +Implement S4 — Backup, restore, locks, and recovery journal. Read repository +instructions and verify the S1 configuration and metadata interfaces before +implementing their consumers. + +Follow sections 2.2 and 2.5 and S4. Provide reusable backup/restore operations +with explicit DB connection and database selection, a shared host-visible +operation lock, maintenance ingress, durable journals, private checkpoint +files, retention, and space checks. Include the required database, content, +configuration, and version/digest metadata. Define how interrupted operations +reconcile actual state and recover stale locks without stealing a live +operation's lock. + +Document optional-service state and remote-state limitations. Keep recovery +checkpoints separate from portable migration bundles. Do not claim success or +rollback completion until the destination or restored system has been +verified. + +Run a restore drill against a representative disposable site, verifying +database, theme/assets, and configuration. Inject interrupted backup/restore, +full disk, stale lock, and SQL pipeline failures; ensure incomplete restored +sites stay inaccessible. + +Complete this step only. Update the relevant documentation and focused tests +as part of the implementation. Finish with a summary of changed +behavior/files, verification results, and any unmet acceptance criteria or +blockers for dependent steps. +``` + +### S5 — Bundle import and migration cutover + +Repo: ghost-docker. Deps: S2, S3, S4. Implement §2.4 and wire `install.sh --import`. +Use a bootstrap helper without dependencies, staged data, the explicit DB target, +isolated portable import, and verified cutover. Support manifest defaults overridden +by flags and deliberate source-restart/recovery instructions. + +Acceptance: exercise both bundle kinds produced by the exporter, raw config values, +separate admin URLs, asset/theme/redirect fidelity, partial retries, invalid archives, +and a same-server migration. Verify the documented portable losses. Only then delete +`scripts/migrate.sh` and `scripts/config-to-env.js` and replace their documentation. + +**Implementation prompt** + +```text +Read docs/ghost-cli-replacement.md in this repository first. Treat the +referenced step, its dependencies, and the architecture contracts as the +requirements for this implementation. + +Implement S5 — Bundle import and migration cutover. Read repository +instructions and verify S2 installer, S3 exporter contract/fixtures, and S4 +recovery primitives are available. Consult the Ghost-CLI migration-bundle +document in the exporter branch. + +Follow section 2.4 and S5. Implement scripts/import.sh and install.sh --import +using private staging, safe path/link validation, an independent pinned +manifest helper, required v1 metadata, raw config values, explicit target DB +selection, and the exact source Ghost image. Do not support the unpublished +quoted-config draft format. + +Handle mysql-dump and isolated authenticated portable imports, explicitly +mount helper scripts, preserve URL/admin overrides, and verify +content/assets/configuration before cutover. Define safe partial retries and +source write-freeze/restart instructions. Rehearsal imports need documented +outbound-side-effect controls. Public ingress must not expose an uninitialized +or partially imported site. + +Test real exporter-produced bundles of both kinds, special config values, +separate admin URLs, themes/assets/redirects, retries, invalid archives, and +same-server cutover. Verify portable losses explicitly. Delete +migrate.sh/config-to-env.js only after these replacement fidelity and recovery +gates pass; update all migration documentation. + +Complete this step only. Update the relevant documentation and focused tests +as part of the implementation. Finish with a summary of changed +behavior/files, verification results, and any unmet acceptance criteria or +blockers for dependent steps. +``` + +### S6 — Stack releases, updater, and legacy migration + +Repo: ghost-docker. Deps: S1, S2, S4. Implement §2.7, release-please configuration, +beta resolution, bootstrap update instructions, and the transactional migration +framework. Preserve the exact Ghost version across stack updates. Use a stable +updater runtime while replacing the checkout itself. + +Acceptance: update from the pre-S1 layout, including existing optional profiles, +custom Caddy routes/overrides, absent metadata, and an untagged starting commit. +Inject failures before/after hooks, pull, and startup; verify complete recovery or an +accurate recovery-required outcome. Test dependency-only release generation. + +**Implementation prompt** + +```text +Read docs/ghost-cli-replacement.md in this repository first. Treat the +referenced step, its dependencies, and the architecture contracts as the +requirements for this implementation. + +Implement S6 — Stack releases, updater, and legacy migration. Read repository +instructions and verify S1, S2, and S4 interfaces and recovery behavior before +implementing stack updates. + +Follow section 2.7 and S6. Configure release-please and stable/beta version +resolution, including explicit dependency-only patch releases. Implement +update.sh and a bootstrap path for installations where that script does not +exist. Stack updates preserve the exact Ghost pin and reject incompatible +upgrade ordering. + +Run the updater from a stable location while replacing the checkout. Record +the previous commit SHA, acquire the operation lock, snapshot +configuration/metadata, journal pre/post-checkout migrations, validate +Compose/Caddy, and verify readiness. Restore code and configuration together +on safe failures; stateful schema changes require the applicable checkpoint +recovery or an accurate recovery-required state. + +Cover pre-S1 installs with absent or existing analytics/activitypub profiles, +untracked custom Caddyfiles, Compose overrides, absent metadata, and untagged +starting commits. Preserve custom routes. Test skipped releases, repeated +hooks, dependency-only releases, and failures during hooks/pull/startup with +complete recovery verification. + +Complete this step only. Update the relevant documentation and focused tests +as part of the implementation. Finish with a summary of changed +behavior/files, verification results, and any unmet acceptance criteria or +blockers for dependent steps. +``` + +### S7 — Host-driven Ghost upgrades + +Repo: ghost-docker. Deps: S4 and S6 compatibility rules. Implement `scripts/upgrade.sh +[version|latest]` following §2.5, initially without a supervisor. Specify the reusable +execution interface so the supervisor cannot diverge from backup/recovery behavior. +Keep supported majors/downgrades constrained and feature compatibility explicit. + +Acceptance: upgrade across an actual database migration, verify optional analytics +sync/deploy, inject startup failure after migration, and restore the checkpoint. +Test process interruption at each mutation boundary, two concurrent requests, and +refusal of unsupported transitions. A live database restore drill is a release gate. + +**Implementation prompt** + +```text +Read docs/ghost-cli-replacement.md in this repository first. Treat the +referenced step, its dependencies, and the architecture contracts as the +requirements for this implementation. + +Implement S7 — Host-driven Ghost upgrades. Read repository instructions and +verify S4's tested checkpoint/restore primitives and S6's +compatibility/version rules are available. + +Follow section 2.5 and S7. Implement scripts/upgrade.sh [version|latest] and a +reusable execution interface that S8 can invoke without duplicating recovery +logic. Acquire the shared site lock, resolve an exact supported image, pull +before downtime, enforce a verified checkpoint, freeze writes, journal +mutations, and verify readiness before resuming traffic. Reject unsupported +major changes and arbitrary downgrades. + +Support the defined per-site optional-service sequence, including Tinybird +sync and deploy. Detect unsupported remote-schema recovery combinations during +preflight. Image reversion alone is not database rollback: restore and verify +the checkpoint before reporting rolled-back, otherwise retain maintenance and +report recovery-required. + +Exercise an actual database migration, failed startup after migration, a +verified restore, interruption at mutation boundaries, concurrent requests, +and unsupported transitions. Do not add the supervisor or Admin feature in +this step. + +Complete this step only. Update the relevant documentation and focused tests +as part of the implementation. Finish with a summary of changed +behavior/files, verification results, and any unmet acceptance criteria or +blockers for dependent steps. +``` + +### S8 — Supervisor image, protocol, and installer integration + +Repo: ghost-docker. Deps: S7. Write `docs/upgrade-supervisor.md` with the exact §2.6 +schemas, transitions, ownership, policy, and recovery rules, then implement the +supervisor with a supported Node runtime and minimal dependencies. Reuse S7 behavior. +Wire `--with supervisor`, request submission/status tooling, and pinned image versions. +Publish amd64/arm64 images to approved registries through CI. + +Acceptance: handwritten requests work before Ghost gains an adapter; duplicate and +malformed requests, permission violations, stale status, supervisor crashes, and +host-operation conflicts behave correctly. Verify actual exchange permissions as +Ghost's runtime uid. Installer must not enable an incompatible Ghost adapter. + +**Implementation prompt** + +```text +Read docs/ghost-cli-replacement.md in this repository first. Treat the +referenced step, its dependencies, and the architecture contracts as the +requirements for this implementation. + +Implement S8 — Supervisor image, protocol, and installer integration. Read +repository instructions and verify S7 exposes a tested reusable +upgrade/recovery interface. + +Follow section 2.6 and S8. First write docs/upgrade-supervisor.md with exact +versioned JSON schemas, transitions, host policy, exchange ownership/mount +layout, durability, request claiming/deduplication, stale status, retention, +and interrupted-job recovery. Then implement the supervisor and container +packaging using S7's operation contract. Use the same site lock as host +operations. Keep Docker socket authority and supported context/path/uid +behavior explicit. + +Wire install.sh --with supervisor, request/status tooling, pinned image +versions, and amd64/arm64 image publishing CI. Enable FileDropUpgradeAdapter +only on compatible Ghost versions with an initialized exchange; manual request +submission must work before the Ghost integration lands. Use strict target +validation and argument-array subprocesses. + +Test successful requests, duplicates/malformed files, permissions as Ghost's +actual uid, stale status, supervisor crashes, host-operation conflicts, and +recovery. Ghost must not be able to replace supervisor-owned status/job files +through a writable parent. + +Complete this step only. Update the relevant documentation and focused tests +as part of the implementation. Finish with a summary of changed +behavior/files, verification results, and any unmet acceptance criteria or +blockers for dependent steps. +``` + +### S9 — Ghost upgrade adapter and Admin API + +Repo: Ghost core. Deps: S8 protocol contract. Follow repository adapter/API guidance. +Implement `NoopUpgradeAdapter` and `FileDropUpgradeAdapter`, permission-checked status/ +request/job APIs, rate limiting, capability reporting, and update-notification metadata. +Noop is default; missing/stale supervisor and malformed protocol data produce useful +status without making unrelated Ghost startup depend on supervisor availability. + +Acceptance: adapter/controller tests, API permission tests, tmp-exchange protocol +tests, version compatibility, and initialization ordering. Keep Admin UI separate. + +**Implementation prompt** + +```text +Read docs/ghost-cli-replacement.md in this repository first. Treat the +referenced step, its dependencies, and the architecture contracts as the +requirements for this implementation. + +Implement S9 — Ghost upgrade adapter and Admin API in the TryGhost/Ghost +repository. Read repository and subsystem instructions, including the relevant +Admin API skill, and verify S8's versioned protocol contract is available in +the Docker repository's docs/upgrade-supervisor.md. + +Follow section 2.6 and S9. Add the upgrade adapter type with +NoopUpgradeAdapter as the default and FileDropUpgradeAdapter as the canonical +file-exchange implementation. Implement status, request creation, and job +retrieval using the agreed schemas and normal Ghost owner/admin authorization, +rate limiting, and error conventions. Add capability/update-notification +metadata for the later Admin UI. + +Handle pending/unknown jobs, malformed data, stale or absent supervisors, and +protocol mismatches deliberately. Supervisor availability must not become a +dependency for unrelated Ghost startup. Respect host-controlled policy rather +than letting request input waive backups or compatibility restrictions. + +Add adapter/controller, permissions, initialization, and temporary-exchange +protocol tests. Keep Admin UI changes for S10 and verify the canonical adapter +names match configuration, documentation, and the supervisor installer. + +Complete this step only. Update the relevant documentation and focused tests +as part of the implementation. Finish with a summary of changed +behavior/files, verification results, and any unmet acceptance criteria or +blockers for dependent steps. +``` + +### S10 — Admin update experience + +Repo: Ghost Admin. Deps: S9. Add the current-version/available-update panel using the +repository's current React/Shade and API conventions. Feature-detect older backends, +show host capabilities, confirm downtime/backup behavior, and display durable job +progress with bounded reconnection and recovery guidance. Wire notification links. + +Acceptance: older backend, unsupported adapter, owner/admin permissions, successful +restart/reconnect, queued job, stale supervisor, failed restore, and recovery-required +states. Include an integration test with the real supervisor after mocked UI tests. + +**Implementation prompt** + +```text +Read docs/ghost-cli-replacement.md in this repository first. Treat the +referenced step, its dependencies, and the architecture contracts as the +requirements for this implementation. + +Implement S10 — Admin update experience in the TryGhost/Ghost repository. Read +the current Admin/React/Shade/API instructions and verify S9's endpoints and +capabilities before building the UI. + +Follow section 2.6 and S10. Add the current/available Ghost version panel, +host capability handling, update confirmation, job progress, and notification +links using established Admin conventions. Feature-detect missing endpoints on +older backends as well as an unsupported adapter. Backup/downtime promises +must reflect the host-enforced policy. + +Persist enough job identity to resume progress after Ghost restarts. Use +bounded reconnection/backoff and distinguish queued, stale, failed, restoring, +rolled-back, and recovery-required outcomes with useful operator guidance. Do +not treat every 404 as an indefinitely restarting service. + +Test older backends, unsupported adapters, owner/admin permissions, successful +restart and reconnect, queued requests, stale supervisors, failed restores, +and recovery-required states. Include a real supervisor integration scenario +after the focused mocked UI tests. + +Complete this step only. Update the relevant documentation and focused tests +as part of the implementation. Finish with a summary of changed +behavior/files, verification results, and any unmet acceptance criteria or +blockers for dependent steps. +``` + +### S11 — Optional file-based secrets + +Repo: ghost-docker. Deps: S1 and the credential consumers in S4-S8. Add Compose secret +files and `_FILE` wiring only for Ghost versions known to support it. Importing older +Ghost 6 images must still work via their supported credential mechanism. Migrate +without changing existing initialized MySQL credentials accidentally. + +Update MySQL init scripts, healthchecks, ActivityPub, backup/import/restore helpers, +and supervisor consumers together. Do not assume ActivityPub supports MySQL image +`_FILE` conventions. Set file ownership/readability for actual container users; +host mode 0600 alone does not guarantee container access. + +Acceptance: root credentials remain absent from Ghost regardless of this feature; +file-enabled supported services do not expose their secret values in environment; +legacy environment-based installs, older imports, restart, and restore still work. + +**Implementation prompt** + +```text +Read docs/ghost-cli-replacement.md in this repository first. Treat the +referenced step, its dependencies, and the architecture contracts as the +requirements for this implementation. + +Implement S11 — Optional file-based secrets. Read repository instructions and +inspect the existing S1 configuration split and credential consumers in S4–S8 +before changing credential transport. This is optional hardening, not a new +requirement for every installation. + +Follow section 2.2 and S11. Add private Compose secret files and _FILE wiring +only for verified compatible Ghost images. Preserve supported +environment-based operation for existing installs and imports of older Ghost +images; never rotate initialized MySQL credentials accidentally during +conversion. + +Update MySQL initialization, healthchecks, ActivityPub, backup/import/restore +helpers, and supervisor consumers together. Verify each service's actual +file-secret support and runtime uid/read permissions rather than assuming +MySQL conventions apply to all. + +Test migration, restart, restore, supported file-enabled services, and +older-image imports. Confirm root credentials never enter Ghost's environment +and file-enabled services do not expose their resolved secrets there. Document +remaining service-specific limitations and compatibility boundaries. + +Complete this step only. Update the relevant documentation and focused tests +as part of the implementation. Finish with a summary of changed +behavior/files, verification results, and any unmet acceptance criteria or +blockers for dependent steps. +``` + +### S12 — Single-site release qualification and documentation + +Repo: ghost-docker, with cross-repo fixtures. Deps: S1-S10; include S11 if shipping. +Consolidate CI and qualify the actual minimum supported tools and image versions. +Run fresh local/production install, optional-service variants, CLI migration, +legacy stack update, Ghost upgrade/recovery, supervisor/Admin, and restore scenarios. +Include Linux runtime tests and macOS-compatible shell/configuration checks. + +README/help include quick starts, prerequisites, version/compatibility policy, +backup/restore, migration losses and cutover, custom proxy configuration, diagnostics, +and uninstall. Document deletion of bind-mounted data separately from `down -v`, with +explicit recovery consequences. Explain command equivalences without claiming full +CLI parity for unsupported features. Keep shared infra marked deferred. + +**Implementation prompt** + +```text +Read docs/ghost-cli-replacement.md in this repository first. Treat the +referenced step, its dependencies, and the architecture contracts as the +requirements for this implementation. + +Implement S12 — Single-site release qualification and documentation, using the +Ghost-CLI and Ghost repositories for cross-repo fixtures/integration as +needed. Read repository instructions and verify S1–S10 are implemented; +include S11 qualification only if it is part of the release. + +Follow S12 and the acceptance contracts throughout the plan. Consolidate CI +around the actual supported minimum/current tools and image versions. Exercise +fresh local and production installation, optional services, CLI bundle +migration, existing-stack update, Ghost upgrade and recovery, supervisor/Admin +interaction, and a real restore drill. Include Linux runtime checks and +macOS-compatible shell/configuration checks. + +Finish README/help and operational documentation covering version policy, +custom proxies, migration fidelity/cutover, backup/restore, diagnostics, and +uninstall. Explain bind-mounted data deletion separately from down -v, and do +not claim unsupported CLI parity. Keep shared infra explicitly deferred to +S13. + +Resolve qualification defects within this release scope. Report the tested +matrix and any remaining release blockers with evidence; do not infer +readiness from successful happy-path installs alone or start the optional +shared-infra phase. + +Complete this step only. Update the relevant documentation and focused tests +as part of the implementation. Finish with a summary of changed +behavior/files, verification results, and any unmet acceptance criteria or +blockers for dependent steps. +``` + +### S13 — Optional shared Caddy/MySQL infrastructure + +Repo: ghost-docker. Deps: S12. Only now introduce `infra` and `site` modes and +`--infra-only`/`--infra` installation. ActivityPub, traffic analytics, and Tinybird +jobs remain per-site; no shared analytics/federation variants in this step. + +Design/implementation requirements: + +- Extend the complete profile/dependency matrix: infra has no Ghost, Caddy must not + require an inactive Ghost, and URL cannot remain universally required. Optional + jobs must not accidentally provision a member's local database. +- Assign unique database names/users and restricted grants per site, with distinct + ActivityPub database/grants where enabled. Backups/restores target a site's DBs, + never the whole shared MySQL instance during a site operation. +- Make DB endpoint selection explicit. If private DB opt-in is offered, add a real + `db` profile and isolate/address it without ambiguous shared `db` aliases. +- Separate private service networks from shared ingress/database connectivity where + appropriate. Shared networks are not tenant isolation; document the trust model. +- Parameterize every optional-service upstream. No global `ghost`, `activitypub`, + or `traffic-analytics` alias is safe across member projects. Verify that shared + Caddy routes ActivityPub assets to the correct site's storage-serving endpoint. +- Resolve network names from actual infra configuration. If a Compose override is + needed for external networking, explicitly support it: persist the file list, + make helpers honor it, document override auto-loading changes and IPv6 composition. + Do not claim COMPOSE_FILE is unused if a fallback actually sets it. +- Add register/unregister/list/check. Serialize registrations, reject duplicate + domains/project identities, provision DB grants safely, validate/reload Caddy, and + leave recoverable state after partial failure. Purge requires explicit selection + and cannot delete another site's data/user grants. +- Route shared-DB operations through the established DB abstraction. The supervisor + cannot assume `exec db` in its own project or access an unmounted infra checkout. + Keep infra root credentials out of member configuration and supervisor scope. +- Define supported Ghost/MySQL/infra version combinations and infra maintenance/ + backup policy. An infra `down` or DB update affects every member; verify recovery + and clearly report this blast radius. Set capacity limits and connection budgets. +- Default new shared setups to dedicated infra. Do not promise converting an existing + single-site installation by merely changing profiles. If offering a conversion, + include stopped MySQL transfer, Caddy certificate volume ownership/identity, + unchanged data, checkpoint, verification, and tested rollback as part of this phase. + +Acceptance: dedicated infra plus two sites on different exact Ghost versions, each +with per-site optional services; upgrade/restore/remove one without touching the +other's data; exercise private DB opt-in if supported, concurrent registration, +cross-site route checks, infra restart/outage, and partial provisioning recovery. +Document unsupported combinations rather than silently falling back to shared aliases. + +**Implementation prompt** + +```text +Read docs/ghost-cli-replacement.md in this repository first. Treat the +referenced step, its dependencies, and the architecture contracts as the +requirements for this implementation. + +Implement S13 — Optional shared Caddy/MySQL infrastructure. Read repository +instructions and verify S12's single-site qualification is complete before +expanding the architecture. + +Follow all S13 design requirements and the existing backup/upgrade/config +contracts. Add dedicated infra/site modes and --infra-only/--infra +provisioning, revisiting profile dependencies and globally required variables. +ActivityPub, traffic analytics, and Tinybird deployment jobs remain per-site; +do not add shared versions of them. + +Use unique restricted DB identities, explicit DB targets, unambiguous service +aliases, and deliberate private/shared networks. Integrate registration locks, +duplicate checks, validated Caddy reload, recoverable provisioning, and safe +site-scoped removal. Ensure site backup/restore/supervisor operations cannot +affect another site's databases or require its supervisor to hold infra root +credentials. Define infra capacity, version, and maintenance policy. Persist +any required Compose file selection consistently. + +Default to dedicated infra. If offering conversion or private DB opt-in, +implement and test their complete data/volume/recovery paths rather than just +changing profiles. Test two Ghost versions with per-site optional services, +isolated upgrade/restore/purge, concurrent registration, route/asset +separation, infra outages, and partial failures. Document unsupported +combinations and the infrastructure-wide maintenance impact. + +Complete this step only. Update the relevant documentation and focused tests +as part of the implementation. Finish with a summary of changed +behavior/files, verification results, and any unmet acceptance criteria or +blockers for dependent steps. +``` + +### S14 — Dual-published service images and registry selection + +Repos: ghost-docker plus the traffic-analytics and ActivityPub publishing repositories. +Deps: S12; independent of S13. Follow §2.9. Publish traffic-analytics, ActivityPub, and +ActivityPub migrations to both Docker Hub and GHCR, then add +`install.sh --image-registry dockerhub|ghcr` and a documented registry-switch operation. +Persist full image identities and registry choice; apply it consistently to upgrade, +stack update, recovery, and supervisor flows. Preserve historical mixed registry +locations until a site explicitly switches. Do not broaden the flag to unmirrored +third-party images. + +Acceptance: both registries serve equivalent releases on supported architectures; +partial publication is not advertised as complete; unauthenticated public pulls work; +select/install/update/restore and same-version registry switches succeed. Exercise +missing artifacts, unavailable registries, and ActivityPub app/migration version +alignment. Confirm switches do not mutate application data or upgrade versions. + +**Implementation prompt** + +```text +Read docs/ghost-cli-replacement.md in this repository, especially section 2.9 +and S14. Implement S14 here and in the actual traffic-analytics/ActivityPub +image publishing repositories. Read each repository's instructions and inspect +its release workflows before editing. Verify S12 is complete; S13 is not +required. + +Add dual publishing to Docker Hub and GHCR from one tested release build, +covering traffic-analytics, ActivityPub, and ActivityPub migrations. Resolve +actual repository names/permissions and publish matching versions/platforms +with verified identities. Define how partial publication is handled before +releases are advertised. + +Add --image-registry dockerhub|ghcr, durable registry/image settings, and an +explicit same-version switch workflow. Default new eligible installs to Docker +Hub after both registries are ready; preserve existing image locations on +updates. Carry the selected references through Compose, updater, supervisor, +backup, and recovery. The flag covers these services only; nightly Ghost +remains GHCR-only and third-party images retain their declared registries. Do +not add silent fallback to another registry. + +Test both registries/platforms, public pulls, missing/partial artifacts, +same-version switches, data preservation, app/migration alignment, and +restore. Update help/docs and report implementation changes, validation, and +any unmet acceptance criteria. Implement this step only; do not begin nightly +or Redis work. +``` + +### S15 — Opt-in Ghost nightly channel on GHCR + +Repos: Ghost/image publishing workflow and ghost-docker; Ghost Admin/API if channel +or build metadata requires extending the existing upgrade interface. Deps: S14 image +resolution and existing S7-S10 upgrade integration. Follow §2.9. Add +`--ghost-channel stable|nightly`, with stable as default and nightly explicitly opted +in. Nightly images are published to GHCR with immutable source/build identities. +Keep the stack `--channel` independent and do not equate nightly selection with +unattended upgrades. + +Acceptance: stable installs never select nightlies; opt-in resolves an exact GHCR +build on supported architectures; successive builds with identical Ghost semver are +distinguishable; tinybird-sync uses the selected Ghost artifact. Exercise discovery +failure, missing images, host major-policy enforcement, backup/recovery, and explicit +channel transitions. Nightly-to-stable must refuse unsafe schema transitions rather +than pretending that image selection rolls back the database. + +**Implementation prompt** + +```text +Read docs/ghost-cli-replacement.md in this repository, especially section 2.9 +and S15. Implement S15 in the verified Ghost image publishing workflow and +here, with targeted Ghost upgrade API/Admin changes if required. Read +repository instructions and inspect the S14 image resolver and S7-S10 upgrade +integration before changing them. + +Publish opt-in Ghost nightlies to the agreed GHCR repository with immutable +tags, source commit/build metadata, supported architecture manifests, and +clear retention. Add --ghost-channel stable|nightly; stable is default. +Persist the Ghost channel separately from the stack stable/beta channel and +service image registry choice. Resolve discovery tags to exact builds/digests +and use that artifact for tinybird-sync. + +Extend discovery/status/job data to distinguish builds that report identical +semver. Maintain trusted repository allowlists, host major policy, mandatory +production checkpoints, and database-aware recovery. Show channel/build +identity to operators. Require explicit compatible channel transitions; +returning to stable may require a later compatible release or checkpoint +restore. Channel selection does not authorize automatic background upgrades. + +Test stable isolation, explicit opt-in, consecutive same-semver builds, GHCR +failures, architecture coverage, sync-image consistency, channel transitions, +and recovery after a migrated nightly fails. Update docs/tests and report +remaining blockers. Complete S15 only; do not start Redis implementation. +``` + +### S16 — Default per-site Redis caching with opt-out + +Repo: ghost-docker, verifying behavior against supported Ghost versions. Deps: S12 +and existing configuration, installer, backup, upgrade, and secret interfaces; +independent of S13-S15. Follow §2.9. Make Redis the default for new local/production +sites, with explicit `--without redis` opt-out and documented adoption for existing +sites. Add the service/profile, version-aware Ghost cache wiring, private network, +healthchecks, credentials, resource policy, diagnostics, and enable/disable migration. +Keep Redis per-site even when shared infra exists. + +Acceptance: new local/production installs use Redis by default; opt-out starts a +working site without it; existing operator overrides survive migration. Verify real +cache reads/writes and invalidation, resource limits, restart/outage/reconnect, +upgrade/restore behavior, secret handling, and supported Ghost version coverage. +If S13 has shipped, verify two sites do not share cache data or expose Redis through +the shared ingress network. Specify cache rebuilding/persistence behavior explicitly. +Do not wire speculative traffic-analytics/ActivityPub consumers until their released +interfaces exist and their cache-versus-durable-state requirements are established. + +**Implementation prompt** + +```text +Read docs/ghost-cli-replacement.md in this repository, especially section 2.9 +and S16. Implement S16 here. Read repository instructions and verify S12 plus +the installer/configuration, backup/upgrade, and secret interfaces. S13-S15 +are not prerequisites. + +Add a pinned, private per-site Redis service and enable its profile by default +on new local and production installs. Implement --without redis and a +documented existing-site enable/disable migration that preserves operator +cache overrides. Configure the Redis adapter/cache features actually supported +by each selected Ghost image, using +docs/codebase/internal-caching.md in the TryGhost/Ghost repository and the +corresponding adapter code as references; verify older image compatibility. + +Define credentials, healthchecks, startup ordering, bounded memory/eviction, +optional cache persistence, runtime failure/reconnect behavior, and +invalidation on restore or upgrade. No host port is published. Disabling Redis +must revert the generated Ghost configuration too. Do not silently change an +existing site's cache backend on update or promise runtime fallback without +verifying Ghost's implementation. + +Initially treat Redis as rebuildable Ghost cache. Document extension points +for future analytics/ActivityPub consumers, but do not invent their +configuration or mix durable state into an evictable cache instance. Different +durability/eviction requirements need separate state policy and, where +appropriate, separate instances. + +Test default installs, opt-out, existing-site migration/overrides, actual +cache use, restart/outage, resource limits, secrets, upgrade/restore, and +older Ghost versions. Test site isolation if shared infra is present. Update +help/docs and report changed behavior, verification, and unmet acceptance +criteria. Complete this step only. +``` + +## 4. Reference notes from review + +- Compose interpolates inactive services. `env_file` does not supply Compose's own + `${...}` interpolation. Double-quoted dotenv values can interpolate dollar signs. +- A Compose profile named in the environment enables nothing unless a service lists + it. Profiles do not change a service's fields or combine as logical AND conditions. +- `COMPOSE_FILE` affects override auto-loading; explicit `-f` replaces the selected + list. Every helper, supervisor invocation, and IPv6 example must use one contract. +- Host and container bind paths must agree for a container invoking host Docker. + Docker context and user namespace differences also affect paths/ownership. +- Restart policy is not migration orchestration or readiness. One-shot jobs must + remain one-shot, and failed schema migrations need database-aware recovery. +- The existing MySQL init script reads root credentials from environment; update it + as well as the healthcheck when introducing secret files. +- Git checkout cannot overwrite an untracked file with a tracked file. Pre-checkout + migration backups and recovery must include configuration, not just a Git ref. +- Authoritative references: [Compose profiles](https://docs.docker.com/compose/how-tos/profiles/), + [dotenv interpolation](https://docs.docker.com/compose/how-tos/environment-variables/variable-interpolation/), + [Caddy commands](https://caddyserver.com/docs/command-line), and + [release-please](https://github.com/googleapis/release-please). diff --git a/ghost.env.example b/ghost.env.example new file mode 100644 index 00000000..b8f846ab --- /dev/null +++ b/ghost.env.example @@ -0,0 +1,35 @@ +# Ghost application settings. +# +# This is the only `env_file` of the ghost service. Put Ghost configuration +# here, in Ghost's `section__subsection__key` form +# (https://ghost.org/docs/config/). +# +# Do NOT put operator or infrastructure settings here: COMPOSE_*, PROJECT_DIR, +# DOMAIN, ports, data locations and DATABASE_ROOT_PASSWORD belong in `.env`. +# +# The following keys are owned by the container and set as explicit Compose +# `environment` entries, which override this file. Setting them here has no +# effect and `scripts/config.sh validate` rejects them: +# +# url, admin__url, NODE_ENV, server__*, paths__*, database__* +# +# Values are interpolated by Compose. Do NOT hand-edit a value containing `$`: +# `s3cr$t!` reaches Ghost as `s3cr!`, and `Pa$$w0rd!` reaches it as `Pa$w0rd!`, +# with no error anywhere. Use `scripts/config.sh set ghost.env KEY VALUE`, which +# encodes it correctly. +# +# This file holds credentials (SMTP, integrations). Keep it mode 0600. + +# SMTP email (https://ghost.org/docs/config/#mail) +# Transactional email is required for logins, staff invites and password +# resets. This is unrelated to bulk mail / newsletter sending. +mail__transport="SMTP" +mail__options__host="smtp.example.com" +mail__options__port="465" +mail__options__secure="true" +mail__options__auth__user="support@example.com" +mail__options__auth__pass="change-me" +mail__from="'Acme Support' " + +# Developer experiments required by the analytics and ActivityPub features. +# labs__publicAPI="true" diff --git a/help b/help index 2bd4c4c4..68a9054b 100755 --- a/help +++ b/help @@ -1,25 +1,51 @@ #!/usr/bin/env bash -cat << 'EOF' +cat << 'HELPEOF' ════════════════════════════════════════════════════════════════════ GHOST DOCKER HELP & COMMANDS ════════════════════════════════════════════════════════════════════ +SITE MODES (exactly one, selected in COMPOSE_PROFILES in .env): + local ghost + db, published on 127.0.0.1:${GHOST_PORT} + production ghost + db + caddy, HTTPS on ${HTTP_PORT}/${HTTPS_PORT} + + Optional, additive, per-site profiles: analytics, activitypub + COMMON COMMANDS: - docker compose logs -f ghost # View real-time logs - docker compose logs -f caddy # View Caddy webserver logs - docker compose ps # Check service status + docker compose up -d # Start the services for the selected mode docker compose down # Stop all services - docker compose up -d # Start all services + docker compose ps # Check service status + docker compose logs -f ghost # View real-time Ghost logs + docker compose logs -f caddy # View Caddy webserver logs docker compose restart ghost # Restart Ghost container +CONFIGURATION: + .env Compose and operator settings, including the MySQL root + password. Never passed into the Ghost container. + ghost.env Ghost application settings only (section__key form). + + scripts/config.sh validate # Check both files by mode + scripts/config.sh set ghost.env KEY VALUE # Write safely and atomically + scripts/config.sh unset ghost.env KEY # Remove a setting + + Restart Ghost after changes: docker compose up -d + +ROUTING (production): + scripts/caddy.sh apply # Render, validate, install, reload, verify + scripts/caddy.sh render # Render the candidate without installing it + scripts/caddy.sh validate # Validate the installed configuration + + Generated routes: caddy/sites/*.caddy (do not edit) + Your own routes: caddy/custom/*.caddy + Global options: caddy/global/*.caddy + TROUBLESHOOTING: docker compose exec ghost sh # Access Ghost container shell docker compose logs --tail=100 # View last 100 log lines docker stats # Monitor resource usage DATABASE ACCESS: - docker compose exec mysql mysql -u root -p + docker compose exec db mysql -u root -p # Use the DATABASE_ROOT_PASSWORD from your .env file UPGRADES: @@ -32,17 +58,18 @@ UPGRADES: 2. Read Ghost's release notes 3. Test in a staging environment -CONFIGURATION: - - Edit .env file for environment variables - - Restart Ghost after changes: docker compose up -d - USEFUL PATHS: - Content: ./data/ghost/ - Database: ./data/mysql/ + Content: ${UPLOAD_LOCATION:-./data/ghost} + Database: ${MYSQL_DATA_LOCATION:-./data/mysql} + Metadata: ./.ghost-docker.json Logs: docker compose logs - Config: ./.env + +DOCUMENTATION: + docs/configuration.md Configuration split, profiles, lifecycle, metadata + docs/caddy.md Route generation, custom routes, validation + docs/bundle-v1.md Migration bundle contract MORE HELP: Ghost Docs: https://ghost.org/docs/ Ghost Community: https://forum.ghost.org/ -EOF +HELPEOF diff --git a/scripts/caddy.sh b/scripts/caddy.sh new file mode 100755 index 00000000..5c30b10e --- /dev/null +++ b/scripts/caddy.sh @@ -0,0 +1,45 @@ +#!/usr/bin/env bash +# Generate, validate and install this site's Caddy routes. +# +# scripts/caddy.sh render [DIR] render the candidate tree into caddy/.staging +# scripts/caddy.sh validate [DIR] validate the installed routes +# scripts/caddy.sh apply [DIR] render, validate, install, reload, verify +# scripts/caddy.sh reload [DIR] explicit reload of the running Caddy +# +# Production uses an explicit reload; Caddy's --watch is a development feature. +set -euo pipefail + +# shellcheck source=scripts/lib/common.sh +. "$(dirname -- "$0")/lib/common.sh" + +cmd=${1:-} +(($#)) && shift || true +dir="${1:-$GD_ROOT_DIR}" + +case "$cmd" in + render) + staged=$(caddy_render "$dir") + printf 'staged routes in %s\n' "$staged" + cat "$staged"/*.caddy + ;; + validate) + caddy_validate "$dir" + printf 'installed Caddy configuration is valid\n' + ;; + apply) + caddy_apply "$dir" + printf 'Caddy routes applied\n' + ;; + reload) + caddy_reload "$dir" + printf 'Caddy reloaded\n' + ;; + '' | -h | --help | help) + usage + ;; + *) + printf 'unknown command: %s\n' "$cmd" >&2 + usage >&2 + exit 2 + ;; +esac diff --git a/scripts/config-to-env.js b/scripts/config-to-env.js index 54dccc66..37970e1f 100644 --- a/scripts/config-to-env.js +++ b/scripts/config-to-env.js @@ -11,8 +11,9 @@ const HARDCODED_EXCLUSIONS = [ 'logging', 'process', 'paths', - // We don't need URL because its already set in our env + // We don't need URL or admin__url because the container owns them 'url', + 'admin__url', ]; // Parse command line arguments @@ -72,15 +73,25 @@ function flattenObject(obj, prefix = '') { return result; } -// Format value for shell output +// Serialize a value for a Docker Compose env file. +// +// Compose interpolates dotenv values, including inside double quotes, so a +// literal `$` must be written `$$`. Values are always double quoted, which is +// the only encoding that can represent every value; see the encoding contract +// at the top of scripts/lib/env.sh. function formatValue(value) { - // If value contains spaces, newlines, or quotes, wrap in quotes - if (typeof value === 'string' && (value.includes(' ') || value.includes('\n') || value.includes('"') || value.includes("'"))) { - // Escape any existing double quotes - value = value.replace(/"/g, '\\"'); - return `"${value}"`; + const text = String(value); + let out = ''; + for (const c of text) { + if (c === '\\') { out += '\\\\'; } + else if (c === '"') { out += '\\"'; } + else if (c === '$') { out += '$$'; } + else if (c === '\n') { out += '\\n'; } + else if (c === '\r') { out += '\\r'; } + else if (c === '\t') { out += '\\t'; } + else { out += c; } } - return value; + return `"${out}"`; } // Main function @@ -131,7 +142,7 @@ function main() { continue; } - // Output in KEY=VALUE format + // Output in KEY="VALUE" format, ready for ghost.env console.log(`${key}=${formatValue(value)}`); } } diff --git a/scripts/config.sh b/scripts/config.sh new file mode 100755 index 00000000..0d04f36e --- /dev/null +++ b/scripts/config.sh @@ -0,0 +1,54 @@ +#!/usr/bin/env bash +# Inspect and validate a site's configuration split. +# +# scripts/config.sh validate [DIR] validate .env and ghost.env by mode +# scripts/config.sh set FILE KEY VALUE write one value, safely and atomically +# scripts/config.sh unset FILE KEY +# scripts/config.sh mode [DIR] print the selected site mode +set -euo pipefail + +# shellcheck source=scripts/lib/common.sh +. "$(dirname -- "$0")/lib/common.sh" + +cmd=${1:-} +(($#)) && shift || true + +case "$cmd" in + validate) + dir="${1:-$GD_ROOT_DIR}" + if config_validate "$dir"; then + printf 'configuration in %s is valid\n' "$dir" + else + exit 1 + fi + ;; + set) + (($# == 3)) || { + usage + exit 2 + } + env_set "$1" "$2" "$3" + # Key names only. Values are never printed: any of them may be a + # credential, and a list of "sensitive" names would silently miss one. + printf 'updated %s in %s\n' "$2" "$1" >&2 + ;; + unset) + (($# == 2)) || { + usage + exit 2 + } + env_unset "$1" "$2" + ;; + mode) + dir="${1:-$GD_ROOT_DIR}" + compose_site_mode "$(env_get "$dir/.env" COMPOSE_PROFILES)" + ;; + '' | -h | --help | help) + usage + ;; + *) + printf 'unknown command: %s\n' "$cmd" >&2 + usage >&2 + exit 2 + ;; +esac diff --git a/scripts/lib/caddy.sh b/scripts/lib/caddy.sh new file mode 100644 index 00000000..8a034c9c --- /dev/null +++ b/scripts/lib/caddy.sh @@ -0,0 +1,280 @@ +#!/usr/bin/env bash +# Caddy route generation, validation, installation and explicit reload. +# +# Layout (all paths relative to the site directory): +# +# caddy/Caddyfile tracked, generic; imports the two directories below +# caddy/sites/*.caddy generated, gitignored, owned by this script +# caddy/custom/*.caddy operator owned, gitignored, never touched here +# caddy/snippets/* tracked, imported by absolute path with arguments +# caddy/.staging/ candidate tree, validated before it is installed +# +# Production uses an explicit `caddy reload`. Caddy documents `--watch` as a +# local development feature, so it is not used here. + +# shellcheck disable=SC2034 +GD_CADDY_LIB_LOADED=1 + +GD_CADDY_CONTAINER_ROOT="/etc/caddy" + +# caddy_render DIR +# Renders the candidate tree into DIR/caddy/.staging/sites. Returns 1 on error. +# _caddy_site_block DOMAIN GHOST_UPSTREAM ADMIN_DOMAIN OPTIONAL_ROUTES +_caddy_site_block() { + cat <&2 + return 1 + } + + profiles=$(env_get "$env" COMPOSE_PROFILES 2>/dev/null || printf '') + mode=$(compose_site_mode "$profiles") || { + printf 'error: COMPOSE_PROFILES=%s must name exactly one site mode\n' "$profiles" >&2 + return 1 + } + if [[ $mode != "production" ]]; then + printf 'error: caddy routes are only rendered in production mode (mode is %s)\n' "$mode" >&2 + return 1 + fi + + project=$(env_get "$env" COMPOSE_PROJECT_NAME 2>/dev/null || printf '') + domain=$(env_get "$env" DOMAIN 2>/dev/null || printf '') + for required in "$project" "$domain"; do + [[ -n $required ]] || { + printf 'error: COMPOSE_PROJECT_NAME and DOMAIN are required in production mode\n' >&2 + return 1 + } + done + admin=$(env_get "$env" ADMIN_DOMAIN 2>/dev/null || printf '') + url=$(env_get "$env" URL 2>/dev/null || printf '') + www=$(env_get "$env" WWW_REDIRECT 2>/dev/null || printf '') + + # Optional routes, in the order they must appear inside a site block. + # Analytics is rendered only with its profile; ActivityPub is always + # routed, to this site's own service or to the hosted one. + optional="" + case ",$profiles," in + *,analytics,*) + optional=" + # Traffic Analytics service + import $GD_CADDY_CONTAINER_ROOT/snippets/TrafficAnalytics traffic-analytics-$project:3000" + ;; + esac + case ",$profiles," in + *,activitypub,*) ap="activitypub-$project:8080" ;; + *) + ap=$(env_get "$env" ACTIVITYPUB_TARGET 2>/dev/null || printf '') + [[ -n $ap ]] || ap="https://ap.ghost.org" + ;; + esac + optional="$optional + # ActivityPub service + import $GD_CADDY_CONTAINER_ROOT/snippets/ActivityPub $ap +" + + staging="$dir/caddy/.staging" + rm -rf "$staging" + mkdir -p "$staging/sites" || return 1 + + { + printf '# Generated by scripts/caddy.sh for %s. Do not edit.\n' "$project" + printf '# Add your own routes as .caddy files in caddy/custom/ instead.\n\n' + _caddy_site_block "$domain" "ghost-$project:2368" "$admin" "$optional" + if [[ -n $admin ]]; then + printf '\n' + _caddy_site_block "$admin" "ghost-$project:2368" "$admin" "$optional" + fi + if [[ -n $www ]]; then + printf '\n%s {\n\timport %s/snippets/Logging\n\tredir %s{uri}\n}\n' \ + "$www" "$GD_CADDY_CONTAINER_ROOT" "$url" + fi + } >"$staging/sites/site.caddy" || return 1 + + printf '%s\n' "$staging/sites" +} + +# caddy_validate DIR [SITES_DIR_IN_CONTAINER] +# Validates the tracked Caddyfile against a sites directory. Missing import +# arguments only produce a Caddy warning during adaptation, so they are +# promoted to errors here. +caddy_validate() { + local dir=$1 sites=${2:-$GD_CADDY_CONTAINER_ROOT/sites} out rc + + out=$(compose_run "$dir" run --rm --no-deps -T \ + -e "CADDY_SITES_DIR=$sites" \ + -e "CADDY_CUSTOM_DIR=$GD_CADDY_CONTAINER_ROOT/custom" \ + -e "CADDY_GLOBAL_DIR=$GD_CADDY_CONTAINER_ROOT/global" \ + caddy caddy validate \ + --config "$GD_CADDY_CONTAINER_ROOT/Caddyfile" \ + --adapter caddyfile 2>&1) + rc=$? + + if ((rc != 0)); then + printf '%s\n' "$out" >&2 + return 1 + fi + if printf '%s' "$out" | grep -q 'index is out of bounds'; then + printf 'error: a Caddy import is missing an argument:\n%s\n' "$out" >&2 + return 1 + fi + if printf '%s' "$out" | grep -q "No files matching import glob pattern\",\"pattern\":\"$sites"; then + printf 'error: no site routes were found in %s\n' "$sites" >&2 + return 1 + fi + return 0 +} + +# caddy_install DIR +# Atomically replaces the generated routes with the staged candidate. +# Prints the path of the backup of the previous routes. +caddy_install() { + local dir=$1 staging=$1/caddy/.staging/sites live=$1/caddy/sites backup f base + [[ -d $staging ]] || { + printf 'error: nothing staged; run caddy_render first\n' >&2 + return 1 + } + mkdir -p "$live" || return 1 + + backup=$(fs_mktemp_dir ghost-docker-caddy) || return 1 + for f in "$live"/*.caddy; do + [[ -e $f ]] || continue + cp -p "$f" "$backup/" || return 1 + done + + # Remove generated routes that the new candidate no longer produces. + for f in "$live"/*.caddy; do + [[ -e $f ]] || continue + base=$(basename "$f") + [[ -e $staging/$base ]] || rm -f "$f" + done + + for f in "$staging"/*.caddy; do + [[ -e $f ]] || continue + base=$(basename "$f") + cat "$f" | fs_atomic_write "$live/$base" 0644 || return 1 + done + + printf '%s\n' "$backup" +} + +# caddy_restore DIR BACKUP_DIR +caddy_restore() { + local dir=$1 backup=$2 live=$1/caddy/sites f + live="$dir/caddy/sites" + [[ -d $backup ]] || return 1 + rm -f "$live"/*.caddy + for f in "$backup"/*.caddy; do + [[ -e $f ]] || continue + cp -p "$f" "$live/" || return 1 + done +} + +# caddy_running DIR +caddy_running() { + local id + id=$(compose_run "$1" ps -q caddy 2>/dev/null) + [[ -n $id ]] +} + +# caddy_reload DIR +# Explicit reload. Never `--watch`. +caddy_reload() { + compose_run "$1" exec -T caddy caddy reload \ + --config "$GD_CADDY_CONTAINER_ROOT/Caddyfile" \ + --adapter caddyfile +} + +# caddy_verify DIR DOMAIN... +# Confirms the reloaded server is actually routing the expected hostnames, by +# reading Caddy's own loaded configuration rather than trusting the reload +# exit status. +caddy_verify() { + local dir=$1 cfg domain rc=0 + shift + cfg=$(compose_run "$dir" exec -T caddy \ + wget -q -O - http://127.0.0.1:2019/config/apps/http/servers 2>/dev/null) || { + printf 'error: could not read the running Caddy configuration\n' >&2 + return 1 + } + for domain in "$@"; do + [[ -n $domain ]] || continue + if ! printf '%s' "$cfg" | grep -q "\"$domain\""; then + printf 'error: %s is not routed by the running Caddy configuration\n' "$domain" >&2 + rc=1 + fi + done + return $rc +} + +# caddy_apply DIR +# Render, validate, install, reload, verify. The previous on-disk configuration +# is restored if validation, reload or verification fails. +caddy_apply() { + local dir=$1 backup env domain admin + + caddy_render "$dir" >/dev/null || return 1 + + if ! caddy_validate "$dir" "$GD_CADDY_CONTAINER_ROOT/.staging/sites"; then + printf 'error: the candidate Caddy configuration is not valid; nothing was changed\n' >&2 + return 1 + fi + + backup=$(caddy_install "$dir") || return 1 + + if ! caddy_validate "$dir"; then + printf 'error: the installed Caddy configuration failed validation; restoring the previous routes\n' >&2 + caddy_restore "$dir" "$backup" + return 1 + fi + + if ! caddy_running "$dir"; then + printf 'caddy is not running; routes installed, start the stack to apply them\n' + rm -rf "$backup" + return 0 + fi + + if ! caddy_reload "$dir"; then + printf 'error: caddy reload failed; restoring the previous routes\n' >&2 + caddy_restore "$dir" "$backup" + caddy_reload "$dir" || printf 'error: restoring the previous configuration also failed\n' >&2 + return 1 + fi + + env="$dir/.env" + domain=$(env_get "$env" DOMAIN 2>/dev/null || printf '') + admin=$(env_get "$env" ADMIN_DOMAIN 2>/dev/null || printf '') + if ! caddy_verify "$dir" "$domain" "$admin"; then + printf 'error: routing verification failed; restoring the previous routes\n' >&2 + caddy_restore "$dir" "$backup" + caddy_reload "$dir" || printf 'error: restoring the previous configuration also failed\n' >&2 + return 1 + fi + + rm -rf "$backup" + rm -rf "$dir/caddy/.staging" + return 0 +} diff --git a/scripts/lib/common.sh b/scripts/lib/common.sh new file mode 100644 index 00000000..f43380bc --- /dev/null +++ b/scripts/lib/common.sh @@ -0,0 +1,38 @@ +#!/usr/bin/env bash +# Loads every ghost-docker shell library in dependency order. +# +# Usage from a script in scripts/: +# . "$(dirname "$0")/lib/common.sh" + +# shellcheck disable=SC2034 +GD_COMMON_LIB_LOADED=1 + +GD_LIB_DIR=$(CDPATH='' cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd) +GD_ROOT_DIR=$(CDPATH='' cd -- "$GD_LIB_DIR/../.." && pwd) + +# shellcheck source=scripts/lib/fs.sh +. "$GD_LIB_DIR/fs.sh" +# shellcheck source=scripts/lib/env.sh +. "$GD_LIB_DIR/env.sh" +# shellcheck source=scripts/lib/compose.sh +. "$GD_LIB_DIR/compose.sh" +# shellcheck source=scripts/lib/config.sh +. "$GD_LIB_DIR/config.sh" +# shellcheck source=scripts/lib/caddy.sh +. "$GD_LIB_DIR/caddy.sh" + +# usage +# Prints the calling script's header comment block: everything from the line +# after the shebang up to the first non-comment line, with the leading `# ` +# stripped. `$0` is the CLI that sourced this file, not this file. +usage() { + local line + { + read -r line # shebang + while IFS= read -r line; do + [[ $line == '#'* ]] || break + line=${line#\#} + printf '%s\n' "${line# }" + done + } <"$0" +} diff --git a/scripts/lib/compose.sh b/scripts/lib/compose.sh new file mode 100644 index 00000000..7a6c2ae3 --- /dev/null +++ b/scripts/lib/compose.sh @@ -0,0 +1,97 @@ +#!/usr/bin/env bash +# Docker Compose invocation contract and profile rules. +# +# Helpers run Compose from outside the site directory, so they go through +# compose_run: `--project-directory` (never `-C`), an explicit file list, and +# no inherited `COMPOSE_FILE`, which would change override auto-loading. +# Operators running `docker compose` from the site directory get the same +# result. + +# shellcheck disable=SC2034 +GD_COMPOSE_LIB_LOADED=1 + +# Declared minimum supported versions, verified in tests against this exact +# minimum and the current release. install.sh checks them during preflight. +# - `depends_on: required` needs Compose >= 2.20.0 +# - `env_file: [{path, required}]` needs Compose >= 2.24.0 +# - `healthcheck.start_interval` needs Docker Engine >= 25.0.0 +readonly GD_MIN_COMPOSE_VERSION="2.24.0" +readonly GD_MIN_DOCKER_VERSION="25.0.0" + +readonly GD_SITE_MODES=(local production) +readonly GD_OPTIONAL_PROFILES=(analytics activitypub supervisor) + +# Services whose lifecycle is one-shot. They keep `restart: "no"`. +readonly GD_ONE_SHOT_SERVICES=(activitypub-migrate tinybird-login tinybird-sync tinybird-deploy) + +# compose_run DIR ARGS... +# Set GD_COMPOSE_OVERRIDES to a comma separated list of extra files, for +# example `compose.ipv6.yml`. +compose_run() { + local dir=$1 + shift + local -a files=(-f "$dir/compose.yml") + + if [[ -n ${GD_COMPOSE_OVERRIDES:-} ]]; then + local override + local -a overrides + IFS=, read -ra overrides <<<"$GD_COMPOSE_OVERRIDES" + for override in "${overrides[@]}"; do + [[ -n $override ]] || continue + if [[ $override == /* ]]; then + files+=(-f "$override") + else + files+=(-f "$dir/$override") + fi + done + fi + + ( + unset COMPOSE_FILE + exec docker compose --project-directory "$dir" "${files[@]}" "$@" + ) +} + +# _gd_split_profiles PROFILES -> one trimmed, non-empty profile per line +_gd_split_profiles() { + local -a parts + local part + IFS=, read -ra parts <<<"$1" + for part in "${parts[@]}"; do + # Trim surrounding whitespace without relying on GNU-only behaviour. + part=${part#"${part%%[![:space:]]*}"} + part=${part%"${part##*[![:space:]]}"} + [[ -n $part ]] && printf '%s\n' "$part" + done +} + +# compose_site_mode PROFILES +# Prints the single site mode named in a COMPOSE_PROFILES value, or fails when +# zero or more than one is present. Profiles are additive: optional profiles +# never select a mode and never combine as a condition. +compose_site_mode() { + local profile mode found="" count=0 + while read -r profile; do + for mode in "${GD_SITE_MODES[@]}"; do + if [[ $profile == "$mode" ]]; then + found=$profile + ((count++)) + fi + done + done < <(_gd_split_profiles "$1") + + ((count == 1)) || return 1 + printf '%s\n' "$found" +} + +# compose_unknown_profiles PROFILES +# Prints any profile that is neither a site mode nor a known optional profile. +compose_unknown_profiles() { + local profile known + while read -r profile; do + for known in "${GD_SITE_MODES[@]}" "${GD_OPTIONAL_PROFILES[@]}"; do + [[ $profile == "$known" ]] && continue 2 + done + printf '%s\n' "$profile" + done < <(_gd_split_profiles "$1") +} diff --git a/scripts/lib/config.sh b/scripts/lib/config.sh new file mode 100644 index 00000000..fa837777 --- /dev/null +++ b/scripts/lib/config.sh @@ -0,0 +1,282 @@ +#!/usr/bin/env bash +# The application/operator configuration split. +# +# .env Compose and operator settings: project identity, mode, ports, +# data locations, restart policy, and INFRASTRUCTURE credentials +# such as DATABASE_ROOT_PASSWORD. Read by Compose for `${...}` +# interpolation. It is never passed into the Ghost container. +# +# ghost.env Ghost application settings only, in Ghost's `section__key` +# form. It is the only `env_file` of the ghost service. Keys that +# the container owns are set as explicit Compose `environment` +# entries, which override `env_file`, and must not appear here. +# +# Requirements are validated by mode, not by putting `:?` guards on +# optional-service variables. Compose interpolates inactive services too, so a +# `:?` guard on a production-only variable would break local mode. + +# shellcheck disable=SC2034 +GD_CONFIG_LIB_LOADED=1 + +GD_ENV_FILE_NAME=".env" +GD_GHOST_ENV_FILE_NAME="ghost.env" + +# Keys required in .env that nothing else would catch. URL, DATABASE_PASSWORD +# and DATABASE_ROOT_PASSWORD are deliberately absent: compose.yml guards those +# with `:?`, so Compose reports them itself, at the point of use. +readonly GD_REQUIRED_KEYS_COMMON=( + COMPOSE_PROFILES + COMPOSE_PROJECT_NAME + SITE_MODE + PROJECT_DIR + GHOST_VERSION +) + +readonly GD_REQUIRED_KEYS_PRODUCTION=(DOMAIN) + +# config_ghost_environment DIR +# The environment Compose actually gives the ghost container, as KEYVALUE. +# `docker compose config` is pure parsing and needs no daemon, so this works +# before anything is started. Its output re-escapes `$` as `$$`, which is undone +# here so values compare against decoded ones. +config_ghost_environment() { + compose_run "$1" config --format json 2>/dev/null | + jq -r '.services.ghost.environment // {} + | to_entries[] + | [.key, (.value // "" | tostring | gsub("[$][$]"; "$"))] + | @tsv' +} + +# config_operator_variables DIR +# The variables Compose interpolates from `.env`. Derived from compose.yml +# itself rather than listed, so it cannot drift when a service gains a setting. +config_operator_variables() { + local ref + while read -r ref; do + printf '%s\n' "${ref#\$\{}" + done < <(grep -oE '\$\{[A-Za-z_][A-Za-z0-9_]*' "$1/compose.yml" 2>/dev/null | sort -u) +} + +# config_image_content_path IMAGE +# The content path the Ghost image itself declares, via its GHOST_CONTENT +# environment variable. Returns 1 when the image is not available locally, so +# callers treat this as a best-effort check rather than a requirement. +config_image_content_path() { + local line + while IFS= read -r line; do + if [[ $line == GHOST_CONTENT=* ]]; then + printf '%s\n' "${line#GHOST_CONTENT=}" + return 0 + fi + done < <(docker image inspect "$1" --format '{{range .Config.Env}}{{println .}}{{end}}' 2>/dev/null) + return 1 +} + +# config_validate_env DIR +# Validates .env for the mode it declares. Prints findings; returns 1 on error. +# +# Only checks what nothing else catches. A missing URL or DATABASE_PASSWORD, a +# bad restart policy and a bad port are left to Compose and Docker, which +# reject them with clear errors of their own. +config_validate_env() { + local dir=$1 file=$1/$GD_ENV_FILE_NAME rc=0 + local profiles mode declared_mode unknown key value url domain ap_db extra lint mode_bits + + if [[ ! -f $file ]]; then + printf 'error: %s is missing\n' "$file" + return 1 + fi + + profiles=$(env_get "$file" COMPOSE_PROFILES 2>/dev/null) || profiles= + if [[ -z $profiles ]]; then + printf 'error: COMPOSE_PROFILES is not set; it must name exactly one site mode (%s)\n' \ + "${GD_SITE_MODES[*]}" + return 1 + fi + + if ! mode=$(compose_site_mode "$profiles"); then + printf 'error: COMPOSE_PROFILES=%s must name exactly one site mode (%s)\n' \ + "$profiles" "${GD_SITE_MODES[*]}" + mode= + rc=1 + fi + + unknown=$(compose_unknown_profiles "$profiles") + if [[ -n $unknown ]]; then + printf 'error: unknown profile(s) in COMPOSE_PROFILES: %s\n' "${unknown//$'\n'/ }" + rc=1 + fi + + declared_mode=$(env_get "$file" SITE_MODE 2>/dev/null) || declared_mode= + if [[ -n $mode && -n $declared_mode && $mode != "$declared_mode" ]]; then + printf 'error: SITE_MODE=%s does not match the site mode in COMPOSE_PROFILES (%s)\n' \ + "$declared_mode" "$mode" + rc=1 + fi + + local -a required=("${GD_REQUIRED_KEYS_COMMON[@]}") + [[ $mode == production ]] && required+=("${GD_REQUIRED_KEYS_PRODUCTION[@]}") + + for key in "${required[@]}"; do + if ! value=$(env_get "$file" "$key" 2>/dev/null); then + printf 'error: %s is required for mode %s\n' "$key" "${mode:-unknown}" + rc=1 + continue + fi + if [[ -z $value ]]; then + printf 'error: %s is empty\n' "$key" + rc=1 + fi + done + + # Caddy serves a certificate for DOMAIN while Ghost is configured for URL, + # so a mismatch produces a working server that serves the wrong site. + if [[ $mode == production ]]; then + url=$(env_get "$file" URL 2>/dev/null) || url= + domain=$(env_get "$file" DOMAIN 2>/dev/null) || domain= + if [[ $url != "https://$domain" && $url != "https://$domain/"* ]]; then + printf 'error: URL (%s) and DOMAIN (%s) disagree\n' "$url" "$domain" + rc=1 + fi + fi + + # The extra databases are created once, on first initialisation. A custom + # ActivityPub database name that is not in that list would never exist. + ap_db=$(env_get "$file" ACTIVITYPUB_DATABASE_NAME 2>/dev/null) || ap_db= + if [[ -n $ap_db ]]; then + extra=$(env_get "$file" DATABASE_EXTRA_DATABASES 2>/dev/null) || extra=activitypub + if [[ ,$extra, != *",$ap_db,"* ]]; then + printf 'error: ACTIVITYPUB_DATABASE_NAME=%s is not listed in DATABASE_EXTRA_DATABASES (%s)\n' \ + "$ap_db" "$extra" + rc=1 + fi + fi + + # The image layout moved between variants: `next` installs Ghost under + # /home/ghost, older tags under /var/lib/ghost. Ask the image what it + # expects rather than mapping tag names, which would drift. Best effort: + # skipped when the image has not been pulled. + local image version declared expected + image=$(env_get "$file" GHOST_IMAGE 2>/dev/null) || image= + version=$(env_get "$file" GHOST_VERSION 2>/dev/null) || version= + declared=$(env_get "$file" GHOST_CONTENT_PATH 2>/dev/null) || declared=/home/ghost/content + if [[ -n $version ]] && expected=$(config_image_content_path "${image:-ghost}:$version"); then + if [[ $declared != "$expected" ]]; then + printf 'error: GHOST_CONTENT_PATH is %s but %s expects %s; set GHOST_CONTENT_PATH and GHOST_TINYBIRD_PATH to match the image\n' \ + "$declared" "${image:-ghost}:$version" "$expected" + rc=1 + fi + fi + + if ! lint=$(env_lint "$file"); then + printf '%s\n' "$lint" + rc=1 + fi + + mode_bits=$(fs_stat_mode "$file" 2>/dev/null) || mode_bits= + case $mode_bits in + 600 | 400 | 640 | '') ;; + *) printf 'warning: %s mode is %s; it holds credentials and should be 0600\n' "$file" "$mode_bits" ;; + esac + + return $rc +} + +# config_validate_ghost_env DIR +# ghost.env holds Ghost application settings only. Two mistakes matter, and +# both are detected rather than listed: +# +# * a key the container owns. Compose `environment` overrides `env_file`, so +# setting it here looks effective and is silently ignored. Detected by +# asking Compose what the container actually receives. +# * an operator setting in the wrong file. Detected from the variables +# compose.yml interpolates, plus anything already set in `.env`. +config_validate_ghost_env() { + local dir=$1 file=$1/$GD_GHOST_ENV_FILE_NAME rc=0 + local key value resolved lint mode_bits + local -a operator_vars=() + + # ghost.env is optional: a site can run entirely on container-owned config. + [[ -f $file ]] || return 0 + + # KEYVALUE of what the container really gets. Empty when compose.yml + # cannot be resolved, in which case the override check is skipped rather + # than reporting nonsense. + local container + container=$(config_ghost_environment "$dir") + if [[ -z $container ]]; then + printf 'warning: could not resolve the Compose configuration; skipping the container-owned key check\n' + fi + + while read -r key; do + operator_vars+=("$key") + done < <(config_operator_variables "$dir") + + while read -r key; do + value=$(env_get "$file" "$key" 2>/dev/null) || continue + + # Compose merges env_file into the service environment, so every + # ghost.env key appears here. A different value means an explicit + # `environment` entry took precedence. + if [[ -n $container ]] && resolved=$(_gd_container_value "$container" "$key"); then + if [[ $resolved != "$value" ]]; then + printf 'error: %s is set by the container (%s) and is ignored in %s\n' \ + "$key" "${resolved:-}" "$GD_GHOST_ENV_FILE_NAME" + rc=1 + continue + fi + fi + + if _gd_is_operator_key "$key" "$dir" "${operator_vars[@]}"; then + printf 'error: %s is an operator setting and belongs in %s\n' "$key" "$GD_ENV_FILE_NAME" + rc=1 + fi + done < <(env_keys "$file") + + if ! lint=$(env_lint "$file"); then + printf '%s\n' "$lint" + rc=1 + fi + + mode_bits=$(fs_stat_mode "$file" 2>/dev/null) || mode_bits= + case $mode_bits in + 600 | 400 | 640 | '') ;; + *) printf 'warning: %s mode is %s; it holds credentials and should be 0600\n' "$file" "$mode_bits" ;; + esac + + return $rc +} + +# _gd_container_value TSV KEY +# Prints the container's value for KEY, or returns 1 when it has none. +_gd_container_value() { + local k v + while IFS=$'\t' read -r k v; do + if [[ $k == "$2" ]]; then + printf '%s' "$v" + return 0 + fi + done <<<"$1" + return 1 +} + +# _gd_is_operator_key KEY DIR OPERATOR_VAR... +# True when KEY belongs in .env: Compose interpolates it, it is one of +# Compose's own settings, or .env already defines it. +_gd_is_operator_key() { + local key=$1 dir=$2 var + shift 2 + [[ $key == COMPOSE_* ]] && return 0 + for var in "$@"; do + [[ $key == "$var" ]] && return 0 + done + env_get "$dir/$GD_ENV_FILE_NAME" "$key" >/dev/null 2>&1 +} + +# config_validate DIR +config_validate() { + local rc=0 + config_validate_env "$1" || rc=1 + config_validate_ghost_env "$1" || rc=1 + return $rc +} diff --git a/scripts/lib/env.sh b/scripts/lib/env.sh new file mode 100644 index 00000000..0a85afbd --- /dev/null +++ b/scripts/lib/env.sh @@ -0,0 +1,257 @@ +#!/usr/bin/env bash +# Safe dotenv serialization and parsing for Docker Compose. +# +# This library NEVER sources or evaluates an env file. Env files are data, and +# bash's own parser is not a substitute: it expands `$$` to the process id and +# executes `$(...)`, where Compose treats both as text. A validator that +# disagrees with Compose is worse than none. +# +# The encoding rules this implements are documented in docs/configuration.md +# ("Value encoding"), and verified against real containers in +# tests/env-compose.test.mjs. In short: Compose interpolates dotenv values even +# inside double quotes, so a literal `$` is written `$$`, and double quotes are +# the only form that can represent every value. +# +# Decoding needs a single left-to-right pass so that `\\n` yields a backslash +# followed by `n` rather than a newline; jq's alternation does that in one +# expression. Encoding has no such ordering hazard and is plain substitution. +# +# A value whose quotes span several lines is valid dotenv but is not editable +# through these helpers: it is skipped when listing keys, and reading or +# writing that key fails with a clear message. + +# shellcheck disable=SC2034 +GD_ENV_LIB_LOADED=1 + +# _gd_env_scan FILE +# Locates assignments. Emits LINEKEYTYPEBODY, where TYPE is +# `d` for double quoted, `s` for single quoted, `u` for unquoted or `x` for a +# value that runs past the end of the line. BODY is still escaped. +# +# Every consumer shares this one grammar, so reading and rewriting can never +# disagree about where a value starts and ends. The regexes live in variables +# because bash 3.2 treats a quoted `=~` right-hand side as a literal. +_gd_env_scan() { + [[ -f $1 ]] || return 1 + local line raw key n=0 open='' + local double='^"((\\.|[^"\\])*)"' + local single="^'((\\\\'|[^'])*)'" + local closes_double='^([^"\\]|\\.)*"' + local valid_key='^[A-Za-z_][A-Za-z0-9_]*$' + local trailing_comment='^(.*)[[:space:]]#' + + while IFS= read -r line || [[ -n $line ]]; do + ((n++)) + + # Inside a multi-line value: skip it rather than mistaking one of its + # lines for an assignment. + if [[ -n $open ]]; then + if [[ $open == '"' && $line =~ $closes_double ]]; then + open='' + elif [[ $open == "'" && $line == *"'"* ]]; then + open='' + fi + continue + fi + + line=${line#"${line%%[![:space:]]*}"} + [[ -z $line || $line == '#'* ]] && continue + if [[ $line == export[[:space:]]* ]]; then + line=${line#export} + line=${line#"${line%%[![:space:]]*}"} + fi + [[ $line == *=* ]] || continue + + key=${line%%=*} + key=${key%"${key##*[![:space:]]}"} + [[ $key =~ $valid_key ]] || continue + raw=${line#*=} + + if [[ $raw == '"'* ]]; then + if [[ $raw =~ $double ]]; then + printf '%d\t%s\td\t%s\n' "$n" "$key" "${BASH_REMATCH[1]}" + else + open='"' + printf '%d\t%s\tx\t\n' "$n" "$key" + fi + elif [[ $raw == "'"* ]]; then + if [[ $raw =~ $single ]]; then + printf '%d\t%s\ts\t%s\n' "$n" "$key" "${BASH_REMATCH[1]}" + else + open="'" + printf '%d\t%s\tx\t\n' "$n" "$key" + fi + else + # Unquoted: ` #` starts a comment, trailing whitespace is trimmed. + [[ $raw =~ $trailing_comment ]] && raw=${BASH_REMATCH[1]} + raw=${raw%"${raw##*[![:space:]]}"} + printf '%d\t%s\tu\t%s\n' "$n" "$key" "$raw" + fi + done <"$1" +} + +# _gd_env_escape VALUE +# Encodes VALUE for the inside of a double-quoted dotenv value. Backslashes +# first: later substitutions introduce their own, which must not be escaped a +# second time. +_gd_env_escape() { + local v=$1 + v=${v//\\/\\\\} + v=${v//\"/\\\"} + v=${v//\$/\$\$} + v=${v//$'\n'/\\n} + v=${v//$'\r'/\\r} + v=${v//$'\t'/\\t} + printf '%s' "$v" +} + +# _gd_env_unescape ENCODED +# One pass over the escapes Compose applies, so `\\n` is a backslash then `n`. +_gd_env_unescape() { + jq -jn --arg v "$1" ' + $v | gsub("(?\\\\.)|(?[$][$])"; + if .e then + (.e[1:2]) as $c + | if $c == "n" then "\n" + elif $c == "t" then "\t" + elif $c == "r" then "\r" + elif $c == "\\" or $c == "\"" or $c == "\u0027" or $c == "$" then $c + else .e end + else "$" end)' +} + +# _gd_env_valid_key KEY +_gd_env_valid_key() { + local valid='^[A-Za-z_][A-Za-z0-9_]*$' + [[ $1 =~ $valid ]] +} + +# _gd_env_serialize KEY VALUE +_gd_env_serialize() { + _gd_env_valid_key "$1" || return 1 + printf '%s="%s"\n' "$1" "$(_gd_env_escape "$2")" +} + +# env_keys FILE +env_keys() { + local n key type body + while IFS=$'\t' read -r n key type body; do + printf '%s\n' "$key" + done < <(_gd_env_scan "$1") +} + +# env_get FILE KEY +# Prints the decoded value followed by a newline. The last assignment wins, +# matching Compose. Returns non-zero when the key is absent or spans lines. +env_get() { + local n key type body found=0 found_type='' found_body='' + _gd_env_valid_key "$2" || return 2 + while IFS=$'\t' read -r n key type body; do + [[ $key == "$2" ]] || continue + found=1 + found_type=$type + found_body=$body + done < <(_gd_env_scan "$1") + ((found)) || return 1 + + case $found_type in + x) + printf 'error: %s spans several lines; edit it by hand\n' "$2" >&2 + return 1 + ;; + # Single quoted: literal, and a backslash before a quote is the only escape. + s) printf '%s\n' "${found_body//\\\'/\'}" ;; + *) + _gd_env_unescape "$found_body" + printf '\n' + ;; + esac +} + +# _gd_env_locate FILE KEY +# Prints the line number of the last assignment of KEY, `x` when that value +# spans several lines, or nothing when the key is absent. +_gd_env_locate() { + local n key type body at='' + while IFS=$'\t' read -r n key type body; do + [[ $key == "$2" ]] || continue + if [[ $type == x ]]; then at=x; else at=$n; fi + done < <(_gd_env_scan "$1") + [[ -n $at ]] && printf '%s\n' "$at" + return 0 +} + +# _gd_env_write FILE LINE [REPLACEMENT] [MODE] +# Replaces LINE with REPLACEMENT, deletes it when REPLACEMENT is empty, or +# appends when LINE is 0. The file is replaced atomically. +_gd_env_write() { + local file=$1 target=${2:-0} replacement=${3:-} mode=${4:-} line n=0 + { + while IFS= read -r line || [[ -n $line ]]; do + ((n++)) + if ((n == target)); then + if [[ -n $replacement ]]; then + printf '%s\n' "$replacement" + fi + continue + fi + printf '%s\n' "$line" + done <"$file" + if ((target == 0)) && [[ -n $replacement ]]; then + printf '%s\n' "$replacement" + fi + } | fs_atomic_write "$file" "$mode" +} + +# env_set FILE KEY VALUE [MODE] +# Replaces an existing assignment in place, keeping its position and the +# comments around it, or appends a new one. +env_set() { + local file=$1 key=$2 value=$3 mode=${4:-} new at + _gd_env_valid_key "$key" || return 2 + new=$(_gd_env_serialize "$key" "$value") || return 1 + + if [[ ! -f $file ]]; then + printf '%s\n' "$new" | fs_atomic_write "$file" "${mode:-0600}" + return + fi + + at=$(_gd_env_locate "$file" "$key") + if [[ $at == x ]]; then + printf 'error: %s spans several lines; edit it by hand\n' "$key" >&2 + return 1 + fi + _gd_env_write "$file" "${at:-0}" "$new" "$mode" +} + +# env_unset FILE KEY +env_unset() { + local at + _gd_env_valid_key "$2" || return 2 + [[ -f $1 ]] || return 0 + at=$(_gd_env_locate "$1" "$2") + [[ -n $at && $at != x ]] || return 0 + _gd_env_write "$1" "$at" "" +} + +# env_lint FILE +# Reports entries whose on-disk encoding is interpolated by Compose and so does +# not hold the literal value the operator probably intended. Returns 1 when any +# were found. +env_lint() { + [[ -f $1 ]] || return 0 + local n key type body probe rc=0 + while IFS=$'\t' read -r n key type body; do + [[ $type == d || $type == u ]] || continue + # Remove every escaped or doubled form, in this order. Any `$` still + # standing is one Compose will interpolate. + probe=${body//\\\\/} + probe=${probe//\\\$/} + probe=${probe//\$\$/} + if [[ $probe == *'$'* ]]; then + printf '%s: value is interpolated by Compose (unescaped $); write $$ for a literal dollar sign\n' "$key" + rc=1 + fi + done < <(_gd_env_scan "$1") + return $rc +} diff --git a/scripts/lib/fs.sh b/scripts/lib/fs.sh new file mode 100644 index 00000000..059b410c --- /dev/null +++ b/scripts/lib/fs.sh @@ -0,0 +1,66 @@ +#!/usr/bin/env bash +# Filesystem helpers: private temporary files and atomic replacement. +# +# Every helper that writes a credential-bearing file goes through +# fs_atomic_write so that: +# * the file is created with a restrictive mode before any content is written +# * readers never observe a partially written file +# * an existing file's mode and ownership are preserved + +# shellcheck disable=SC2034 +GD_FS_LIB_LOADED=1 + +# fs_stat_mode PATH -> octal mode, or non-zero when the path does not exist +fs_stat_mode() { + [[ -e $1 ]] || return 1 + stat -f '%Lp' "$1" 2>/dev/null || stat -c '%a' "$1" 2>/dev/null +} + +# _gd_stat_owner PATH -> uid:gid +_gd_stat_owner() { + [[ -e $1 ]] || return 1 + stat -f '%u:%g' "$1" 2>/dev/null || stat -c '%u:%g' "$1" 2>/dev/null +} + +# fs_mktemp_dir [PREFIX] -> path to a new private directory +fs_mktemp_dir() { + local prefix=${1:-ghost-docker} + (umask 077 && mktemp -d "${TMPDIR:-/tmp}/${prefix}.XXXXXXXX") +} + +# fs_atomic_write PATH [MODE] +# +# Reads the new content from stdin and replaces PATH atomically. MODE defaults +# to the existing file's mode, or 0600 for a new file: credential-bearing files +# are the common case, so pass 0644 explicitly for public ones. +fs_atomic_write() { + local path=$1 mode=${2:-} + local dir existing_mode existing_owner tmp + + dir=$(dirname "$path") + [[ -d $dir ]] || return 1 + + existing_mode=$(fs_stat_mode "$path" 2>/dev/null) || existing_mode= + existing_owner=$(_gd_stat_owner "$path" 2>/dev/null) || existing_owner= + [[ -n $mode ]] || mode=${existing_mode:-0600} + + tmp="$path.tmp.$$" + # Create the temporary file privately before any content reaches it. + (umask 077 && : >"$tmp") || return 1 + + if ! cat >"$tmp"; then + rm -f "$tmp" + return 1 + fi + if ! chmod "$mode" "$tmp"; then + rm -f "$tmp" + return 1 + fi + # Best effort: only meaningful when running with sufficient privileges. + [[ -n $existing_owner ]] && chown "$existing_owner" "$tmp" 2>/dev/null + + if ! mv -f "$tmp" "$path"; then + rm -f "$tmp" + return 1 + fi +} diff --git a/scripts/migrate.sh b/scripts/migrate.sh index 21c52260..6655a209 100644 --- a/scripts/migrate.sh +++ b/scripts/migrate.sh @@ -170,7 +170,7 @@ WHAT WONT HAPPEN: ✓ Original installation remains intact REQUIREMENTS: - ✓ .env file configured for Docker + ✓ .env file configured for Docker (see .env.example) ✓ MySQL credentials with dump permissions ✓ Sufficient disk space for migration @@ -571,9 +571,16 @@ main() { node "${PWD}/scripts/config-to-env.js" "${current_location}/config.production.json" echo "" - echo -e "\n# Configuration imported from existing Ghost install at ${current_location}" >> "${PWD}/.env" - node "${PWD}/scripts/config-to-env.js" "${current_location}/config.production.json" >> "${PWD}/.env" - echo "✓ Configuration imported" + # Ghost application configuration goes to ghost.env, which is the only + # env_file of the ghost service. .env holds Compose and operator settings, + # including the MySQL root password, and is never passed into the container. + if [[ ! -f "${PWD}/ghost.env" ]]; then + (umask 077 && : > "${PWD}/ghost.env") + fi + echo -e "\n# Configuration imported from existing Ghost install at ${current_location}" >> "${PWD}/ghost.env" + node "${PWD}/scripts/config-to-env.js" "${current_location}/config.production.json" >> "${PWD}/ghost.env" + chmod 0600 "${PWD}/ghost.env" + echo "✓ Configuration imported into ghost.env" # Start Ghost echo "" @@ -619,7 +626,8 @@ main() { echo " • Original files: $current_location" echo " • Original database: $mysql_database on $mysql_host" echo " • New content location: ${PWD}/data/ghost/" - echo " • Configuration: ${PWD}/.env" + echo " • Operator configuration: ${PWD}/.env" + echo " • Ghost configuration: ${PWD}/ghost.env" echo "" echo "QUICK START COMMANDS:" echo " View logs: docker compose logs -f ghost" diff --git a/tests/caddy.test.mjs b/tests/caddy.test.mjs new file mode 100644 index 00000000..a4ed6c59 --- /dev/null +++ b/tests/caddy.test.mjs @@ -0,0 +1,157 @@ +// Caddy route generation, validation and installation. +// +// Rendering needs no Docker; validation and the missing-argument guard use the +// Caddy image. +import { test, describe, before, after, beforeEach } from 'node:test'; +import assert from 'node:assert/strict'; +import { readFileSync, writeFileSync, rmSync, readdirSync } from 'node:fs'; +import { join } from 'node:path'; +import { tempDir, cleanup, makeSite, writeEnv, sh, shOk, shSucceeds, dockerAvailable, q } from './helpers.mjs'; + +const CADDY_ROOT = '/etc/caddy'; +const STAGED_SITES = `${CADDY_ROOT}/.staging/sites`; + +describe('caddy.sh', () => { + let dir; + let site; + + before(() => { + dir = tempDir('caddy'); + site = makeSite(dir); + }); + after(() => cleanup(dir)); + + const setup = (overrides = {}) => + writeEnv(join(site, '.env'), { + COMPOSE_PROFILES: 'production', + SITE_MODE: 'production', + COMPOSE_PROJECT_NAME: 'ghost-example-com', + PROJECT_DIR: site, + NODE_ENV: 'production', + URL: 'https://example.com', + DOMAIN: 'example.com', + RESTART_POLICY: 'unless-stopped', + GHOST_VERSION: '6-next-alpine', + DATABASE_HOST: 'db', + DATABASE_NAME: 'ghost', + DATABASE_USER: 'ghost', + DATABASE_PASSWORD: 'app-password', + DATABASE_ROOT_PASSWORD: 'root-password', + ...overrides, + }); + + const render = () => { + const result = sh(`caddy_render ${q(site)}`); + assert.equal(result.status, 0, result.stderr.toString()); + return readFileSync(join(site, 'caddy', '.staging', 'sites', 'site.caddy'), 'utf8'); + }; + + describe('rendering', () => { + beforeEach(() => setup()); + + test('a plain production site', () => { + const routes = render(); + assert.match(routes, /^example\.com \{$/m); + assert.match(routes, /reverse_proxy ghost-ghost-example-com:2368/); + // The bare service name must never be used for addressing. + assert.doesNotMatch(routes, /reverse_proxy ghost:2368/); + assert.match(routes, /import \/etc\/caddy\/snippets\/SecurityHeaders ""/); + assert.match(routes, /import \/etc\/caddy\/snippets\/ActivityPub https:\/\/ap\.ghost\.org/); + assert.doesNotMatch(routes, /TrafficAnalytics/); + assert.doesNotMatch(routes, /\{\{|\$\{/, 'an unsubstituted placeholder reached the output'); + }); + + test('optional profiles point at this site\'s own services', () => { + setup({ COMPOSE_PROFILES: 'production,analytics,activitypub' }); + const routes = render(); + assert.match(routes, /import \/etc\/caddy\/snippets\/TrafficAnalytics traffic-analytics-ghost-example-com:3000/); + assert.match(routes, /import \/etc\/caddy\/snippets\/ActivityPub activitypub-ghost-example-com:8080/); + assert.doesNotMatch(routes, /ap\.ghost\.org/); + }); + + test('an admin domain gets its own block and the frame-ancestors argument', () => { + setup({ ADMIN_DOMAIN: 'admin.example.com', WWW_REDIRECT: 'www.example.com' }); + const routes = render(); + assert.match(routes, /^admin\.example\.com \{$/m); + assert.match(routes, /import \/etc\/caddy\/snippets\/SecurityHeaders "admin\.example\.com"/); + assert.match(routes, /redir https:\/\/example\.com\{uri\}/); + }); + + test('local mode renders no routes at all', () => { + setup({ COMPOSE_PROFILES: 'local', SITE_MODE: 'local', URL: 'http://localhost:2368', DOMAIN: undefined }); + assert.ok(!shSucceeds(`caddy_render ${q(site)}`)); + }); + }); + + describe('validation', { skip: dockerAvailable() ? false : 'docker is not available' }, () => { + beforeEach(() => { + setup({ COMPOSE_PROFILES: 'production,analytics,activitypub' }); + render(); + for (const f of readdirSync(join(site, 'caddy', 'custom'))) { + if (f.endsWith('.caddy')) rmSync(join(site, 'caddy', 'custom', f)); + } + }); + + const validate = (sites = STAGED_SITES) => sh(`caddy_validate ${q(site)} ${q(sites)}`); + + test('the candidate configuration validates', () => { + assert.equal(validate().status, 0, validate().stderr.toString()); + }); + + test('operator routes are validated alongside the generated ones', () => { + writeFileSync( + join(site, 'caddy', 'custom', 'extra.caddy'), + 'status.example.com {\n\timport /etc/caddy/snippets/Logging\n\trespond "ok" 200\n}\n', + ); + assert.equal(validate().status, 0); + + writeFileSync(join(site, 'caddy', 'custom', 'broken.caddy'), 'this is not a caddyfile {\n'); + assert.notEqual(validate().status, 0, 'a broken operator route must fail validation'); + }); + + test('global options are validated alongside the generated routes', () => { + writeFileSync(join(site, 'caddy', 'global', 'tls.caddy'), 'local_certs\n'); + assert.equal(validate().status, 0); + writeFileSync(join(site, 'caddy', 'global', 'tls.caddy'), 'not_a_global_option\n'); + assert.notEqual(validate().status, 0); + rmSync(join(site, 'caddy', 'global', 'tls.caddy')); + }); + + test('a missing import argument is an error, not a warning', () => { + // Caddy only warns about this during adaptation, and the resulting + // server misbehaves at runtime. + const staged = join(site, 'caddy', '.staging', 'sites', 'site.caddy'); + writeFileSync(staged, readFileSync(staged, 'utf8').replace(' ""', '')); + const result = validate(); + assert.notEqual(result.status, 0); + assert.match(result.stderr.toString(), /missing an argument/); + }); + + test('no generated routes at all is an error', () => { + const stagedDir = join(site, 'caddy', '.staging', 'sites'); + for (const f of readdirSync(stagedDir)) rmSync(join(stagedDir, f)); + assert.notEqual(validate().status, 0); + }); + }); + + describe('install and restore', { skip: dockerAvailable() ? false : 'docker is not available' }, () => { + test('installs, validates in place, and can be rolled back', () => { + setup(); + render(); + shOk(`caddy_install ${q(site)}`); + + const live = join(site, 'caddy', 'sites', 'site.caddy'); + const installed = readFileSync(live, 'utf8'); + assert.match(installed, /reverse_proxy ghost-ghost-example-com:2368/); + assert.equal(sh(`caddy_validate ${q(site)}`).status, 0); + + setup({ DOMAIN: 'changed.example.com', URL: 'https://changed.example.com' }); + render(); + const backup = shOk(`caddy_install ${q(site)}`).trim(); + assert.match(readFileSync(live, 'utf8'), /^changed\.example\.com \{$/m); + + shOk(`caddy_restore ${q(site)} ${q(backup)}`); + assert.equal(readFileSync(live, 'utf8'), installed); + }); + }); +}); diff --git a/tests/compose-matrix.test.mjs b/tests/compose-matrix.test.mjs new file mode 100644 index 00000000..b1ccd632 --- /dev/null +++ b/tests/compose-matrix.test.mjs @@ -0,0 +1,217 @@ +// Every supported mode / optional-service combination, resolved by Compose +// itself, on the declared minimum Compose and on the installed one. +// +// Set GD_TEST_MIN_COMPOSE to a Compose binary at the declared minimum version +// to include it in the matrix. +import { test, describe, before, after } from 'node:test'; +import assert from 'node:assert/strict'; +import { writeFileSync, chmodSync } from 'node:fs'; +import { join } from 'node:path'; +import { + tempDir, cleanup, makeSite, writeEnv, compose, composeConfig, + composeBinaries, dockerAvailable, sh, shOk, q, +} from './helpers.mjs'; + +const LONG_RUNNING = ['ghost', 'db', 'caddy', 'traffic-analytics', 'activitypub']; +const ONE_SHOT = ['activitypub-migrate', 'tinybird-login', 'tinybird-sync', 'tinybird-deploy']; + +const MATRIX = [ + { profiles: 'local', mode: 'local', services: ['db', 'ghost'] }, + { profiles: 'local,analytics', mode: 'local', services: ['db', 'ghost', 'tinybird-deploy', 'tinybird-login', 'tinybird-sync', 'traffic-analytics'] }, + { profiles: 'local,activitypub', mode: 'local', services: ['activitypub', 'activitypub-migrate', 'db', 'ghost'] }, + { profiles: 'local,analytics,activitypub', mode: 'local', services: ['activitypub', 'activitypub-migrate', 'db', 'ghost', 'tinybird-deploy', 'tinybird-login', 'tinybird-sync', 'traffic-analytics'] }, + { profiles: 'production', mode: 'production', services: ['caddy', 'db', 'ghost'] }, + { profiles: 'production,analytics', mode: 'production', services: ['caddy', 'db', 'ghost', 'tinybird-deploy', 'tinybird-login', 'tinybird-sync', 'traffic-analytics'] }, + { profiles: 'production,activitypub', mode: 'production', services: ['activitypub', 'activitypub-migrate', 'caddy', 'db', 'ghost'] }, + { profiles: 'production,analytics,activitypub', mode: 'production', services: ['activitypub', 'activitypub-migrate', 'caddy', 'db', 'ghost', 'tinybird-deploy', 'tinybird-login', 'tinybird-sync', 'traffic-analytics'] }, +]; + +// Edge-case literals, to prove the whole path from .env into the resolved +// service configuration. +const APP_PASSWORD = 'app-p$ss"word'; +const ROOT_PASSWORD = 'root-p@ss word'; +const SMTP_PASSWORD = 'smtp-p$ss'; + +// `docker compose config` output is itself a compose file, so it re-escapes a +// literal `$` as `$$`. Container fidelity is proven in env-compose.test.mjs and +// ingress.test.mjs, which read the value back out of a running container. +const asComposeConfigEscapes = (value) => value.replaceAll('$', () => '$$'); + +describe('compose mode matrix', { skip: dockerAvailable() ? false : 'docker is not available' }, () => { + let dir; + let site; + + before(() => { + dir = tempDir('matrix'); + site = makeSite(dir); + // Application configuration, the only env_file of the ghost service. + const ghostEnv = join(site, 'ghost.env'); + writeFileSync(ghostEnv, '', { mode: 0o600 }); + chmodSync(ghostEnv, 0o600); + shOk(`env_set ${q(ghostEnv)} mail__transport SMTP`); + shOk(`env_set ${q(ghostEnv)} mail__options__auth__pass ${q(SMTP_PASSWORD)}`); + }); + after(() => cleanup(dir)); + + const setup = ({ profiles, mode }) => + writeEnv(join(site, '.env'), { + COMPOSE_PROFILES: profiles, + SITE_MODE: mode, + COMPOSE_PROJECT_NAME: 'ghost-example-com', + PROJECT_DIR: site, + GHOST_IMAGE: 'ghost', + GHOST_VERSION: '6-next-alpine', + GHOST_PORT: '2368', + DATABASE_HOST: 'db', + DATABASE_PORT: '3306', + DATABASE_NAME: 'ghost', + DATABASE_USER: 'ghost', + DATABASE_PASSWORD: APP_PASSWORD, + DATABASE_ROOT_PASSWORD: ROOT_PASSWORD, + UPLOAD_LOCATION: './data/ghost', + MYSQL_DATA_LOCATION: './data/mysql', + ...(mode === 'production' + ? { NODE_ENV: 'production', URL: 'https://example.com', DOMAIN: 'example.com', RESTART_POLICY: 'unless-stopped' } + : { NODE_ENV: 'development', URL: 'http://localhost:2368', RESTART_POLICY: 'no' }), + }); + + for (const { label, bin } of composeBinaries()) { + describe(label, () => { + for (const entry of MATRIX) { + describe(entry.profiles, () => { + let config; + let ghost; + + before(() => { + setup(entry); + config = composeConfig(site, { bin }); + ghost = config.services.ghost; + }); + + test('enables exactly the expected services', () => { + assert.deepEqual(Object.keys(config.services).sort(), entry.services); + }); + + test('Ghost receives no infrastructure root credentials', () => { + const serialized = JSON.stringify(ghost); + assert.doesNotMatch(serialized, /root-p@ss word/); + assert.doesNotMatch(serialized, /DATABASE_ROOT_PASSWORD/); + assert.doesNotMatch(serialized, /MYSQL_ROOT_PASSWORD/); + }); + + test('Ghost receives no operator-only settings', () => { + for (const key of ['COMPOSE_PROFILES', 'COMPOSE_PROJECT_NAME', 'PROJECT_DIR', + 'RESTART_POLICY', 'UPLOAD_LOCATION', 'MYSQL_DATA_LOCATION', 'LOG_MAX_SIZE', 'SITE_MODE']) { + assert.equal(ghost.environment[key], undefined, `${key} reached Ghost`); + } + }); + + test('ghost.env reaches Ghost and nothing else', () => { + assert.equal(ghost.environment.mail__transport, 'SMTP'); + assert.equal( + ghost.environment.mail__options__auth__pass, + asComposeConfigEscapes(SMTP_PASSWORD), + ); + assert.equal(config.services.db.environment.mail__transport, undefined); + }); + + test('the application database password reaches Ghost', () => { + assert.equal( + ghost.environment.database__connection__password, + asComposeConfigEscapes(APP_PASSWORD), + ); + }); + + test('the database connection is fully parameterized', () => { + assert.equal(ghost.environment.database__client, 'mysql'); + assert.equal(ghost.environment.database__connection__host, 'db'); + assert.equal(ghost.environment.database__connection__port, '3306'); + assert.equal(ghost.environment.database__connection__user, 'ghost'); + assert.equal(ghost.environment.database__connection__database, 'ghost'); + }); + + test('services have unique network aliases', () => { + const aliases = (name) => config.services[name]?.networks?.ghost_network?.aliases ?? []; + assert.ok(aliases('ghost').includes('ghost-ghost-example-com')); + assert.ok(aliases('db').includes('db-ghost-example-com')); + if (entry.services.includes('activitypub')) { + assert.ok(aliases('activitypub').includes('activitypub-ghost-example-com')); + } + if (entry.services.includes('traffic-analytics')) { + assert.ok(aliases('traffic-analytics').includes('traffic-analytics-ghost-example-com')); + } + }); + + test('Ghost publishes on the loopback interface only', () => { + assert.deepEqual(ghost.ports.map((p) => p.host_ip), ['127.0.0.1']); + assert.equal(ghost.ports[0].target, 2368); + }); + + test('Ghost has a real readiness probe', () => { + assert.ok(ghost.healthcheck?.test?.length, 'no healthcheck'); + assert.ok(ghost.healthcheck.start_interval, 'no start_interval'); + }); + + test('long-running services use the site restart policy', () => { + const expected = entry.mode === 'local' ? 'no' : 'unless-stopped'; + for (const name of LONG_RUNNING.filter((n) => entry.services.includes(n))) { + assert.equal(config.services[name].restart, expected, name); + } + }); + + test('one-shot jobs stay one-shot', () => { + for (const name of ONE_SHOT.filter((n) => entry.services.includes(n))) { + assert.equal(config.services[name].restart, 'no', name); + } + }); + + test('every service caps its logs and carries site labels', () => { + for (const [name, service] of Object.entries(config.services)) { + assert.ok(service.logging?.options?.['max-size'], `${name} has no log cap`); + assert.ok(service.logging?.options?.['max-file'], `${name} has no log file limit`); + assert.equal(service.labels['org.ghost.docker.site'], 'ghost-example-com', name); + assert.equal(service.labels['org.ghost.docker.mode'], entry.mode, name); + assert.equal( + service.labels['org.ghost.docker.lifecycle'], + ONE_SHOT.includes(name) ? 'one-shot' : 'long-running', + name, + ); + } + }); + }); + } + }); + } + + test('the IPv6 override loads through the helper file contract', () => { + setup(MATRIX.find((m) => m.profiles === 'production')); + const result = sh(`compose_run ${q(site)} config --format json`, { + env: { GD_COMPOSE_OVERRIDES: 'compose.ipv6.yml' }, + }); + assert.equal(result.status, 0, result.stderr.toString()); + const config = JSON.parse(result.stdout.toString()); + assert.equal(config.networks.ghost_network.enable_ipv6, true); + }); + + test('an unset COMPOSE_FILE cannot change the helper file list', () => { + setup(MATRIX.find((m) => m.profiles === 'production')); + const result = sh(`compose_run ${q(site)} config --services`, { + env: { COMPOSE_FILE: '/nonexistent/compose.yml' }, + }); + assert.equal(result.status, 0, result.stderr.toString()); + }); + + test('URL is required in every supported mode', () => { + setup(MATRIX.find((m) => m.profiles === 'production')); + shOk(`env_unset ${q(join(site, '.env'))} URL`); + const result = compose(site, ['config']); + assert.notEqual(result.status, 0); + assert.match(result.stderr, /URL is required/); + }); + + test('DOMAIN is not guarded, so local mode is unaffected by it', () => { + setup(MATRIX.find((m) => m.profiles === 'local')); + const result = compose(site, ['config']); + assert.equal(result.status, 0, result.stderr); + }); +}); diff --git a/tests/config.test.mjs b/tests/config.test.mjs new file mode 100644 index 00000000..bb24d42c --- /dev/null +++ b/tests/config.test.mjs @@ -0,0 +1,245 @@ +// The application/operator configuration split and its validation rules. +import { test, describe, before, after, beforeEach } from 'node:test'; +import assert from 'node:assert/strict'; +import { mkdirSync, appendFileSync, writeFileSync, readFileSync, rmSync, chmodSync } from 'node:fs'; +import { join } from 'node:path'; +import { tempDir, cleanup, makeSite, sh, shOk, shSucceeds, writeEnv, q, REPO_DIR } from './helpers.mjs'; + +const productionEnv = (site) => ({ + PROJECT_DIR: site, + COMPOSE_PROFILES: 'production', + SITE_MODE: 'production', + COMPOSE_PROJECT_NAME: 'ghost-example-com', + NODE_ENV: 'production', + URL: 'https://example.com', + DOMAIN: 'example.com', + RESTART_POLICY: 'unless-stopped', + GHOST_VERSION: '6-next-alpine', + DATABASE_HOST: 'db', + DATABASE_NAME: 'ghost', + DATABASE_USER: 'ghost', + DATABASE_PASSWORD: 'app-password', + DATABASE_ROOT_PASSWORD: 'root-password', +}); + +describe('site mode selection', () => { + test('exactly one site mode is required', () => { + assert.equal(shOk(`compose_site_mode production`).trim(), 'production'); + assert.equal(shOk(`compose_site_mode local,analytics,activitypub`).trim(), 'local'); + assert.ok(!shSucceeds(`compose_site_mode analytics`), 'no site mode'); + assert.ok(!shSucceeds(`compose_site_mode local,production`), 'two site modes'); + }); + + test('unknown profiles are reported, reserved ones are not', () => { + assert.equal(shOk(`compose_unknown_profiles production,analytics,bogus`).trim(), 'bogus'); + assert.equal(shOk(`compose_unknown_profiles production,supervisor`).trim(), ''); + }); + + test('one-shot services are the ones that must keep restart: "no"', () => { + assert.equal( + shOk('printf %s "${GD_ONE_SHOT_SERVICES[*]}"'), + 'activitypub-migrate tinybird-login tinybird-sync tinybird-deploy', + ); + }); +}); + +describe('config_validate_env', () => { + let dir; + let site; + let envFile; + + before(() => { + dir = tempDir('config'); + site = join(dir, 'site'); + mkdirSync(site); + envFile = join(site, '.env'); + }); + after(() => cleanup(dir)); + beforeEach(() => writeEnv(envFile, productionEnv(site))); + + const validate = () => sh(`config_validate_env ${q(site)}`); + const expectRejected = (why) => { + const result = validate(); + assert.notEqual(result.status, 0, `expected rejection: ${why}`); + return result.stdout.toString(); + }; + + test('a complete production .env validates', () => { + const result = validate(); + assert.equal(result.status, 0, result.stdout.toString()); + }); + + test('Compose guards URL itself, so validation does not duplicate it', () => { + // In production, URL is still cross-checked against DOMAIN. In local mode + // there is nothing to cross-check, so a missing URL is left to Compose's + // own `:?` guard rather than reported twice. + writeEnv(envFile, { + ...productionEnv(site), + DOMAIN: undefined, + URL: undefined, + COMPOSE_PROFILES: 'local', + SITE_MODE: 'local', + RESTART_POLICY: 'no', + }); + assert.equal(validate().status, 0, validate().stdout.toString()); + }); + + test('production requires DOMAIN', () => { + shOk(`env_unset ${q(envFile)} DOMAIN`); + assert.match(expectRejected('missing DOMAIN'), /DOMAIN is required/); + }); + + + + + // URL scheme, port format and restart policy are left to Compose and Docker, + // which reject them with clear errors of their own. What follows is only + // what nothing else catches. + test('URL and DOMAIN must agree', () => { + shOk(`env_set ${q(envFile)} DOMAIN other.example.com`); + assert.match(expectRejected('mismatched domain'), /disagree/); + }); + + + + + test('an ActivityPub database that is never provisioned is rejected', () => { + shOk(`env_set ${q(envFile)} ACTIVITYPUB_DATABASE_NAME ap_custom`); + assert.match(expectRejected('unprovisioned database'), /DATABASE_EXTRA_DATABASES/); + shOk(`env_set ${q(envFile)} DATABASE_EXTRA_DATABASES ap_custom`); + assert.equal(validate().status, 0); + }); + + test('the image layout and the configured content path must agree', () => { + // Derived from the image's own GHOST_CONTENT, not from the tag name, so a + // future layout change is caught without updating a mapping. Skipped when + // the images are not present locally. + const probe = sh(`config_image_content_path ghost:6-next-alpine`); + if (probe.status !== 0) return; // image not pulled + + shOk(`env_set ${q(envFile)} GHOST_VERSION 6-next-alpine`); + shOk(`env_set ${q(envFile)} GHOST_CONTENT_PATH /var/lib/ghost/content`); + assert.match(expectRejected('old path with next image'), /GHOST_CONTENT_PATH is/); + + shOk(`env_set ${q(envFile)} GHOST_CONTENT_PATH ${probe.stdout.toString().trim()}`); + assert.equal(validate().status, 0, validate().stdout.toString()); + }); + + test('a value Compose would interpolate by accident is rejected', () => { + appendFileSync(envFile, 'INTERPOLATED=costs $5\n'); + assert.match(expectRejected('unescaped dollar'), /interpolated by Compose/); + }); + + test('SITE_MODE must match COMPOSE_PROFILES', () => { + shOk(`env_set ${q(envFile)} SITE_MODE local`); + assert.match(expectRejected('mode mismatch'), /does not match/); + }); + + test('a local site is valid with no DOMAIN', () => { + writeEnv(envFile, { + ...productionEnv(site), + DOMAIN: undefined, + COMPOSE_PROFILES: 'local', + SITE_MODE: 'local', + NODE_ENV: 'development', + URL: 'http://localhost:2368', + GHOST_PORT: '2368', + RESTART_POLICY: 'no', + }); + const result = validate(); + assert.equal(result.status, 0, result.stdout.toString()); + }); +}); + +describe('config_validate_ghost_env', () => { + let dir; + let site; + let ghostEnv; + + before(() => { + dir = tempDir('ghost-env'); + // A real site: the container-owned check resolves the actual Compose + // configuration rather than consulting a hardcoded list. + site = makeSite(dir); + writeEnv(join(site, '.env'), productionEnv(site)); + ghostEnv = join(site, 'ghost.env'); + }); + after(() => cleanup(dir)); + beforeEach(() => { + writeFileSync(ghostEnv, '', { mode: 0o600 }); + chmodSync(ghostEnv, 0o600); + shOk(`env_set ${q(ghostEnv)} mail__transport SMTP`); + shOk(`env_set ${q(ghostEnv)} labs__publicAPI true`); + }); + + const validate = () => sh(`config_validate_ghost_env ${q(site)}`); + + test('application settings are accepted', () => { + assert.equal(validate().status, 0, validate().stdout.toString()); + }); + + for (const key of ['url', 'admin__url', 'database__connection__host', 'server__port', 'NODE_ENV']) { + test(`the container-owned key ${key} is rejected`, () => { + shOk(`env_set ${q(ghostEnv)} ${q(key)} anything`); + const result = validate(); + assert.notEqual(result.status, 0); + assert.match(result.stdout.toString(), /set by the container/); + }); + } + + for (const key of ['DATABASE_ROOT_PASSWORD', 'COMPOSE_PROFILES', 'PROJECT_DIR']) { + test(`the operator-only key ${key} is rejected`, () => { + shOk(`env_set ${q(ghostEnv)} ${q(key)} anything`); + const result = validate(); + assert.notEqual(result.status, 0); + assert.match(result.stdout.toString(), /operator setting/); + }); + } + + test('a key added to compose.yml is caught with no list to update', () => { + // The whole reason this is derived rather than listed: add an environment + // entry to the ghost service and validation must notice immediately. + const composeFile = join(site, 'compose.yml'); + const original = readFileSync(composeFile, 'utf8'); + try { + writeFileSync( + composeFile, + original.replace(' database__client: mysql', ' database__client: mysql\n brand__new__key: owned-by-container'), + ); + shOk(`env_set ${q(ghostEnv)} brand__new__key mine`); + const result = validate(); + assert.notEqual(result.status, 0); + assert.match(result.stdout.toString(), /brand__new__key is set by the container \(owned-by-container\)/); + } finally { + writeFileSync(composeFile, original); + } + }); + + test('ghost.env is optional', () => { + rmSync(ghostEnv); + assert.equal(validate().status, 0); + }); +}); + +describe('scripts/config.sh set', () => { + test('logs the key name and never the value', () => { + const dir = tempDir('secret-log'); + try { + const file = join(dir, 'ghost.env'); + writeFileSync(file, '', { mode: 0o600 }); + // The helpers use bash features (arrays, process substitution), so the + // CLI must be run with its own shebang, not forced through sh. + const result = sh( + `${q(join(REPO_DIR, 'scripts/config.sh'))} set ${q(file)} mail__options__auth__pass hunter2`, + ); + assert.equal(result.status, 0, result.stderr.toString()); + const logged = result.stdout.toString() + result.stderr.toString(); + assert.match(logged, /mail__options__auth__pass/); + assert.doesNotMatch(logged, /hunter2/, 'a value was logged'); + // The value still reached the file. + assert.equal(readFileSync(file, 'utf8').trim(), 'mail__options__auth__pass="hunter2"'); + } finally { + cleanup(dir); + } + }); +}); diff --git a/tests/env-compose.test.mjs b/tests/env-compose.test.mjs new file mode 100644 index 00000000..3a331712 --- /dev/null +++ b/tests/env-compose.test.mjs @@ -0,0 +1,120 @@ +// Round trips edge-case values through Docker Compose into a real container. +// +// Comparing serializer output against expected strings is not enough: Compose +// interpolates env_file values, so the only proof is what the container sees. +import { test, describe, before, after } from 'node:test'; +import assert from 'node:assert/strict'; +import { writeFileSync } from 'node:fs'; +import { join } from 'node:path'; +import { tempDir, cleanup, writeEnv, compose, dockerAvailable } from './helpers.mjs'; + +const PROBE_IMAGE = process.env.GD_TEST_PROBE_IMAGE ?? 'alpine:3.20'; + +const VALUES = [ + 'plain', + 'with spaces ', + 'dollar $VAR', + 'braced ${VAR}', + 'double dollar $$', + 'lone dollar $', + 'double"quote', + "single'quote", + 'back\\slash', + "backslash quote \\'", + '', + '["a", "b"]', + '{"k": "v", "n": [1, 2]}', + 'hash # not a comment', + 'line1\nline2', + 'tab\there', + 'trailing backslash \\', + '-----BEGIN KEY-----\nabc/def+gh==\n-----END KEY-----', +]; + +describe('Compose round trip', { skip: dockerAvailable() ? false : 'docker is not available' }, () => { + let dir; + let seen; + + before(() => { + dir = tempDir('env-compose'); + + // A variable that must NOT leak into any value through interpolation. + process.env.VAR = 'INTERPOLATED'; + + const appEnv = join(dir, 'app.env'); + writeEnv(appEnv, Object.fromEntries(VALUES.map((v, i) => [`V${i}`, v]))); + + // The probe script is mounted rather than inlined, so Compose never + // interpolates the shell syntax that reads the values back. + writeFileSync( + join(dir, 'probe.sh'), + ['#!/bin/sh', 'i=0', 'while [ "$i" -lt "$COUNT" ]; do', + ' eval "v=\\${V$i}"', + " printf '%s' \"$v\" | base64 | tr -d '\\n'", + " printf '\\n'", + ' i=$((i + 1))', 'done', ''].join('\n'), + { mode: 0o755 }, + ); + + writeFileSync( + join(dir, 'compose.yml'), + [ + 'services:', + ' probe:', + ' image: ${GD_PROBE_IMAGE}', + ' env_file:', + ' - path: app.env', + ' required: true', + ' environment:', + ' COUNT: ${COUNT}', + ' volumes:', + ' - ./probe.sh:/probe.sh:ro', + ' command: ["sh", "/probe.sh"]', + '', + ].join('\n'), + ); + + // COUNT and the image come through Compose interpolation of `.env`, which + // exercises the `.env` side of the same contract. + writeEnv(join(dir, '.env'), { COUNT: String(VALUES.length), GD_PROBE_IMAGE: PROBE_IMAGE }); + + const result = compose(dir, ['run', '--rm', '--no-deps', '-T', 'probe']); + assert.equal(result.status, 0, result.stderr); + seen = result.stdout.trim().split('\n').map((line) => Buffer.from(line.trim(), 'base64').toString()); + }); + + after(() => { + compose(dir, ['down', '-v', '--remove-orphans']); + cleanup(dir); + }); + + VALUES.forEach((value, index) => { + test(`the container sees value ${index} verbatim: ${JSON.stringify(value)}`, () => { + assert.equal(seen[index], value); + }); + }); + + test('a .env value survives Compose interpolation into a service', () => { + const tricky = `tricky $VAR \${VAR} "quoted" 'single' back\\slash # hash`; + writeFileSync( + join(dir, 'interp.yml'), + [ + 'services:', + ' probe:', + ' image: ${GD_PROBE_IMAGE}', + ' environment:', + ' ECHOED: ${TRICKY}', + ' command: ["sh", "-c", "printf \'%s\' \\"$$ECHOED\\" | base64 | tr -d \'\\\\n\'"]', + '', + ].join('\n'), + ); + writeEnv(join(dir, '.env'), { + COUNT: String(VALUES.length), + GD_PROBE_IMAGE: PROBE_IMAGE, + TRICKY: tricky, + }); + const result = compose(dir, ['-f', join(dir, 'interp.yml'), 'run', '--rm', '--no-deps', '-T', 'probe']); + assert.equal(result.status, 0, result.stderr); + assert.equal(Buffer.from(result.stdout.trim(), 'base64').toString(), tricky); + }); +}); diff --git a/tests/env.test.mjs b/tests/env.test.mjs new file mode 100644 index 00000000..355b5012 --- /dev/null +++ b/tests/env.test.mjs @@ -0,0 +1,221 @@ +// The dotenv serializer and parser in scripts/lib/env.sh. +// +// These values are exactly the ones Compose gets wrong if the encoding is +// naive: interpolation markers, quotes, backslashes and newlines. +import { test, describe, before, after } from 'node:test'; +import assert from 'node:assert/strict'; +import { writeFileSync, readFileSync, existsSync } from 'node:fs'; +import { join } from 'node:path'; +import { tempDir, cleanup, sh, shOk, shSucceeds, shValue, q } from './helpers.mjs'; + +const VALUES = { + plain: 'plain', + padded: ' leading and trailing ', + dollar: 'dollar $VAR and ${VAR} and $$ and $', + doubleQuote: 'double"quote', + singleQuote: "single'quote", + backslash: 'back\\slash', + backslashBeforeQuote: "backslash before quote: \\'", + backslashBeforeDouble: 'backslash before double: \\"', + empty: '', + jsonArray: '["a", "b", 1, null]', + jsonObject: '{"nested": {"k": "v"}}', + hash: 'hash # not a comment', + multiline: 'line1\nline2\nline3', + tab: 'tab\tseparated', + unicode: 'unicode: héllo — ✓', + trailingBackslash: 'trailing backslash \\', + pem: '-----BEGIN KEY-----\nabc/def+gh==\n-----END KEY-----', +}; + +describe('env.sh', () => { + let dir; + let file; + + before(() => { + dir = tempDir('env'); + file = join(dir, 'round.env'); + writeFileSync(file, ''); + for (const [key, value] of Object.entries(VALUES)) { + shOk(`env_set ${q(file)} ${q(key)} ${q(value)}`); + } + }); + after(() => cleanup(dir)); + + for (const [key, value] of Object.entries(VALUES)) { + test(`round trips ${key}`, () => { + assert.equal(shValue(`env_get ${q(file)} ${q(key)}`).replace(/\n$/, ''), value); + }); + } + + test('writes one line per key', () => { + const lines = readFileSync(file, 'utf8').trimEnd().split('\n'); + assert.equal(lines.length, Object.keys(VALUES).length); + assert.ok(lines.every((l) => /^[A-Za-z_][A-Za-z0-9_]*="/.test(l))); + }); + + test('lists every key once, in order', () => { + const keys = shOk(`env_keys ${q(file)}`).trim().split('\n'); + assert.deepEqual(keys, Object.keys(VALUES)); + }); + + test('overwrites in place without duplicating', () => { + shOk(`env_set ${q(file)} plain replaced`); + assert.equal(shValue(`env_get ${q(file)} plain`).trim(), 'replaced'); + assert.equal(shOk(`env_keys ${q(file)}`).trim().split('\n').length, Object.keys(VALUES).length); + assert.equal(shOk(`env_keys ${q(file)}`).trim().split('\n')[0], 'plain'); + }); + + test('preserves comments and blank lines around an edit', () => { + const commented = join(dir, 'comments.env'); + writeFileSync(commented, '# a leading comment\nA="one"\n\n# a comment about B\nB="two"\n'); + shOk(`env_set ${q(commented)} B changed`); + const text = readFileSync(commented, 'utf8'); + assert.match(text, /# a comment about B/); + assert.equal(shValue(`env_get ${q(commented)} B`).trim(), 'changed'); + + shOk(`env_unset ${q(commented)} A`); + assert.ok(!shSucceeds(`env_get ${q(commented)} A`)); + assert.match(readFileSync(commented, 'utf8'), /B="changed"/); + }); + + describe('reading formats it did not write', () => { + let foreign; + before(() => { + foreign = join(dir, 'foreign.env'); + writeFileSync( + foreign, + [ + 'UNQUOTED=hello world', + 'UNQUOTED_COMMENT=value # trailing comment', + "SINGLE='literal $NOPE'", + "SINGLE_ESC='it\\'s here'", + 'DOUBLE="escaped \\$LITERAL"', + 'export EXPORTED="yes"', + 'MULTILINE="first\nsecond"', + 'DUP="one"', + 'DUP="two"', + ].join('\n') + '\n', + ); + }); + + const cases = { + UNQUOTED: 'hello world', + UNQUOTED_COMMENT: 'value', + // Single quotes are literal: no interpolation, only \' is an escape. + SINGLE: 'literal $NOPE', + SINGLE_ESC: "it's here", + // Double quotes interpolate, so \$ and $$ both mean a literal dollar. + DOUBLE: 'escaped $LITERAL', + EXPORTED: 'yes', + // The last assignment wins, matching Compose. + DUP: 'two', + }; + + for (const [key, expected] of Object.entries(cases)) { + test(key, () => { + assert.equal(shValue(`env_get ${q(foreign)} ${key}`).replace(/\n$/, ''), expected); + }); + } + }); + + describe('values spanning several lines', () => { + // Valid dotenv, never written by these helpers, and not editable through + // them. What matters is that such a file still parses cleanly around the + // multi-line value rather than being corrupted or mis-read. + let pemFile; + before(() => { + pemFile = join(dir, 'pem.env'); + writeFileSync( + pemFile, + [ + 'mail__transport="SMTP"', + 'TLS_KEY="-----BEGIN KEY-----', + 'MIIEvQIBADANBg==', + '-----END KEY-----"', + 'labs__publicAPI="true"', + '', + ].join('\n'), + ); + }); + + test('keys around it are still listed, and its body is not mistaken for one', () => { + assert.deepEqual(shOk(`env_keys ${q(pemFile)}`).trim().split('\n'), [ + 'mail__transport', + 'TLS_KEY', + 'labs__publicAPI', + ]); + }); + + test('reading it fails with an actionable message', () => { + const result = sh(`env_get ${q(pemFile)} TLS_KEY`); + assert.notEqual(result.status, 0); + assert.match(result.stderr.toString(), /spans several lines; edit it by hand/); + }); + + test('writing it is refused rather than corrupting the file', () => { + const before = readFileSync(pemFile, 'utf8'); + const result = sh(`env_set ${q(pemFile)} TLS_KEY replaced`); + assert.notEqual(result.status, 0); + assert.match(result.stderr.toString(), /spans several lines/); + assert.equal(readFileSync(pemFile, 'utf8'), before, 'the file was modified'); + }); + + test('other keys in the same file are still editable', () => { + shOk(`env_set ${q(pemFile)} mail__transport Direct`); + assert.equal(shValue(`env_get ${q(pemFile)} mail__transport`).trim(), 'Direct'); + assert.match(readFileSync(pemFile, 'utf8'), /MIIEvQIBADANBg==/); + }); + + test('it does not trip the interpolation lint', () => { + assert.ok(shSucceeds(`env_lint ${q(pemFile)}`)); + }); + }); + + test('distinguishes a missing key from an empty value', () => { + shOk(`env_set ${q(file)} PRESENT_BUT_EMPTY ''`); + assert.ok(!shSucceeds(`env_get ${q(file)} NOPE`), 'a missing key must fail'); + assert.ok(shSucceeds(`env_get ${q(file)} PRESENT_BUT_EMPTY`), 'an empty value must succeed'); + assert.equal(shValue(`env_get ${q(file)} PRESENT_BUT_EMPTY`), '\n'); + shOk(`env_unset ${q(file)} PRESENT_BUT_EMPTY`); + }); + + test('rejects an invalid key', () => { + // Through the public API: the serializer is internal until S2 needs it. + assert.ok(!shSucceeds(`env_set ${q(file)} 'bad-key' value`)); + assert.ok(!shSucceeds(`env_get ${q(file)} 'bad-key'`)); + }); + + test('never evaluates an env file', () => { + const evil = join(dir, 'evil.env'); + const marker = join(dir, 'pwned'); + writeFileSync(evil, `EVIL="$(touch ${marker})"\nBACKTICK=\`touch ${marker}\`\n`); + sh(`env_get ${q(evil)} EVIL; env_keys ${q(evil)}; env_set ${q(evil)} OTHER value`); + assert.ok(!existsSync(marker), 'command substitution in an env file was executed'); + }); + + describe('env_lint', () => { + let lintFile; + before(() => { + lintFile = join(dir, 'lint.env'); + writeFileSync( + lintFile, + ['GOOD="$$literal"', "ALSO_GOOD='$literal'", 'BAD="costs $5"', 'BAD_UNQUOTED=costs $5'].join('\n') + '\n', + ); + }); + + test('flags values Compose would interpolate', () => { + const result = sh(`env_lint ${q(lintFile)}`); + assert.notEqual(result.status, 0); + const out = result.stdout.toString(); + assert.match(out, /^BAD:/m); + assert.match(out, /^BAD_UNQUOTED:/m); + assert.doesNotMatch(out, /^GOOD:/m); + assert.doesNotMatch(out, /^ALSO_GOOD:/m); + }); + + test('passes on generated files', () => { + assert.ok(shSucceeds(`env_lint ${q(file)}`)); + }); + }); +}); diff --git a/tests/helpers.mjs b/tests/helpers.mjs new file mode 100644 index 00000000..367f7f64 --- /dev/null +++ b/tests/helpers.mjs @@ -0,0 +1,177 @@ +import { execFileSync, execFile, spawnSync } from 'node:child_process'; +import { promisify } from 'node:util'; +import { mkdtempSync, mkdirSync, rmSync, cpSync, writeFileSync, chmodSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join, dirname } from 'node:path'; +import { fileURLToPath } from 'node:url'; + +const execFileAsync = promisify(execFile); + +export const TESTS_DIR = dirname(fileURLToPath(import.meta.url)); +export const REPO_DIR = join(TESTS_DIR, '..'); + +const LIBS = ['fs', 'env', 'compose', 'config', 'caddy']; + +/** + * Run a bash snippet with every ghost-docker library sourced. + * + * The libraries under test are shell, so they are exercised through a real + * shell. Everything around them — building fixtures, parsing structured + * output, asserting — is JavaScript. + * + * Returns { stdout, stderr, status }. Throws only if the shell itself cannot + * be started. + */ +export function sh(script, { cwd = REPO_DIR, env = {}, input } = {}) { + const preamble = LIBS.map((l) => `. "${REPO_DIR}/scripts/lib/${l}.sh"`).join('\n'); + const result = spawnSync('bash', ['-c', `${preamble}\n${script}`], { + cwd, + input, + env: { ...process.env, ...env }, + }); + if (result.error) throw result.error; + return { + stdout: result.stdout ?? Buffer.alloc(0), + stderr: result.stderr ?? Buffer.alloc(0), + status: result.status, + }; +} + +/** Run a shell snippet and return its trimmed stdout, failing the call on a non-zero exit. */ +export function shOk(script, options) { + const result = sh(script, options); + if (result.status !== 0) { + throw new Error(`shell exited ${result.status}: ${result.stderr.toString()}${result.stdout.toString()}`); + } + return result.stdout.toString(); +} + +/** True when the snippet exits zero. */ +export function shSucceeds(script, options) { + return sh(script, options).status === 0; +} + +/** Shell-quote a value for single-quoted embedding. */ +export function q(value) { + return `'${String(value).replaceAll("'", `'\\''`)}'`; +} + +/** + * Read a value back byte-exactly, including trailing newlines, by having the + * shell base64 it. `$(...)` would strip trailing newlines. + */ +export function shValue(script, options) { + const out = shOk(`{ ${script} ; } | base64 | tr -d '\\n'`, options); + return Buffer.from(out.trim(), 'base64').toString(); +} + +export function tempDir(prefix = 'ghost-docker-test') { + const dir = mkdtempSync(join(tmpdir(), `${prefix}-`)); + return dir; +} + +export function cleanup(dir) { + rmSync(dir, { recursive: true, force: true }); +} + +/** Copy the pieces of the repo a site needs into a scratch directory. */ +export function makeSite(dir, { withGhostEnv = false } = {}) { + const site = join(dir, 'site'); + cpSync(join(REPO_DIR, 'compose.yml'), join(site, 'compose.yml'), { recursive: true }); + cpSync(join(REPO_DIR, 'compose.ipv6.yml'), join(site, 'compose.ipv6.yml')); + for (const sub of ['caddy', 'mysql-init', 'tinybird']) { + cpSync(join(REPO_DIR, sub), join(site, sub), { recursive: true }); + } + // Start with no generated routes, whatever the developer's tree contains. + rmSync(join(site, 'caddy', 'sites'), { recursive: true, force: true }); + mkdirSync(join(site, 'caddy', 'sites'), { recursive: true }); + if (withGhostEnv) { + writeFileSync(join(site, 'ghost.env'), '', { mode: 0o600 }); + } + return site; +} + +/** Write a `.env` from a plain object, using the library's own serializer. */ +export function writeEnv(file, values) { + writeFileSync(file, '', { mode: 0o600 }); + chmodSync(file, 0o600); + for (const [key, value] of Object.entries(values)) { + if (value === undefined) continue; + shOk(`env_set ${q(file)} ${q(key)} ${q(value)}`); + } +} + +/** Run `docker compose` for a site directory. `bin` may be a path to another Compose. */ +export function compose(site, args, { bin, env = {} } = {}) { + const base = ['--project-directory', site, '-f', join(site, 'compose.yml')]; + const command = bin ?? 'docker'; + const argv = bin ? [...base, ...args] : ['compose', ...base, ...args]; + try { + const stdout = execFileSync(command, argv, { + encoding: 'utf8', + env: { ...process.env, ...env, COMPOSE_FILE: '' }, + stdio: ['pipe', 'pipe', 'pipe'], + }); + return { stdout, stderr: '', status: 0 }; + } catch (error) { + if (error.status === undefined) throw error; + return { stdout: error.stdout ?? '', stderr: error.stderr ?? '', status: error.status }; + } +} + +/** The fully resolved Compose project, as Compose itself sees it. */ +export function composeConfig(site, options) { + const result = compose(site, ['config', '--format', 'json'], options); + if (result.status !== 0) { + throw new Error(`docker compose config failed: ${result.stderr}`); + } + return JSON.parse(result.stdout); +} + +export async function composeAsync(site, args, options = {}) { + const base = ['compose', '--project-directory', site, '-f', join(site, 'compose.yml')]; + return execFileAsync('docker', [...base, ...args], { + encoding: 'utf8', + env: { ...process.env, ...options.env, COMPOSE_FILE: '' }, + maxBuffer: 32 * 1024 * 1024, + }); +} + +export function dockerAvailable() { + try { + execFileSync('docker', ['info'], { stdio: 'ignore' }); + return true; + } catch { + return false; + } +} + +export function dockerInspect(id, format) { + return execFileSync('docker', ['inspect', '-f', format, id], { encoding: 'utf8' }).trim(); +} + +export const sleep = (ms) => new Promise((resolve) => setTimeout(resolve, ms)); + +/** Poll until `check()` resolves truthy, or the deadline passes. */ +export async function waitFor(check, { timeoutMs, intervalMs = 5000 } = {}) { + const deadline = Date.now() + timeoutMs; + for (;;) { + const result = await check(); + if (result) return result; + if (Date.now() > deadline) return null; + await sleep(intervalMs); + } +} + +/** The Compose binaries to run the mode matrix against. */ +export function composeBinaries() { + const bins = [{ label: `compose ${composeVersion()}`, bin: undefined }]; + const min = process.env.GD_TEST_MIN_COMPOSE; + if (min) bins.push({ label: `compose ${composeVersion(min)} (declared minimum)`, bin: min }); + return bins; +} + +function composeVersion(bin) { + const argv = bin ? ['version', '--short'] : ['compose', 'version', '--short']; + return execFileSync(bin ?? 'docker', argv, { encoding: 'utf8' }).trim().replace(/^v/, ''); +} diff --git a/tests/ingress.test.mjs b/tests/ingress.test.mjs new file mode 100644 index 00000000..a5f06e73 --- /dev/null +++ b/tests/ingress.test.mjs @@ -0,0 +1,234 @@ +// End-to-end smoke tests: a local site published on the loopback interface, a +// production site served over HTTPS by Caddy, Ghost readiness, and one-shot +// jobs that stay stopped after they complete. +// +// These start real containers and pull images. Set GD_TEST_INGRESS=1 to run +// them; they are skipped otherwise. +import { test, describe, before, after } from 'node:test'; +import assert from 'node:assert/strict'; +import { writeFileSync } from 'node:fs'; +import { join } from 'node:path'; +import { + tempDir, cleanup, makeSite, writeEnv, sh, shOk, compose, composeAsync, + dockerAvailable, dockerInspect, waitFor, sleep, q, +} from './helpers.mjs'; + +const enabled = process.env.GD_TEST_INGRESS === '1' && dockerAvailable(); +const skip = enabled ? false : 'set GD_TEST_INGRESS=1 with a working Docker daemon'; + +const GHOST_PORT = process.env.GD_TEST_GHOST_PORT ?? '23680'; +const HTTP_PORT = process.env.GD_TEST_HTTP_PORT ?? '8080'; +const HTTPS_PORT = process.env.GD_TEST_HTTPS_PORT ?? '8443'; +const PROJECT = `ghost-docker-test-${process.pid}`; + +// Edge-case characters, to prove the whole path from .env into the container. +const APP_PASSWORD = 'app p$ss"word'; +const ROOT_PASSWORD = 'root-password'; + +let dir; +let site; + +const commonEnv = () => ({ + COMPOSE_PROJECT_NAME: PROJECT, + PROJECT_DIR: site, + GHOST_IMAGE: 'ghost', + GHOST_VERSION: process.env.GD_TEST_GHOST_VERSION ?? '6-next-alpine', + GHOST_PORT, + DATABASE_HOST: 'db', + DATABASE_PORT: '3306', + DATABASE_NAME: 'ghost', + DATABASE_USER: 'ghost', + DATABASE_PASSWORD: APP_PASSWORD, + DATABASE_ROOT_PASSWORD: ROOT_PASSWORD, + UPLOAD_LOCATION: join(site, 'data', 'ghost'), + MYSQL_DATA_LOCATION: join(site, 'data', 'mysql'), +}); + +const serviceId = (name) => compose(site, ['ps', '-q', name]).stdout.trim(); + +/** Wait for a service's health check to report healthy. Fails fast if it exits. */ +const waitHealthy = (name, timeoutMs) => + waitFor(() => { + const id = serviceId(name); + if (!id) return false; + if (dockerInspect(id, '{{.State.Running}}') !== 'true') { + throw new Error(`${name} stopped while waiting for it to become healthy`); + } + return dockerInspect(id, '{{.State.Health.Status}}') === 'healthy'; + }, { timeoutMs }); + +/** Run a snippet of Node inside the Ghost container and return its stdout. */ +const inGhost = async (script) => { + const { stdout } = await composeAsync(site, ['exec', '-T', 'ghost', 'node', '-e', script]); + return stdout.trim(); +}; + +describe('ingress smoke tests', { skip, concurrency: 1 }, () => { + before(() => { + dir = tempDir('ingress'); + site = makeSite(dir); + }); + after(() => { + if (site) compose(site, ['down', '-v', '--remove-orphans']); + if (dir) cleanup(dir); + }); + + describe('local mode', () => { + before(async () => { + writeEnv(join(site, '.env'), { + ...commonEnv(), + COMPOSE_PROFILES: 'local', + SITE_MODE: 'local', + NODE_ENV: 'development', + URL: `http://localhost:${GHOST_PORT}`, + RESTART_POLICY: 'no', + }); + const result = compose(site, ['up', '-d']); + assert.equal(result.status, 0, result.stderr); + }); + after(() => compose(site, ['down', '-v', '--remove-orphans'])); + + test('the configuration validates', () => { + const result = sh(`config_validate ${q(site)}`); + assert.equal(result.status, 0, result.stdout.toString()); + }); + + test('the database becomes healthy', async () => { + assert.ok(await waitHealthy('db', 300_000), 'db never became healthy'); + }); + + test('Ghost reports ready through its readiness probe', async () => { + assert.ok(await waitHealthy('ghost', 420_000), 'ghost never became healthy'); + }); + + test('Ghost answers on the published loopback port', async () => { + const response = await fetch(`http://127.0.0.1:${GHOST_PORT}/ghost/api/admin/site/`); + assert.equal(response.status, 200); + assert.ok((await response.json()).site, 'no site payload'); + }); + + test('Ghost is published on the loopback interface only', () => { + const bindings = JSON.parse(dockerInspect(serviceId('ghost'), '{{json .HostConfig.PortBindings}}')); + const hosts = Object.values(bindings).flat().map((b) => b.HostIp); + assert.deepEqual(hosts, ['127.0.0.1']); + }); + + test('the container received the exact database password', async () => { + const encoded = await inGhost( + 'process.stdout.write(Buffer.from(process.env.database__connection__password).toString("base64"))', + ); + assert.equal(Buffer.from(encoded, 'base64').toString(), APP_PASSWORD); + }); + + test('the container received no infrastructure root credentials', async () => { + const env = await inGhost('process.stdout.write(JSON.stringify(process.env))'); + const parsed = JSON.parse(env); + assert.equal(parsed.DATABASE_ROOT_PASSWORD, undefined); + assert.equal(parsed.MYSQL_ROOT_PASSWORD, undefined); + assert.ok( + !Object.values(parsed).includes(ROOT_PASSWORD), + 'the MySQL root password reached the Ghost container', + ); + }); + }); + + describe('production mode', () => { + before(async () => { + writeEnv(join(site, '.env'), { + ...commonEnv(), + COMPOSE_PROFILES: 'production,activitypub', + SITE_MODE: 'production', + NODE_ENV: 'production', + URL: 'https://ghost.test', + DOMAIN: 'ghost.test', + RESTART_POLICY: 'unless-stopped', + HTTP_PORT, + HTTPS_PORT, + }); + // `ghost.test` is not a public name, so issue from Caddy's internal CA + // rather than attempting a real ACME order. + writeFileSync(join(site, 'caddy', 'global', 'tls.caddy'), 'local_certs\n'); + + const applied = sh(`caddy_apply ${q(site)}`); + assert.equal(applied.status, 0, applied.stderr.toString()); + + const result = compose(site, ['up', '-d']); + assert.equal(result.status, 0, result.stderr); + }); + + test('the configuration validates', () => { + const result = sh(`config_validate ${q(site)}`); + assert.equal(result.status, 0, result.stdout.toString()); + }); + + test('the database becomes healthy', async () => { + assert.ok(await waitHealthy('db', 300_000), 'db never became healthy'); + }); + + test('Ghost reports ready through its readiness probe', async () => { + assert.ok(await waitHealthy('ghost', 420_000), 'ghost never became healthy'); + }); + + test('Caddy routes the site domain and reloads explicitly', () => { + assert.ok(sh(`caddy_running ${q(site)}`).status === 0, 'caddy is not running'); + assert.equal(sh(`caddy_verify ${q(site)} ghost.test`).status, 0); + assert.equal(sh(`caddy_reload ${q(site)}`).status, 0); + }); + + test('HTTPS through Caddy reaches Ghost', async () => { + // From inside the network, with the correct SNI and Host. Trusting + // Caddy's internal CA is unnecessary for a routing check. + const out = await inGhost(` + const https = require("https"); + const req = https.request({ + host: "caddy", port: 443, servername: "ghost.test", + path: "/ghost/api/admin/site/", headers: { Host: "ghost.test" }, + rejectUnauthorized: false, timeout: 20000, + }, (res) => { + let body = ""; + res.on("data", (c) => { body += c; }); + res.on("end", () => process.stdout.write(JSON.stringify({ status: res.statusCode, body }))); + }); + req.on("error", (e) => process.stdout.write(JSON.stringify({ error: e.message }))); + req.end(); + `); + const result = JSON.parse(out); + assert.equal(result.status, 200, out); + assert.ok(JSON.parse(result.body).site, 'no site payload through Caddy'); + }); + + test('plain HTTP is redirected to HTTPS', async () => { + const out = await inGhost(` + require("http").get( + { host: "caddy", port: 80, path: "/", headers: { Host: "ghost.test" } }, + (res) => process.stdout.write(JSON.stringify({ status: res.statusCode, location: res.headers.location })), + ).on("error", (e) => process.stdout.write(JSON.stringify({ error: e.message }))); + `); + const result = JSON.parse(out); + assert.ok(result.status >= 300 && result.status < 400, out); + assert.match(result.location ?? '', /^https:\/\/ghost\.test/); + }); + + test('the ActivityPub migration job completed and stays stopped', async () => { + const id = compose(site, ['ps', '-a', '-q', 'activitypub-migrate']).stdout.trim(); + assert.ok(id, 'the one-shot migration container was not created'); + assert.equal( + dockerInspect(id, '{{.State.Status}} {{.State.ExitCode}} {{.HostConfig.RestartPolicy.Name}}'), + 'exited 0 no', + ); + await sleep(10_000); + assert.equal(dockerInspect(id, '{{.State.Status}}'), 'exited'); + }); + + test('long-running services keep the site restart policy', () => { + assert.equal(dockerInspect(serviceId('ghost'), '{{.HostConfig.RestartPolicy.Name}}'), 'unless-stopped'); + assert.equal(dockerInspect(serviceId('db'), '{{.HostConfig.RestartPolicy.Name}}'), 'unless-stopped'); + }); + + test('container logs are capped', () => { + const config = JSON.parse(dockerInspect(serviceId('ghost'), '{{json .HostConfig.LogConfig}}')); + assert.ok(config.Config['max-size']); + assert.ok(config.Config['max-file']); + }); + }); +}); diff --git a/tests/legacy-migrate.test.mjs b/tests/legacy-migrate.test.mjs new file mode 100644 index 00000000..3ba9ebea --- /dev/null +++ b/tests/legacy-migrate.test.mjs @@ -0,0 +1,68 @@ +// Guards the legacy Ghost-CLI migration path, which still ships until +// `install.sh --import` replaces it (S2/S5). +// +// This exists because adding a root package.json with `"type": "module"` for +// the test suite silently broke scripts/config-to-env.js, which is CommonJS. +// Nothing caught it: the script has no consumer other than scripts/migrate.sh. +// The package.json is gone now, but the guard stays — it would catch the same +// breakage returning, and it does not depend on the file's extension. +import { test, describe } from 'node:test'; +import assert from 'node:assert/strict'; +import { execFileSync } from 'node:child_process'; +import { writeFileSync, readFileSync } from 'node:fs'; +import { join } from 'node:path'; +import { existsSync } from 'node:fs'; +import { tempDir, cleanup, sh, shValue, q, REPO_DIR } from './helpers.mjs'; + +// Resolved from migrate.sh rather than hardcoded, so this tests the real +// invariant — the helper migrate.sh calls exists and runs — instead of one +// spelling of its filename. +const MIGRATE = readFileSync(join(REPO_DIR, 'scripts', 'migrate.sh'), 'utf8'); +const REFERENCED = [...new Set([...MIGRATE.matchAll(/scripts\/(config-to-env\.[a-z]+)/g)].map((m) => m[1]))]; +const SCRIPT = join(REPO_DIR, 'scripts', REFERENCED[0] ?? 'config-to-env.js'); + +describe('legacy migration helper', () => { + test('converts a Ghost config.json into ghost.env assignments', () => { + const dir = tempDir('legacy'); + try { + const config = join(dir, 'config.production.json'); + writeFileSync( + config, + JSON.stringify({ + url: 'https://example.com', + database: { client: 'mysql', connection: { password: 'secret' } }, + server: { port: 2368 }, + mail: { + transport: 'SMTP', + options: { secure: true, auth: { pass: 'p$ss"word' } }, + }, + labs: { publicAPI: true }, + }), + ); + + const out = execFileSync('node', [SCRIPT, config], { encoding: 'utf8' }); + const lines = out.trim().split('\n'); + + // Container-owned keys are dropped; application settings are kept. + assert.ok(!lines.some((l) => /^(url|database__|server__)/.test(l)), out); + assert.ok(lines.includes('mail__transport="SMTP"')); + assert.ok(lines.includes('labs__publicAPI="true"')); + + // A literal `$` must be written `$$`, or Compose eats it. + assert.ok(lines.includes('mail__options__auth__pass="p$$ss\\"word"'), out); + + // And the round trip through the env parser must give the value back. + const ghostEnv = join(dir, 'ghost.env'); + writeFileSync(ghostEnv, out); + assert.equal(shValue(`env_get ${q(ghostEnv)} mail__options__auth__pass`).trim(), 'p$ss"word'); + assert.ok(sh(`env_lint ${q(ghostEnv)}`).status === 0, 'generated ghost.env fails its own lint'); + } finally { + cleanup(dir); + } + }); + + test('migrate.sh calls exactly one helper, and it exists', () => { + assert.equal(REFERENCED.length, 1, `migrate.sh references ${REFERENCED.length} helpers: ${REFERENCED}`); + assert.ok(existsSync(SCRIPT), `migrate.sh calls ${REFERENCED[0]}, which does not exist`); + }); +}); From c6d640537a51a4442c9bbc3afd93b8d0c56f046e Mon Sep 17 00:00:00 2001 From: Austin Burdine Date: Thu, 3 Sep 2026 13:31:56 -0400 Subject: [PATCH 2/2] fix(fs,lint): portable stat on Linux, and shellcheck 0.9 cleanliness Both surfaced by CI on Linux (the helpers were only run on macOS before). - fs.sh: `stat -f '%Lp'` is BSD/macOS format syntax. On Linux `-f` is `--file-system`, which prints filesystem status for the file and *succeeds*, so the `|| stat -c` GNU fallback never ran and the filesystem blob became the chmod mode ("chmod: invalid mode"). Try GNU `-c` first (macOS rejects it cleanly and falls through to `-f`), fixing fs_stat_mode and _gd_stat_owner. This aborted every test that writes a config file through fs_atomic_write. - scripts/config.sh, scripts/caddy.sh: `(($#)) && shift || true` trips SC2015 on shellcheck 0.9 (Ubuntu's version); use `if (($#)); then shift; fi`. - scripts/lib/caddy.sh: drop a useless `cat` (SC2002), redirect instead. Verified with shellcheck 0.9.0 (CI's version) via koalaman/shellcheck:v0.9.0, and the stat fix reproduced against ubuntu:24.04. Co-Authored-By: Claude Opus 4.8 --- scripts/caddy.sh | 2 +- scripts/config.sh | 2 +- scripts/lib/caddy.sh | 2 +- scripts/lib/fs.sh | 10 ++++++++-- 4 files changed, 11 insertions(+), 5 deletions(-) diff --git a/scripts/caddy.sh b/scripts/caddy.sh index 5c30b10e..2e039227 100755 --- a/scripts/caddy.sh +++ b/scripts/caddy.sh @@ -13,7 +13,7 @@ set -euo pipefail . "$(dirname -- "$0")/lib/common.sh" cmd=${1:-} -(($#)) && shift || true +if (($#)); then shift; fi dir="${1:-$GD_ROOT_DIR}" case "$cmd" in diff --git a/scripts/config.sh b/scripts/config.sh index 0d04f36e..88b8d8fc 100755 --- a/scripts/config.sh +++ b/scripts/config.sh @@ -11,7 +11,7 @@ set -euo pipefail . "$(dirname -- "$0")/lib/common.sh" cmd=${1:-} -(($#)) && shift || true +if (($#)); then shift; fi case "$cmd" in validate) diff --git a/scripts/lib/caddy.sh b/scripts/lib/caddy.sh index 8a034c9c..fb7c2ba3 100644 --- a/scripts/lib/caddy.sh +++ b/scripts/lib/caddy.sh @@ -175,7 +175,7 @@ caddy_install() { for f in "$staging"/*.caddy; do [[ -e $f ]] || continue base=$(basename "$f") - cat "$f" | fs_atomic_write "$live/$base" 0644 || return 1 + fs_atomic_write "$live/$base" 0644 <"$f" || return 1 done printf '%s\n' "$backup" diff --git a/scripts/lib/fs.sh b/scripts/lib/fs.sh index 059b410c..3ac9a3f7 100644 --- a/scripts/lib/fs.sh +++ b/scripts/lib/fs.sh @@ -11,15 +11,21 @@ GD_FS_LIB_LOADED=1 # fs_stat_mode PATH -> octal mode, or non-zero when the path does not exist +# +# GNU `-c` is tried before BSD `-f`, and the order matters: on Linux, BSD's +# `stat -f '%Lp'` is `--file-system` with a bogus operand, which prints file +# system status and *succeeds*, so a `-f`-first form never reaches the GNU +# fallback and returns that blob as the mode. GNU `-c` on macOS fails cleanly +# (unknown option), so `-c`-first is correct on both. fs_stat_mode() { [[ -e $1 ]] || return 1 - stat -f '%Lp' "$1" 2>/dev/null || stat -c '%a' "$1" 2>/dev/null + stat -c '%a' "$1" 2>/dev/null || stat -f '%Lp' "$1" 2>/dev/null } # _gd_stat_owner PATH -> uid:gid _gd_stat_owner() { [[ -e $1 ]] || return 1 - stat -f '%u:%g' "$1" 2>/dev/null || stat -c '%u:%g' "$1" 2>/dev/null + stat -c '%u:%g' "$1" 2>/dev/null || stat -f '%u:%g' "$1" 2>/dev/null } # fs_mktemp_dir [PREFIX] -> path to a new private directory