From 977c65344b712ae1da8d6a7088036784c0cdf56f Mon Sep 17 00:00:00 2001 From: Simon Carr Date: Wed, 13 May 2026 15:50:18 +0100 Subject: [PATCH] feat(dev): add local full-stack dev launcher --- .github/workflows/dev-stack-check.yml | 51 +++ .gitignore | 3 + apps/docs/docs/.vuepress/config.ts | 1 + apps/docs/docs/development/dev-stack.md | 112 ++++++ apps/ingest/internal/config/config.go | 7 + apps/ingest/internal/config/config_test.go | 13 + apps/web/app/layout.tsx | 6 +- deploy/nginx/dev-stack.conf.template | 83 ++++ deploy/scripts/test-dev-stack.sh | 107 +++++ dev-stack.sh | 445 +++++++++++++++++++++ docker-compose.dev-stack.yml | 230 +++++++++++ 11 files changed, 1057 insertions(+), 1 deletion(-) create mode 100644 .github/workflows/dev-stack-check.yml create mode 100644 apps/docs/docs/development/dev-stack.md create mode 100644 deploy/nginx/dev-stack.conf.template create mode 100755 deploy/scripts/test-dev-stack.sh create mode 100755 dev-stack.sh create mode 100644 docker-compose.dev-stack.yml diff --git a/.github/workflows/dev-stack-check.yml b/.github/workflows/dev-stack-check.yml new file mode 100644 index 00000000..f1aa5e2f --- /dev/null +++ b/.github/workflows/dev-stack-check.yml @@ -0,0 +1,51 @@ +name: Dev stack check + +on: + pull_request: + paths: + - "dev-stack.sh" + - "docker-compose.dev-stack.yml" + - "deploy/nginx/dev-stack.conf.template" + - "deploy/scripts/test-dev-stack.sh" + - "apps/ingest/internal/config/**" + - ".github/workflows/dev-stack-check.yml" + - "apps/docs/docs/development/dev-stack.md" + - "apps/docs/docs/.vuepress/config.ts" + push: + branches: + - main + paths: + - "dev-stack.sh" + - "docker-compose.dev-stack.yml" + - "deploy/nginx/dev-stack.conf.template" + - "deploy/scripts/test-dev-stack.sh" + - "apps/ingest/internal/config/**" + - ".github/workflows/dev-stack-check.yml" + - "apps/docs/docs/development/dev-stack.md" + - "apps/docs/docs/.vuepress/config.ts" + +jobs: + validate: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v6 + + - uses: actions/setup-node@v6 + with: + node-version: "20" + + - uses: actions/setup-go@v6 + with: + go-version: "1.25" + cache-dependency-path: apps/ingest/go.sum + + - name: Validate dev stack scripts + run: | + set -euo pipefail + bash -n dev-stack.sh + bash -n deploy/scripts/test-dev-stack.sh + bash deploy/scripts/test-dev-stack.sh + + - name: Test ingest config + working-directory: apps/ingest + run: go test ./internal/config diff --git a/.gitignore b/.gitignore index 3a56cbf6..0da78950 100644 --- a/.gitignore +++ b/.gitignore @@ -14,6 +14,9 @@ out .env.*.local !.env.example +# Local dev-stack runtime config and secrets +.dev/ + # OS .DS_Store Thumbs.db diff --git a/apps/docs/docs/.vuepress/config.ts b/apps/docs/docs/.vuepress/config.ts index d67135c9..0ca7659a 100644 --- a/apps/docs/docs/.vuepress/config.ts +++ b/apps/docs/docs/.vuepress/config.ts @@ -94,6 +94,7 @@ export default defineUserConfig({ text: 'Development', collapsible: true, children: [ + '/development/dev-stack.md', '/development/testing.md', ], }, diff --git a/apps/docs/docs/development/dev-stack.md b/apps/docs/docs/development/dev-stack.md new file mode 100644 index 00000000..a19ef535 --- /dev/null +++ b/apps/docs/docs/development/dev-stack.md @@ -0,0 +1,112 @@ +# Local Dev Stack + +Use `./dev-stack.sh` from the repository root when you want fast local feedback +without waiting for a release, image publication, or a test-server update. + +The command starts the production-adjacent dependencies in Docker and runs the +web app in a Docker dev container with your source checkout bind-mounted for hot +reload: + +- `http://localhost:3000` — local dev proxy and the only browser URL you need +- `http://localhost:3001` — direct Next.js dev server with hot reload inside Docker +- `localhost:9443` — ingest gRPC for agents +- `localhost:55432` — CT-Ops Postgres, bound to loopback only +- Password Manager API and database in Docker + +## Start + +Prerequisite: + +- Docker Desktop or Docker Engine with the Compose plugin + +```bash +./dev-stack.sh +``` + +The first run creates `.dev/dev.env` and `apps/web/.env.local` with local-only +secrets and URLs. The root `.env` used by the release/test-server path is not +modified. + +The `web-migrate` and `web-dev` containers run `pnpm install --frozen-lockfile` +inside Docker using named volumes for `node_modules`, so Linux container +dependencies do not overwrite host dependencies. The `ingest-dev` container uses +Go module and build-cache volumes for the same reason. + +Open: + +```text +http://localhost:3000 +``` + +The dev proxy forwards: + +- `/` to the Next.js dev server +- `/password-manager-api/` to the bundled Password Manager API container +- `/ws/terminal/` to the local ingest container + +## Stop Or Reset + +Stop the stack while preserving local database volumes: + +```bash +./dev-stack.sh --down +``` + +Remove local dev volumes and generated local config: + +```bash +./dev-stack.sh --reset +``` + +## Checks + +After the stack is running, check the proxy, Password Manager API, and ingest +health endpoints: + +```bash +./dev-stack.sh --check +``` + +Show current local service status: + +```bash +./dev-stack.sh --status +``` + +## Rebuilds + +The script builds agent binaries if they are missing. Rebuild them explicitly +after changing agent code: + +```bash +./dev-stack.sh --rebuild-agents +``` + +## Next.js Flags + +By default the script runs Next.js with Turbopack inside Docker: + +```text +CT_OPS_DEV_NEXT_FLAGS=--turbopack +``` + +If you need the webpack dev server instead, set this in `.dev/dev.env` before +starting: + +```text +CT_OPS_DEV_NEXT_FLAGS=--webpack +``` + +## Environment Separation + +The local dev stack deliberately uses separate config, ports, and Docker +volumes from the release bundle: + +- local dev config: `.dev/dev.env` and `apps/web/.env.local` +- release/test-server config: root `.env` +- local Compose project: `ct-ops-dev` +- release Compose project: the bundle directory default + +This means you can use `./dev-stack.sh` locally and still use `./update.sh` or +`./start.sh` on a test server without manually changing environment variables +between modes. diff --git a/apps/ingest/internal/config/config.go b/apps/ingest/internal/config/config.go index e26dc16f..650aeb29 100644 --- a/apps/ingest/internal/config/config.go +++ b/apps/ingest/internal/config/config.go @@ -260,6 +260,13 @@ func applyEnv(cfg *Config) { cfg.GRPCPort = port } } + if v := os.Getenv("INGEST_HTTP_PORT"); v != "" { + var port int + _, _ = fmt.Sscanf(v, "%d", &port) + if port > 0 { + cfg.HTTPPort = port + } + } if v := os.Getenv("INGEST_TLS_CERT"); v != "" { cfg.TLS.CertFile = v } diff --git a/apps/ingest/internal/config/config_test.go b/apps/ingest/internal/config/config_test.go index 480e1384..1b52cccc 100644 --- a/apps/ingest/internal/config/config_test.go +++ b/apps/ingest/internal/config/config_test.go @@ -22,6 +22,19 @@ func TestLoadAppliesTerminalTrustedOriginsEnv(t *testing.T) { } } +func TestLoadAppliesHTTPPortEnv(t *testing.T) { + t.Setenv("INGEST_HTTP_PORT", "18080") + + cfg, err := Load("/tmp/does-not-exist.yaml") + if err != nil { + t.Fatalf("Load() error = %v", err) + } + + if cfg.HTTPPort != 18080 { + t.Fatalf("cfg.HTTPPort = %d, want 18080", cfg.HTTPPort) + } +} + func TestLoadBuildsEncodedDatabaseURLFromPostgresEnv(t *testing.T) { t.Setenv("POSTGRES_USER", "ctops") t.Setenv("POSTGRES_PASSWORD", "Pyth)n2475##") diff --git a/apps/web/app/layout.tsx b/apps/web/app/layout.tsx index b5659139..dd527fb2 100644 --- a/apps/web/app/layout.tsx +++ b/apps/web/app/layout.tsx @@ -45,7 +45,11 @@ export default async function RootLayout({ suppressHydrationWarning > -