diff --git a/.github/workflows/dev-stack-check.yml b/.github/workflows/dev-stack-check.yml
new file mode 100644
index 00000000..f1aa5e2f
--- /dev/null
+++ b/.github/workflows/dev-stack-check.yml
@@ -0,0 +1,51 @@
+name: Dev stack check
+
+on:
+ pull_request:
+ paths:
+ - "dev-stack.sh"
+ - "docker-compose.dev-stack.yml"
+ - "deploy/nginx/dev-stack.conf.template"
+ - "deploy/scripts/test-dev-stack.sh"
+ - "apps/ingest/internal/config/**"
+ - ".github/workflows/dev-stack-check.yml"
+ - "apps/docs/docs/development/dev-stack.md"
+ - "apps/docs/docs/.vuepress/config.ts"
+ push:
+ branches:
+ - main
+ paths:
+ - "dev-stack.sh"
+ - "docker-compose.dev-stack.yml"
+ - "deploy/nginx/dev-stack.conf.template"
+ - "deploy/scripts/test-dev-stack.sh"
+ - "apps/ingest/internal/config/**"
+ - ".github/workflows/dev-stack-check.yml"
+ - "apps/docs/docs/development/dev-stack.md"
+ - "apps/docs/docs/.vuepress/config.ts"
+
+jobs:
+ validate:
+ runs-on: ubuntu-latest
+ steps:
+ - uses: actions/checkout@v6
+
+ - uses: actions/setup-node@v6
+ with:
+ node-version: "20"
+
+ - uses: actions/setup-go@v6
+ with:
+ go-version: "1.25"
+ cache-dependency-path: apps/ingest/go.sum
+
+ - name: Validate dev stack scripts
+ run: |
+ set -euo pipefail
+ bash -n dev-stack.sh
+ bash -n deploy/scripts/test-dev-stack.sh
+ bash deploy/scripts/test-dev-stack.sh
+
+ - name: Test ingest config
+ working-directory: apps/ingest
+ run: go test ./internal/config
diff --git a/.gitignore b/.gitignore
index 3a56cbf6..0da78950 100644
--- a/.gitignore
+++ b/.gitignore
@@ -14,6 +14,9 @@ out
.env.*.local
!.env.example
+# Local dev-stack runtime config and secrets
+.dev/
+
# OS
.DS_Store
Thumbs.db
diff --git a/apps/docs/docs/.vuepress/config.ts b/apps/docs/docs/.vuepress/config.ts
index d67135c9..0ca7659a 100644
--- a/apps/docs/docs/.vuepress/config.ts
+++ b/apps/docs/docs/.vuepress/config.ts
@@ -94,6 +94,7 @@ export default defineUserConfig({
text: 'Development',
collapsible: true,
children: [
+ '/development/dev-stack.md',
'/development/testing.md',
],
},
diff --git a/apps/docs/docs/development/dev-stack.md b/apps/docs/docs/development/dev-stack.md
new file mode 100644
index 00000000..a19ef535
--- /dev/null
+++ b/apps/docs/docs/development/dev-stack.md
@@ -0,0 +1,112 @@
+# Local Dev Stack
+
+Use `./dev-stack.sh` from the repository root when you want fast local feedback
+without waiting for a release, image publication, or a test-server update.
+
+The command starts the production-adjacent dependencies in Docker and runs the
+web app in a Docker dev container with your source checkout bind-mounted for hot
+reload:
+
+- `http://localhost:3000` — local dev proxy and the only browser URL you need
+- `http://localhost:3001` — direct Next.js dev server with hot reload inside Docker
+- `localhost:9443` — ingest gRPC for agents
+- `localhost:55432` — CT-Ops Postgres, bound to loopback only
+- Password Manager API and database in Docker
+
+## Start
+
+Prerequisite:
+
+- Docker Desktop or Docker Engine with the Compose plugin
+
+```bash
+./dev-stack.sh
+```
+
+The first run creates `.dev/dev.env` and `apps/web/.env.local` with local-only
+secrets and URLs. The root `.env` used by the release/test-server path is not
+modified.
+
+The `web-migrate` and `web-dev` containers run `pnpm install --frozen-lockfile`
+inside Docker using named volumes for `node_modules`, so Linux container
+dependencies do not overwrite host dependencies. The `ingest-dev` container uses
+Go module and build-cache volumes for the same reason.
+
+Open:
+
+```text
+http://localhost:3000
+```
+
+The dev proxy forwards:
+
+- `/` to the Next.js dev server
+- `/password-manager-api/` to the bundled Password Manager API container
+- `/ws/terminal/` to the local ingest container
+
+## Stop Or Reset
+
+Stop the stack while preserving local database volumes:
+
+```bash
+./dev-stack.sh --down
+```
+
+Remove local dev volumes and generated local config:
+
+```bash
+./dev-stack.sh --reset
+```
+
+## Checks
+
+After the stack is running, check the proxy, Password Manager API, and ingest
+health endpoints:
+
+```bash
+./dev-stack.sh --check
+```
+
+Show current local service status:
+
+```bash
+./dev-stack.sh --status
+```
+
+## Rebuilds
+
+The script builds agent binaries if they are missing. Rebuild them explicitly
+after changing agent code:
+
+```bash
+./dev-stack.sh --rebuild-agents
+```
+
+## Next.js Flags
+
+By default the script runs Next.js with Turbopack inside Docker:
+
+```text
+CT_OPS_DEV_NEXT_FLAGS=--turbopack
+```
+
+If you need the webpack dev server instead, set this in `.dev/dev.env` before
+starting:
+
+```text
+CT_OPS_DEV_NEXT_FLAGS=--webpack
+```
+
+## Environment Separation
+
+The local dev stack deliberately uses separate config, ports, and Docker
+volumes from the release bundle:
+
+- local dev config: `.dev/dev.env` and `apps/web/.env.local`
+- release/test-server config: root `.env`
+- local Compose project: `ct-ops-dev`
+- release Compose project: the bundle directory default
+
+This means you can use `./dev-stack.sh` locally and still use `./update.sh` or
+`./start.sh` on a test server without manually changing environment variables
+between modes.
diff --git a/apps/ingest/internal/config/config.go b/apps/ingest/internal/config/config.go
index e26dc16f..650aeb29 100644
--- a/apps/ingest/internal/config/config.go
+++ b/apps/ingest/internal/config/config.go
@@ -260,6 +260,13 @@ func applyEnv(cfg *Config) {
cfg.GRPCPort = port
}
}
+ if v := os.Getenv("INGEST_HTTP_PORT"); v != "" {
+ var port int
+ _, _ = fmt.Sscanf(v, "%d", &port)
+ if port > 0 {
+ cfg.HTTPPort = port
+ }
+ }
if v := os.Getenv("INGEST_TLS_CERT"); v != "" {
cfg.TLS.CertFile = v
}
diff --git a/apps/ingest/internal/config/config_test.go b/apps/ingest/internal/config/config_test.go
index 480e1384..1b52cccc 100644
--- a/apps/ingest/internal/config/config_test.go
+++ b/apps/ingest/internal/config/config_test.go
@@ -22,6 +22,19 @@ func TestLoadAppliesTerminalTrustedOriginsEnv(t *testing.T) {
}
}
+func TestLoadAppliesHTTPPortEnv(t *testing.T) {
+ t.Setenv("INGEST_HTTP_PORT", "18080")
+
+ cfg, err := Load("/tmp/does-not-exist.yaml")
+ if err != nil {
+ t.Fatalf("Load() error = %v", err)
+ }
+
+ if cfg.HTTPPort != 18080 {
+ t.Fatalf("cfg.HTTPPort = %d, want 18080", cfg.HTTPPort)
+ }
+}
+
func TestLoadBuildsEncodedDatabaseURLFromPostgresEnv(t *testing.T) {
t.Setenv("POSTGRES_USER", "ctops")
t.Setenv("POSTGRES_PASSWORD", "Pyth)n2475##")
diff --git a/apps/web/app/layout.tsx b/apps/web/app/layout.tsx
index b5659139..dd527fb2 100644
--- a/apps/web/app/layout.tsx
+++ b/apps/web/app/layout.tsx
@@ -45,7 +45,11 @@ export default async function RootLayout({
suppressHydrationWarning
>
-
+
diff --git a/deploy/nginx/dev-stack.conf.template b/deploy/nginx/dev-stack.conf.template
new file mode 100644
index 00000000..489888cb
--- /dev/null
+++ b/deploy/nginx/dev-stack.conf.template
@@ -0,0 +1,83 @@
+map $http_upgrade $connection_upgrade {
+ default "";
+ ~*^websocket$ upgrade;
+}
+
+upstream ct_ops_dev_web {
+ server ${CT_OPS_DEV_WEB_UPSTREAM};
+ keepalive 16;
+}
+
+upstream ct_ops_dev_ingest_ws {
+ server ${CT_OPS_DEV_INGEST_UPSTREAM};
+ keepalive 8;
+}
+
+upstream ct_password_manager_api {
+ server password-manager-api:8080;
+ keepalive 8;
+}
+
+server {
+ listen 80 default_server;
+ listen [::]:80 default_server;
+ server_name _;
+
+ client_max_body_size 100m;
+
+ location = /nginx-healthz {
+ access_log off;
+ return 200 "ok\n";
+ }
+
+ location /ws/terminal/ {
+ proxy_pass http://ct_ops_dev_ingest_ws;
+ proxy_http_version 1.1;
+ proxy_set_header Upgrade websocket;
+ proxy_set_header Connection $connection_upgrade;
+ proxy_set_header Host $host;
+ proxy_set_header X-Real-IP $remote_addr;
+ proxy_set_header X-Forwarded-For $remote_addr;
+ proxy_set_header X-Forwarded-Proto http;
+ proxy_read_timeout 3600s;
+ proxy_send_timeout 3600s;
+ }
+
+ location ~ ^/api/hosts/[^/]+/stream$ {
+ proxy_pass http://ct_ops_dev_web;
+ proxy_http_version 1.1;
+ proxy_set_header Upgrade "";
+ proxy_set_header Connection "";
+ proxy_set_header Host $host;
+ proxy_set_header X-Real-IP $remote_addr;
+ proxy_set_header X-Forwarded-For $remote_addr;
+ proxy_set_header X-Forwarded-Proto http;
+ proxy_buffering off;
+ proxy_cache off;
+ proxy_read_timeout 3600s;
+ chunked_transfer_encoding on;
+ }
+
+ location /password-manager-api/ {
+ proxy_pass http://ct_password_manager_api/;
+ proxy_http_version 1.1;
+ proxy_set_header Upgrade "";
+ proxy_set_header Connection "";
+ proxy_set_header Host $host;
+ proxy_set_header X-Real-IP $remote_addr;
+ proxy_set_header X-Forwarded-For $remote_addr;
+ proxy_set_header X-Forwarded-Proto http;
+ }
+
+ location / {
+ proxy_pass http://ct_ops_dev_web;
+ proxy_http_version 1.1;
+ proxy_set_header Upgrade $http_upgrade;
+ proxy_set_header Connection $connection_upgrade;
+ proxy_set_header Host $host;
+ proxy_set_header X-Real-IP $remote_addr;
+ proxy_set_header X-Forwarded-For $remote_addr;
+ proxy_set_header X-Forwarded-Proto http;
+ proxy_read_timeout 3600s;
+ }
+}
diff --git a/deploy/scripts/test-dev-stack.sh b/deploy/scripts/test-dev-stack.sh
new file mode 100755
index 00000000..d28b03c1
--- /dev/null
+++ b/deploy/scripts/test-dev-stack.sh
@@ -0,0 +1,107 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+REPO_ROOT="$(cd "${SCRIPT_DIR}/../.." && pwd)"
+
+assert_contains() {
+ local file="$1"
+ local needle="$2"
+
+ if ! grep -Fq "$needle" "$file"; then
+ echo "expected ${file} to contain: ${needle}" >&2
+ exit 1
+ fi
+}
+
+assert_not_contains() {
+ local file="$1"
+ local needle="$2"
+
+ if grep -Fq "$needle" "$file"; then
+ echo "expected ${file} not to contain: ${needle}" >&2
+ exit 1
+ fi
+}
+
+run_config_generation_test() {
+ local tmpdir repo_dir before_root_env private_key expected_public actual_public
+
+ tmpdir="$(mktemp -d "${TMPDIR:-/tmp}/ct-ops-dev-stack-test.XXXXXX")"
+ trap 'rm -rf "'"$tmpdir"'"' RETURN
+
+ repo_dir="${tmpdir}/ct-ops"
+ mkdir -p "$repo_dir"
+ cp -R \
+ "${REPO_ROOT}/apps" \
+ "${REPO_ROOT}/deploy" \
+ "${REPO_ROOT}/dev-stack.sh" \
+ "${REPO_ROOT}/docker-compose.dev-stack.yml" \
+ "${REPO_ROOT}/package.json" \
+ "${REPO_ROOT}/Makefile" \
+ "$repo_dir/"
+ cp "${REPO_ROOT}/deploy/password-manager-release.json" "$repo_dir/deploy/password-manager-release.json"
+
+ cat > "${repo_dir}/.env" <<'EOF'
+BETTER_AUTH_URL=https://test-server.example.com
+BETTER_AUTH_SECRET=production-like-secret
+POSTGRES_PASSWORD=production-like-postgres
+EOF
+ before_root_env="$(cat "${repo_dir}/.env")"
+
+ (
+ cd "$repo_dir"
+ ./dev-stack.sh --write-config-only >/dev/null
+ )
+
+ if [ "$(cat "${repo_dir}/.env")" != "$before_root_env" ]; then
+ echo "dev-stack.sh must not modify root .env" >&2
+ exit 1
+ fi
+
+ test -f "${repo_dir}/.dev/dev.env"
+ test -f "${repo_dir}/apps/web/.env.local"
+
+ assert_contains "${repo_dir}/.dev/dev.env" "BETTER_AUTH_URL=http://localhost:3000"
+ assert_contains "${repo_dir}/.dev/dev.env" "PASSWORD_MANAGER_SESSION_COOKIE_SECURE=false"
+ assert_contains "${repo_dir}/.dev/dev.env" "CT_OPS_TRUST_PROXY_HEADERS=true"
+ assert_contains "${repo_dir}/apps/web/.env.local" "BETTER_AUTH_URL=http://localhost:3000"
+ assert_contains "${repo_dir}/apps/web/.env.local" "DATABASE_URL=postgresql://"
+ assert_contains "${repo_dir}/apps/web/.env.local" "PASSWORD_MANAGER_CT_OPS_ED25519_PRIVATE_KEY="
+ assert_not_contains "${repo_dir}/apps/web/.env.local" "https://test-server.example.com"
+
+ private_key="$(sed -n 's/^PASSWORD_MANAGER_CT_OPS_ED25519_PRIVATE_KEY=//p' "${repo_dir}/.dev/dev.env")"
+ actual_public="$(sed -n 's/^PASSWORD_MANAGER_CT_OPS_ED25519_PUBLIC_KEY=//p' "${repo_dir}/.dev/dev.env")"
+ expected_public="$(
+ CT_PM_PRIVATE_KEY="$private_key" node - <<'EOF'
+const { createPrivateKey, createPublicKey } = require('node:crypto')
+const privateKey = createPrivateKey({
+ key: Buffer.from(process.env.CT_PM_PRIVATE_KEY, 'base64'),
+ format: 'der',
+ type: 'pkcs8',
+})
+process.stdout.write(createPublicKey(privateKey).export({ format: 'der', type: 'spki' }).subarray(-32).toString('base64'))
+EOF
+ )"
+
+ if [ "$actual_public" != "$expected_public" ]; then
+ echo "Password Manager public key does not match generated private key" >&2
+ exit 1
+ fi
+}
+
+run_static_wiring_test() {
+ assert_contains "${REPO_ROOT}/docker-compose.dev-stack.yml" "127.0.0.1:\${CT_OPS_DEV_PROXY_PORT}:80"
+ assert_contains "${REPO_ROOT}/docker-compose.dev-stack.yml" "127.0.0.1:\${POSTGRES_PORT}:5432"
+ assert_contains "${REPO_ROOT}/docker-compose.dev-stack.yml" "host.docker.internal:host-gateway"
+ assert_contains "${REPO_ROOT}/docker-compose.dev-stack.yml" "CT_OPS_DEV_WEB_UPSTREAM: web-dev:3001"
+ assert_contains "${REPO_ROOT}/docker-compose.dev-stack.yml" "CT_OPS_DEV_INGEST_UPSTREAM: ingest-dev:8080"
+ assert_contains "${REPO_ROOT}/deploy/nginx/dev-stack.conf.template" "location /password-manager-api/"
+ assert_contains "${REPO_ROOT}/deploy/nginx/dev-stack.conf.template" "location /ws/terminal/"
+ assert_contains "${REPO_ROOT}/deploy/nginx/dev-stack.conf.template" "proxy_pass http://ct_ops_dev_web"
+}
+
+run_config_generation_test
+run_static_wiring_test
+
+echo "dev-stack tests passed"
diff --git a/dev-stack.sh b/dev-stack.sh
new file mode 100755
index 00000000..9dd62d27
--- /dev/null
+++ b/dev-stack.sh
@@ -0,0 +1,445 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+cd "$SCRIPT_DIR"
+
+COMPOSE_PROJECT_NAME="${COMPOSE_PROJECT_NAME:-ct-ops-dev}"
+COMPOSE_FILE="${SCRIPT_DIR}/docker-compose.dev-stack.yml"
+DEV_DIR="${SCRIPT_DIR}/.dev"
+DEV_ENV="${DEV_DIR}/dev.env"
+WEB_ENV="${SCRIPT_DIR}/apps/web/.env.local"
+INGEST_DATA_DIR="${SCRIPT_DIR}/deploy/dev-ingest-data"
+INGEST_PID=""
+
+DOWN=false
+RESET=false
+STATUS=false
+CHECK=false
+WRITE_CONFIG_ONLY=false
+REBUILD_AGENTS=false
+SKIP_AGENTS=false
+
+usage() {
+ cat <<'EOF'
+CT-Ops local dev stack
+
+Usage:
+ ./dev-stack.sh Start the full local dev stack
+ ./dev-stack.sh --down Stop Docker services
+ ./dev-stack.sh --reset Stop and remove local dev volumes/config
+ ./dev-stack.sh --status Show local dev service status
+ ./dev-stack.sh --check Check local dev endpoints
+ ./dev-stack.sh --rebuild-agents Rebuild agent binaries before starting
+ ./dev-stack.sh --skip-agents Do not build missing agent binaries
+
+Open http://localhost:3000. The dev proxy listens on :3000 and forwards to
+Next.js, Password Manager, and ingest running in Docker.
+EOF
+}
+
+for arg in "$@"; do
+ case "$arg" in
+ --down) DOWN=true ;;
+ --reset) RESET=true ;;
+ --status) STATUS=true ;;
+ --check) CHECK=true ;;
+ --write-config-only) WRITE_CONFIG_ONLY=true ;;
+ --rebuild-agents) REBUILD_AGENTS=true ;;
+ --skip-agents) SKIP_AGENTS=true ;;
+ --help|-h) usage; exit 0 ;;
+ *) echo "Unknown flag: $arg" >&2; usage >&2; exit 1 ;;
+ esac
+done
+
+compose() {
+ docker compose --project-name "$COMPOSE_PROJECT_NAME" --env-file "$DEV_ENV" -f "$COMPOSE_FILE" "$@"
+}
+
+need() {
+ if ! command -v "$1" >/dev/null 2>&1; then
+ echo "ERROR: required command '$1' not found in PATH" >&2
+ exit 1
+ fi
+}
+
+check_start_dependencies() {
+ local missing=0
+
+ if ! command -v docker >/dev/null 2>&1; then
+ echo "ERROR: required command 'docker' not found in PATH" >&2
+ missing=1
+ fi
+
+ if ! docker compose version >/dev/null 2>&1; then
+ echo "ERROR: Docker Compose plugin is not available. Install Docker Desktop or Docker Engine with Compose." >&2
+ missing=1
+ elif ! docker info >/dev/null 2>&1; then
+ echo "ERROR: Docker is installed but the daemon is not running." >&2
+ missing=1
+ fi
+
+ if [ "$missing" -ne 0 ]; then
+ echo "" >&2
+ echo "Install/start the missing prerequisites, then re-run ./dev-stack.sh." >&2
+ exit 1
+ fi
+}
+
+upsert_env_var() {
+ local file="$1"
+ local key="$2"
+ local value="$3"
+
+ mkdir -p "$(dirname "$file")"
+ touch "$file"
+ awk -v key="$key" -v value="$value" '
+ BEGIN { replaced = 0 }
+ $0 ~ "^#?[[:space:]]*" key "=" {
+ print key "=" value
+ replaced = 1
+ next
+ }
+ { print }
+ END {
+ if (!replaced) {
+ print key "=" value
+ }
+ }
+ ' "$file" > "${file}.tmp" && mv "${file}.tmp" "$file"
+ chmod 600 "$file"
+}
+
+read_env_var() {
+ local file="$1"
+ local key="$2"
+ [ -f "$file" ] || return 0
+ sed -n "s/^${key}=//p" "$file" | tail -n1
+}
+
+generate_hex() {
+ local bytes="$1"
+ openssl rand -hex "$bytes"
+}
+
+read_password_manager_digest_reference() {
+ sed -n 's/.*"digest_reference"[[:space:]]*:[[:space:]]*"\([^"]*\)".*/\1/p' \
+ deploy/password-manager-release.json | head -n1
+}
+
+find_optional_command() {
+ local cmd="$1"
+ local dir
+
+ if command -v "$cmd" >/dev/null 2>&1; then
+ command -v "$cmd"
+ return 0
+ fi
+
+ for dir in /opt/homebrew/bin /usr/local/bin; do
+ if [ -x "${dir}/${cmd}" ]; then
+ printf '%s\n' "${dir}/${cmd}"
+ return 0
+ fi
+ done
+
+ return 1
+}
+
+generate_password_manager_launch_keypair() {
+ local node_bin
+
+ if node_bin="$(find_optional_command node)"; then
+ "$node_bin" - <<'EOF'
+const { generateKeyPairSync } = require('node:crypto')
+
+const { privateKey, publicKey } = generateKeyPairSync('ed25519')
+process.stdout.write(privateKey.export({ format: 'der', type: 'pkcs8' }).toString('base64') + '\n')
+process.stdout.write(publicKey.export({ format: 'der', type: 'spki' }).subarray(-32).toString('base64') + '\n')
+EOF
+ return 0
+ fi
+
+ echo "ERROR: Node.js is required to generate Password Manager launch-signing keys." >&2
+ exit 1
+}
+
+derive_password_manager_launch_public_key() {
+ local private_key="$1"
+ local node_bin
+
+ if node_bin="$(find_optional_command node)"; then
+ CT_PM_PRIVATE_KEY="$private_key" "$node_bin" - <<'EOF'
+const { createPrivateKey, createPublicKey } = require('node:crypto')
+
+const privateKey = createPrivateKey({
+ key: Buffer.from(process.env.CT_PM_PRIVATE_KEY, 'base64'),
+ format: 'der',
+ type: 'pkcs8',
+})
+process.stdout.write(createPublicKey(privateKey).export({ format: 'der', type: 'spki' }).subarray(-32).toString('base64') + '\n')
+EOF
+ return 0
+ fi
+
+ return 1
+}
+
+ensure_dev_env() {
+ local proxy_port next_port postgres_port ingest_http_port ingest_grpc_port app_url pm_image
+ local private_key public_key keypair_output repaired_public_key
+
+ need openssl
+ need node
+
+ mkdir -p "$DEV_DIR"
+ touch "$DEV_ENV"
+ chmod 600 "$DEV_ENV"
+
+ proxy_port="$(read_env_var "$DEV_ENV" CT_OPS_DEV_PROXY_PORT)"
+ next_port="$(read_env_var "$DEV_ENV" CT_OPS_DEV_NEXT_PORT)"
+ postgres_port="$(read_env_var "$DEV_ENV" POSTGRES_PORT)"
+ ingest_http_port="$(read_env_var "$DEV_ENV" CT_OPS_DEV_INGEST_HTTP_PORT)"
+ ingest_grpc_port="$(read_env_var "$DEV_ENV" CT_OPS_DEV_INGEST_GRPC_PORT)"
+
+ proxy_port="${proxy_port:-3000}"
+ next_port="${next_port:-3001}"
+ postgres_port="${postgres_port:-55432}"
+ ingest_http_port="${ingest_http_port:-8080}"
+ ingest_grpc_port="${ingest_grpc_port:-9443}"
+ app_url="http://localhost:${proxy_port}"
+
+ upsert_env_var "$DEV_ENV" COMPOSE_PROJECT_NAME "$COMPOSE_PROJECT_NAME"
+ upsert_env_var "$DEV_ENV" CT_OPS_DEV_PROXY_PORT "$proxy_port"
+ upsert_env_var "$DEV_ENV" CT_OPS_DEV_NEXT_PORT "$next_port"
+ upsert_env_var "$DEV_ENV" CT_OPS_DEV_INGEST_HTTP_PORT "$ingest_http_port"
+ upsert_env_var "$DEV_ENV" CT_OPS_DEV_INGEST_GRPC_PORT "$ingest_grpc_port"
+ upsert_env_var "$DEV_ENV" CT_OPS_DEV_APP_URL "$app_url"
+ if [ -z "$(read_env_var "$DEV_ENV" CT_OPS_DEV_NEXT_FLAGS)" ]; then
+ upsert_env_var "$DEV_ENV" CT_OPS_DEV_NEXT_FLAGS "--turbopack"
+ fi
+
+ if [ -z "$(read_env_var "$DEV_ENV" POSTGRES_USER)" ]; then
+ upsert_env_var "$DEV_ENV" POSTGRES_USER "ctops"
+ fi
+ if [ -z "$(read_env_var "$DEV_ENV" POSTGRES_DB)" ]; then
+ upsert_env_var "$DEV_ENV" POSTGRES_DB "ctops"
+ fi
+ if [ -z "$(read_env_var "$DEV_ENV" POSTGRES_PASSWORD)" ]; then
+ upsert_env_var "$DEV_ENV" POSTGRES_PASSWORD "$(generate_hex 16)"
+ fi
+ upsert_env_var "$DEV_ENV" POSTGRES_PORT "$postgres_port"
+
+ if [ -z "$(read_env_var "$DEV_ENV" BETTER_AUTH_SECRET)" ]; then
+ upsert_env_var "$DEV_ENV" BETTER_AUTH_SECRET "$(generate_hex 32)"
+ fi
+ upsert_env_var "$DEV_ENV" BETTER_AUTH_URL "$app_url"
+ upsert_env_var "$DEV_ENV" BETTER_AUTH_TRUSTED_ORIGINS "$app_url"
+ upsert_env_var "$DEV_ENV" REQUIRE_EMAIL_VERIFICATION "false"
+ upsert_env_var "$DEV_ENV" CT_OPS_TRUST_PROXY_HEADERS "true"
+ upsert_env_var "$DEV_ENV" AGENT_DOWNLOAD_BASE_URL "$app_url"
+ upsert_env_var "$DEV_ENV" INGEST_WS_URL ""
+
+ if [ -z "$(read_env_var "$DEV_ENV" PASSWORD_MANAGER_DB_PASSWORD)" ]; then
+ upsert_env_var "$DEV_ENV" PASSWORD_MANAGER_DB_PASSWORD "$(generate_hex 16)"
+ fi
+ if [ -z "$(read_env_var "$DEV_ENV" CT_OPS_INSTANCE_ID)" ]; then
+ upsert_env_var "$DEV_ENV" CT_OPS_INSTANCE_ID "ct-ops-dev-$(generate_hex 4)"
+ fi
+ upsert_env_var "$DEV_ENV" PASSWORD_MANAGER_CT_OPS_ISSUER "$app_url"
+ upsert_env_var "$DEV_ENV" PASSWORD_MANAGER_CT_OPS_AUDIENCE "ct-password-manager"
+ upsert_env_var "$DEV_ENV" PASSWORD_MANAGER_CT_OPS_PRODUCT "ct-password-manager"
+ upsert_env_var "$DEV_ENV" PASSWORD_MANAGER_TRUSTED_ORIGINS "$app_url"
+ upsert_env_var "$DEV_ENV" PASSWORD_MANAGER_SESSION_COOKIE_SECURE "false"
+
+ private_key="$(read_env_var "$DEV_ENV" PASSWORD_MANAGER_CT_OPS_ED25519_PRIVATE_KEY)"
+ if [ -n "$private_key" ] && repaired_public_key="$(derive_password_manager_launch_public_key "$private_key" 2>/dev/null)"; then
+ upsert_env_var "$DEV_ENV" PASSWORD_MANAGER_CT_OPS_ED25519_PUBLIC_KEY "$repaired_public_key"
+ else
+ keypair_output="$(generate_password_manager_launch_keypair)"
+ private_key="$(printf '%s\n' "$keypair_output" | sed -n '1p')"
+ public_key="$(printf '%s\n' "$keypair_output" | sed -n '2p')"
+ upsert_env_var "$DEV_ENV" PASSWORD_MANAGER_CT_OPS_ED25519_PRIVATE_KEY "$private_key"
+ upsert_env_var "$DEV_ENV" PASSWORD_MANAGER_CT_OPS_ED25519_PUBLIC_KEY "$public_key"
+ fi
+
+ pm_image="$(read_password_manager_digest_reference)"
+ if [ -z "$pm_image" ]; then
+ echo "ERROR: could not read Password Manager image from deploy/password-manager-release.json" >&2
+ exit 1
+ fi
+ upsert_env_var "$DEV_ENV" PASSWORD_MANAGER_API_IMAGE "$pm_image"
+}
+
+write_web_env() {
+ # shellcheck disable=SC1090
+ set -a; source "$DEV_ENV"; set +a
+
+ cat > "$WEB_ENV" </dev/null | grep -Fxq "$service"
+}
+
+check_port_available() {
+ local port="$1"
+ local label="$2"
+
+ if command -v lsof >/dev/null 2>&1 && lsof -nP -iTCP:"$port" -sTCP:LISTEN >/dev/null 2>&1; then
+ echo "ERROR: port ${port} is already in use (${label})." >&2
+ lsof -nP -iTCP:"$port" -sTCP:LISTEN >&2 || true
+ exit 1
+ fi
+}
+
+generate_dev_tls() {
+ if [ ! -f deploy/dev-tls/server.crt ] || [ ! -f deploy/dev-tls/server.key ]; then
+ deploy/scripts/gen-dev-tls.sh
+ fi
+}
+
+wait_for_db() {
+ local user db
+ # shellcheck disable=SC1090
+ set -a; source "$DEV_ENV"; set +a
+ user="$POSTGRES_USER"
+ db="$POSTGRES_DB"
+
+ echo "Waiting for CT-Ops database..."
+ until compose exec -T db pg_isready -U "$user" -d "$db" >/dev/null 2>&1; do
+ sleep 1
+ done
+}
+
+build_agent_binaries() {
+ if $SKIP_AGENTS; then
+ return 0
+ fi
+
+ if $REBUILD_AGENTS || [ ! -f apps/web/data/agent-dist/ct-ops-agent-linux-amd64 ]; then
+ echo "Building agent binaries..."
+ make agent
+ fi
+}
+
+cleanup_legacy_ingest_pid() {
+ if [ -f "$DEV_DIR/ingest.pid" ]; then
+ local pid
+ pid="$(cat "$DEV_DIR/ingest.pid" 2>/dev/null || true)"
+ if [ -n "$pid" ]; then
+ kill "$pid" 2>/dev/null || true
+ fi
+ rm -f "$DEV_DIR/ingest.pid"
+ fi
+}
+
+check_endpoints() {
+ # shellcheck disable=SC1090
+ set -a; source "$DEV_ENV"; set +a
+
+ curl -fsS "${BETTER_AUTH_URL}/nginx-healthz" >/dev/null
+ curl -fsS "${BETTER_AUTH_URL}/password-manager-api/healthz" >/dev/null
+ curl -fsS "http://localhost:${CT_OPS_DEV_INGEST_HTTP_PORT}/healthz" >/dev/null
+ echo "Dev stack endpoints are reachable."
+}
+
+show_status() {
+ if [ -f "$DEV_ENV" ]; then
+ # shellcheck disable=SC1090
+ set -a; source "$DEV_ENV"; set +a
+ echo "URL: ${BETTER_AUTH_URL:-http://localhost:3000}"
+ fi
+ if command -v docker >/dev/null 2>&1 && [ -f "$DEV_ENV" ]; then
+ compose ps
+ fi
+}
+
+ensure_dev_env
+write_web_env
+
+if $WRITE_CONFIG_ONLY; then
+ echo "Wrote ${DEV_ENV} and ${WEB_ENV}."
+ exit 0
+fi
+
+if $DOWN || $RESET; then
+ need docker
+ cleanup_legacy_ingest_pid
+ if $RESET; then
+ compose down -v --remove-orphans || true
+ rm -rf "$DEV_DIR" "$WEB_ENV" "$INGEST_DATA_DIR"
+ echo "Local dev stack reset."
+ else
+ compose down --remove-orphans || true
+ echo "Local dev stack stopped. Data volumes are preserved."
+ fi
+ exit 0
+fi
+
+if $STATUS; then
+ show_status
+ exit 0
+fi
+
+if $CHECK; then
+ check_endpoints
+ exit 0
+fi
+
+check_start_dependencies
+
+# shellcheck disable=SC1090
+set -a; source "$DEV_ENV"; set +a
+
+cleanup_legacy_ingest_pid
+
+if ! compose_service_running dev-proxy; then
+ check_port_available "$CT_OPS_DEV_PROXY_PORT" "dev proxy"
+fi
+if ! compose_service_running db; then
+ check_port_available "$POSTGRES_PORT" "CT-Ops Postgres"
+fi
+if ! compose_service_running web-dev; then
+ check_port_available "$CT_OPS_DEV_NEXT_PORT" "Next.js direct"
+fi
+check_port_available "$CT_OPS_DEV_INGEST_HTTP_PORT" "ingest HTTP"
+check_port_available "$CT_OPS_DEV_INGEST_GRPC_PORT" "ingest gRPC"
+
+generate_dev_tls
+build_agent_binaries
+
+echo "Starting Docker backing services..."
+compose up -d db password-manager-db password-manager-migrate password-manager-api web-migrate web-dev ingest-dev dev-proxy
+wait_for_db
+
+echo ""
+echo "CT-Ops dev stack is starting."
+echo " Web UI: ${BETTER_AUTH_URL}"
+echo " Next.js direct: http://localhost:${CT_OPS_DEV_NEXT_PORT}"
+echo " Ingest gRPC: localhost:${CT_OPS_DEV_INGEST_GRPC_PORT}"
+echo " Postgres: localhost:${POSTGRES_PORT}"
+echo ""
+echo "Stop Docker services later with: ./dev-stack.sh --down"
+echo ""
+echo "Tail logs with: docker compose --project-name $COMPOSE_PROJECT_NAME --env-file .dev/dev.env -f docker-compose.dev-stack.yml logs -f web-dev ingest-dev"
diff --git a/docker-compose.dev-stack.yml b/docker-compose.dev-stack.yml
new file mode 100644
index 00000000..15b199d5
--- /dev/null
+++ b/docker-compose.dev-stack.yml
@@ -0,0 +1,230 @@
+##
+# CT-Ops full local development stack.
+#
+# This compose file is intentionally source-tree only. It runs backing services
+# and a dev proxy while Next.js and ingest run from bind-mounted source for
+# fast local feedback.
+# Use ./dev-stack.sh rather than invoking this file directly.
+##
+
+services:
+ db:
+ image: timescale/timescaledb:latest-pg16
+ restart: unless-stopped
+ environment:
+ POSTGRES_USER: ${POSTGRES_USER}
+ POSTGRES_PASSWORD: ${POSTGRES_PASSWORD}
+ POSTGRES_DB: ${POSTGRES_DB}
+ volumes:
+ - db_data:/var/lib/postgresql/data
+ ports:
+ - "127.0.0.1:${POSTGRES_PORT}:5432"
+ healthcheck:
+ test: ["CMD-SHELL", "pg_isready -U ${POSTGRES_USER} -d ${POSTGRES_DB}"]
+ interval: 5s
+ timeout: 5s
+ retries: 20
+
+ password-manager-db:
+ image: postgres:17-alpine@sha256:c7526c0f6c3f30260a563d7bcf8ad778effac59a44f8ffa86678c35418338609
+ restart: unless-stopped
+ environment:
+ POSTGRES_DB: password_manager
+ POSTGRES_USER: password_manager
+ POSTGRES_PASSWORD: ${PASSWORD_MANAGER_DB_PASSWORD}
+ volumes:
+ - password_manager_db_data:/var/lib/postgresql/data
+ healthcheck:
+ test: ["CMD-SHELL", "pg_isready -U password_manager -d password_manager"]
+ interval: 5s
+ timeout: 5s
+ retries: 20
+ start_period: 5s
+
+ password-manager-migrate:
+ image: ${PASSWORD_MANAGER_API_IMAGE:?PASSWORD_MANAGER_API_IMAGE must be set by dev-stack.sh}
+ restart: "no"
+ depends_on:
+ password-manager-db:
+ condition: service_healthy
+ environment:
+ PM_DATABASE_URL: postgres://password_manager:${PASSWORD_MANAGER_DB_PASSWORD}@password-manager-db:5432/password_manager?sslmode=disable
+ PM_CT_OPS_ISSUER: ${PASSWORD_MANAGER_CT_OPS_ISSUER}
+ PM_CT_OPS_AUDIENCE: ${PASSWORD_MANAGER_CT_OPS_AUDIENCE}
+ PM_CT_OPS_PRODUCT: ${PASSWORD_MANAGER_CT_OPS_PRODUCT}
+ PM_CT_OPS_INSTANCE_ID: ${CT_OPS_INSTANCE_ID}
+ PM_CT_OPS_ED25519_PUBLIC_KEY: ${PASSWORD_MANAGER_CT_OPS_ED25519_PUBLIC_KEY}
+ PM_TRUSTED_ORIGINS: ${PASSWORD_MANAGER_TRUSTED_ORIGINS}
+ PM_SESSION_COOKIE_SECURE: ${PASSWORD_MANAGER_SESSION_COOKIE_SECURE}
+ entrypoint: ["/app/password-manager-migrate"]
+ command: ["-action", "up"]
+
+ password-manager-api:
+ image: ${PASSWORD_MANAGER_API_IMAGE:?PASSWORD_MANAGER_API_IMAGE must be set by dev-stack.sh}
+ restart: unless-stopped
+ depends_on:
+ password-manager-db:
+ condition: service_healthy
+ password-manager-migrate:
+ condition: service_completed_successfully
+ environment:
+ PM_DATABASE_URL: postgres://password_manager:${PASSWORD_MANAGER_DB_PASSWORD}@password-manager-db:5432/password_manager?sslmode=disable
+ PM_CT_OPS_ISSUER: ${PASSWORD_MANAGER_CT_OPS_ISSUER}
+ PM_CT_OPS_AUDIENCE: ${PASSWORD_MANAGER_CT_OPS_AUDIENCE}
+ PM_CT_OPS_PRODUCT: ${PASSWORD_MANAGER_CT_OPS_PRODUCT}
+ PM_CT_OPS_INSTANCE_ID: ${CT_OPS_INSTANCE_ID}
+ PM_CT_OPS_ED25519_PUBLIC_KEY: ${PASSWORD_MANAGER_CT_OPS_ED25519_PUBLIC_KEY}
+ PM_TRUSTED_ORIGINS: ${PASSWORD_MANAGER_TRUSTED_ORIGINS}
+ PM_SESSION_COOKIE_SECURE: ${PASSWORD_MANAGER_SESSION_COOKIE_SECURE}
+ healthcheck:
+ test: ["CMD-SHELL", "wget -qO- http://127.0.0.1:8080/healthz >/dev/null"]
+ interval: 10s
+ timeout: 5s
+ retries: 12
+ start_period: 10s
+
+ web-migrate:
+ image: node:20-bookworm-slim
+ restart: "no"
+ working_dir: /workspace
+ depends_on:
+ db:
+ condition: service_healthy
+ environment:
+ DATABASE_URL: postgresql://${POSTGRES_USER}:${POSTGRES_PASSWORD}@db:5432/${POSTGRES_DB}
+ BETTER_AUTH_SECRET: ${BETTER_AUTH_SECRET}
+ BETTER_AUTH_URL: ${BETTER_AUTH_URL}
+ BETTER_AUTH_TRUSTED_ORIGINS: ${BETTER_AUTH_TRUSTED_ORIGINS}
+ volumes:
+ - .:/workspace
+ - web_node_modules:/workspace/node_modules
+ - web_app_node_modules:/workspace/apps/web/node_modules
+ - pnpm_store:/pnpm/store
+ command:
+ - /bin/sh
+ - -lc
+ - |
+ corepack enable
+ corepack prepare pnpm@10.6.5 --activate
+ pnpm config set store-dir /pnpm/store
+ pnpm install --frozen-lockfile
+ cd apps/web
+ pnpm run db:migrate
+
+ web-dev:
+ image: node:20-bookworm-slim
+ restart: unless-stopped
+ working_dir: /workspace
+ depends_on:
+ web-migrate:
+ condition: service_completed_successfully
+ environment:
+ DATABASE_URL: postgresql://${POSTGRES_USER}:${POSTGRES_PASSWORD}@db:5432/${POSTGRES_DB}
+ BETTER_AUTH_SECRET: ${BETTER_AUTH_SECRET}
+ BETTER_AUTH_URL: ${BETTER_AUTH_URL}
+ BETTER_AUTH_TRUSTED_ORIGINS: ${BETTER_AUTH_TRUSTED_ORIGINS}
+ CT_OPS_TRUST_PROXY_HEADERS: ${CT_OPS_TRUST_PROXY_HEADERS}
+ REQUIRE_EMAIL_VERIFICATION: ${REQUIRE_EMAIL_VERIFICATION}
+ AGENT_DOWNLOAD_BASE_URL: ${AGENT_DOWNLOAD_BASE_URL}
+ INGEST_WS_URL: ${INGEST_WS_URL}
+ AGENT_DIST_DIR: ./data/agent-dist
+ INGEST_TLS_CERT: /workspace/deploy/dev-tls/server.crt
+ PASSWORD_MANAGER_CT_OPS_ISSUER: ${PASSWORD_MANAGER_CT_OPS_ISSUER}
+ PASSWORD_MANAGER_CT_OPS_AUDIENCE: ${PASSWORD_MANAGER_CT_OPS_AUDIENCE}
+ PASSWORD_MANAGER_CT_OPS_PRODUCT: ${PASSWORD_MANAGER_CT_OPS_PRODUCT}
+ PASSWORD_MANAGER_CT_OPS_ED25519_PRIVATE_KEY: ${PASSWORD_MANAGER_CT_OPS_ED25519_PRIVATE_KEY}
+ CT_OPS_INSTANCE_ID: ${CT_OPS_INSTANCE_ID}
+ ANSIBLE_API_URL: http://ansible-api:8080
+ CT_OPS_DEV_NEXT_FLAGS: ${CT_OPS_DEV_NEXT_FLAGS}
+ expose:
+ - "3001"
+ ports:
+ - "127.0.0.1:${CT_OPS_DEV_NEXT_PORT}:3001"
+ volumes:
+ - .:/workspace
+ - web_node_modules:/workspace/node_modules
+ - web_app_node_modules:/workspace/apps/web/node_modules
+ - pnpm_store:/pnpm/store
+ command:
+ - /bin/sh
+ - -lc
+ - |
+ corepack enable
+ corepack prepare pnpm@10.6.5 --activate
+ pnpm config set store-dir /pnpm/store
+ pnpm install --frozen-lockfile
+ cd apps/web
+ pnpm exec next dev ${CT_OPS_DEV_NEXT_FLAGS} -H 0.0.0.0 -p 3001
+
+ ingest-dev:
+ image: golang:1.25
+ restart: unless-stopped
+ working_dir: /workspace
+ depends_on:
+ db:
+ condition: service_healthy
+ web-migrate:
+ condition: service_completed_successfully
+ environment:
+ POSTGRES_USER: ${POSTGRES_USER}
+ POSTGRES_PASSWORD: ${POSTGRES_PASSWORD}
+ POSTGRES_HOST: db
+ POSTGRES_PORT: "5432"
+ POSTGRES_DB: ${POSTGRES_DB}
+ BETTER_AUTH_SECRET: ${BETTER_AUTH_SECRET}
+ LDAP_ENCRYPTION_KEY: ${BETTER_AUTH_SECRET}
+ INGEST_GRPC_PORT: "9443"
+ INGEST_HTTP_PORT: "8080"
+ INGEST_TLS_CERT: /workspace/deploy/dev-tls/server.crt
+ INGEST_TLS_KEY: /workspace/deploy/dev-tls/server.key
+ INGEST_JWT_KEY_FILE: /workspace/deploy/dev-ingest-data/jwt_key.pem
+ INGEST_AGENT_DOWNLOAD_BASE_URL: ${AGENT_DOWNLOAD_BASE_URL}
+ INGEST_RELEASE_MANIFEST_PATH: /workspace/.release-please-manifest.json
+ ports:
+ - "127.0.0.1:${CT_OPS_DEV_INGEST_HTTP_PORT}:8080"
+ - "127.0.0.1:${CT_OPS_DEV_INGEST_GRPC_PORT}:9443"
+ volumes:
+ - .:/workspace
+ - go_build_cache:/go-cache
+ - go_mod_cache:/go/pkg/mod
+ command:
+ - /bin/sh
+ - -lc
+ - |
+ mkdir -p /workspace/deploy/dev-ingest-data
+ GOCACHE=/go-cache /usr/local/go/bin/go run ./apps/ingest/cmd/ingest
+
+ dev-proxy:
+ image: nginx@sha256:65645c7bb6a0661892a8b03b89d0743208a18dd2f3f17a54ef4b76fb8e2f2a10
+ restart: unless-stopped
+ depends_on:
+ web-dev:
+ condition: service_started
+ ingest-dev:
+ condition: service_started
+ password-manager-api:
+ condition: service_healthy
+ ports:
+ - "127.0.0.1:${CT_OPS_DEV_PROXY_PORT}:80"
+ environment:
+ CT_OPS_DEV_NEXT_PORT: ${CT_OPS_DEV_NEXT_PORT}
+ CT_OPS_DEV_WEB_UPSTREAM: web-dev:3001
+ CT_OPS_DEV_INGEST_UPSTREAM: ingest-dev:8080
+ extra_hosts:
+ - "host.docker.internal:host-gateway"
+ volumes:
+ - ./deploy/nginx/dev-stack.conf.template:/etc/nginx/templates/default.conf.template:ro
+ healthcheck:
+ test: ["CMD", "wget", "-qO-", "http://127.0.0.1/nginx-healthz"]
+ interval: 10s
+ timeout: 5s
+ retries: 5
+
+volumes:
+ db_data:
+ password_manager_db_data:
+ web_node_modules:
+ web_app_node_modules:
+ pnpm_store:
+ go_build_cache:
+ go_mod_cache: