diff --git a/apps/docs/docs/development/dev-stack.md b/apps/docs/docs/development/dev-stack.md index 1b78a3a0..1937ca9c 100644 --- a/apps/docs/docs/development/dev-stack.md +++ b/apps/docs/docs/development/dev-stack.md @@ -27,6 +27,14 @@ The first run creates `.dev/dev.env` and `apps/web/.env.local` with local-only secrets and URLs. The root `.env` used by the release/test-server path is not modified. +For local Docker agent containers, the generated config sets +`AGENT_DOWNLOAD_BASE_URL=http://dev-proxy`, +`CT_OPS_AGENT_CONTAINER_INGEST_ADDRESS=ingest-dev:9443`, and an empty +`CT_OPS_ENROLMENT_TOKEN=` placeholder. Paste a valid token into that placeholder +before running `deploy/scripts/create-agent-dev-container.sh`. The helper joins +the dev stack Docker network so these service names resolve inside the test +container without exposing the dev stack on your LAN. + The `web-migrate` and `web-dev` containers run `pnpm install --frozen-lockfile` inside Docker using named volumes for `node_modules`, so Linux container dependencies do not overwrite host dependencies. The `ingest-dev` container uses @@ -84,6 +92,15 @@ For browser-based agent install bundles, the app will use `AGENT_DOWNLOAD_BASE_URL=http://100.x.y.z:3000`, so the remote host must also be able to reach port `3000`. +For same-machine Docker agent containers, leave the stack in local mode and run: + +```bash +deploy/scripts/create-agent-dev-container.sh +``` + +The helper reads `.dev/dev.env`, creates one long-lived systemd Ubuntu +container, and runs the normal CT-Ops agent install script inside it. + Switch back to loopback-only mode with: ```bash diff --git a/deploy/docker/agent-dev-container/Dockerfile b/deploy/docker/agent-dev-container/Dockerfile new file mode 100644 index 00000000..876c8c3d --- /dev/null +++ b/deploy/docker/agent-dev-container/Dockerfile @@ -0,0 +1,23 @@ +# trivy:ignore:DS-0002 +# This image intentionally runs systemd as PID 1 and validates the root-only +# CT-Ops agent installer path inside an isolated local development container. +FROM ubuntu:24.04 + +ENV container=docker + +RUN apt-get update \ + && apt-get install -y --no-install-recommends \ + ca-certificates \ + curl \ + sudo \ + systemd \ + systemd-sysv \ + && apt-get clean \ + && rm -rf /var/lib/apt/lists/* + +STOPSIGNAL SIGRTMIN+3 + +HEALTHCHECK --interval=30s --timeout=5s --start-period=10s --retries=3 \ + CMD systemctl is-system-running --quiet || exit 1 + +CMD ["/sbin/init"] diff --git a/deploy/scripts/create-agent-dev-container.sh b/deploy/scripts/create-agent-dev-container.sh new file mode 100755 index 00000000..0b737ff9 --- /dev/null +++ b/deploy/scripts/create-agent-dev-container.sh @@ -0,0 +1,236 @@ +#!/usr/bin/env bash +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +REPO_ROOT="$(cd "${SCRIPT_DIR}/../.." && pwd)" +DOCKERFILE="${REPO_ROOT}/deploy/docker/agent-dev-container/Dockerfile" +DEV_ENV="${CT_OPS_DEV_ENV_FILE:-${REPO_ROOT}/.dev/dev.env}" + +IMAGE_TAG="${CT_OPS_AGENT_DEV_IMAGE:-ct-ops-agent-dev:ubuntu-24.04-systemd}" +APP_URL="${CT_OPS_AGENT_APP_URL:-http://dev-proxy}" +INGEST_ADDRESS="${CT_OPS_AGENT_INGEST_ADDRESS:-ingest-dev:9443}" +CONTAINER_NAME="${CT_OPS_AGENT_CONTAINER_NAME:-ctops-agent-$(date -u +%Y%m%d%H%M%S)-${RANDOM}}" +ENROLMENT_TOKEN="${CT_OPS_ENROLMENT_TOKEN:-}" +DOCKER_NETWORK="${CT_OPS_AGENT_DOCKER_NETWORK:-}" +SKIP_VERIFY="true" +INSTALL_AGENT="true" + +usage() { + cat <&2 + exit 1 +} + +require_command() { + command -v "$1" >/dev/null 2>&1 || die "$1 is required" +} + +read_env_var() { + local file="$1" + local key="$2" + [ -f "$file" ] || return 0 + sed -n "s/^${key}=//p" "$file" | tail -n1 +} + +load_dev_env_defaults() { + local project value + + if [ -z "${CT_OPS_AGENT_APP_URL:-}" ]; then + value="$(read_env_var "$DEV_ENV" AGENT_DOWNLOAD_BASE_URL)" + if [ -n "$value" ]; then + APP_URL="${value%/}" + fi + fi + + if [ -z "${CT_OPS_AGENT_INGEST_ADDRESS:-}" ]; then + value="$(read_env_var "$DEV_ENV" CT_OPS_AGENT_CONTAINER_INGEST_ADDRESS)" + if [ -n "$value" ]; then + INGEST_ADDRESS="$value" + fi + fi + + if [ -z "${CT_OPS_ENROLMENT_TOKEN:-}" ]; then + value="$(read_env_var "$DEV_ENV" CT_OPS_ENROLMENT_TOKEN)" + if [ -n "$value" ]; then + ENROLMENT_TOKEN="$value" + fi + fi + + if [ -z "${CT_OPS_AGENT_DOCKER_NETWORK:-}" ]; then + project="$(read_env_var "$DEV_ENV" COMPOSE_PROJECT_NAME)" + if [ -n "$project" ]; then + DOCKER_NETWORK="${project}_default" + fi + fi +} + +urlencode() { + local value="$1" + local i char out="" + for ((i = 0; i < ${#value}; i++)); do + char="${value:i:1}" + case "$char" in + [a-zA-Z0-9.~_-]) out+="$char" ;; + *) printf -v out '%s%%%02X' "$out" "'$char" ;; + esac + done + printf '%s' "$out" +} + +wait_for_systemd() { + local name="$1" + local attempt state + + for attempt in {1..30}; do + if docker exec "$name" test -d /run/systemd/system >/dev/null 2>&1; then + state="$(docker exec "$name" systemctl is-system-running 2>/dev/null || true)" + case "$state" in + running|degraded) + return 0 + ;; + esac + fi + sleep 1 + done + + docker logs "$name" >&2 || true + die "systemd did not become ready in container ${name}" +} + +load_dev_env_defaults + +while [ "$#" -gt 0 ]; do + case "$1" in + --token) + [ "$#" -ge 2 ] || die "--token requires a value" + ENROLMENT_TOKEN="$2" + shift 2 + ;; + --name) + [ "$#" -ge 2 ] || die "--name requires a value" + CONTAINER_NAME="$2" + shift 2 + ;; + --app-url) + [ "$#" -ge 2 ] || die "--app-url requires a value" + APP_URL="${2%/}" + shift 2 + ;; + --ingest) + [ "$#" -ge 2 ] || die "--ingest requires a value" + INGEST_ADDRESS="$2" + shift 2 + ;; + --image) + [ "$#" -ge 2 ] || die "--image requires a value" + IMAGE_TAG="$2" + shift 2 + ;; + --network) + [ "$#" -ge 2 ] || die "--network requires a value" + DOCKER_NETWORK="$2" + shift 2 + ;; + --no-skip-verify) + SKIP_VERIFY="false" + shift + ;; + --no-install) + INSTALL_AGENT="false" + shift + ;; + -h|--help) + usage + exit 0 + ;; + *) + die "unknown option: $1" + ;; + esac +done + +require_command docker + +if [ "$INSTALL_AGENT" = "true" ] && [ -z "$ENROLMENT_TOKEN" ]; then + die "provide an enrolment token with --token, CT_OPS_ENROLMENT_TOKEN, or ${DEV_ENV}" +fi + +if [ -n "$DOCKER_NETWORK" ] && ! docker network inspect "$DOCKER_NETWORK" >/dev/null 2>&1; then + die "Docker network '${DOCKER_NETWORK}' does not exist. Start the dev stack first with ./dev-stack.sh." +fi + +if docker ps -a --format '{{.Names}}' | grep -Fxq "$CONTAINER_NAME"; then + die "container already exists: ${CONTAINER_NAME}" +fi + +echo "Building ${IMAGE_TAG}..." +docker build -t "$IMAGE_TAG" -f "$DOCKERFILE" "$REPO_ROOT" + +echo "Starting ${CONTAINER_NAME}..." +docker_run_args=( + -d + --name "$CONTAINER_NAME" + --hostname "$CONTAINER_NAME" + --privileged + --cgroupns=host + --restart unless-stopped + --add-host host.docker.internal:host-gateway + --tmpfs /run + --tmpfs /run/lock + --volume /sys/fs/cgroup:/sys/fs/cgroup:rw +) +if [ -n "$DOCKER_NETWORK" ]; then + docker_run_args+=(--network "$DOCKER_NETWORK") +fi +docker_run_args+=("$IMAGE_TAG") +docker run "${docker_run_args[@]}" >/dev/null + +wait_for_systemd "$CONTAINER_NAME" + +if [ "$INSTALL_AGENT" = "true" ]; then + install_url="${APP_URL}/api/agent/install?ingest=$(urlencode "$INGEST_ADDRESS")" + if [ "$SKIP_VERIFY" = "true" ]; then + install_url="${install_url}&skip_verify=true" + fi + + echo "Installing CT-Ops agent in ${CONTAINER_NAME}..." + echo "Tip: the CT-Ops web app should have AGENT_DOWNLOAD_BASE_URL=${APP_URL} for container installs." + docker exec \ + -e CT_OPS_ENROLMENT_TOKEN="$ENROLMENT_TOKEN" \ + -e CT_OPS_AGENT_INSTALL_URL="$install_url" \ + "$CONTAINER_NAME" \ + sh -euc 'tmp="$(mktemp)"; curl -fsSLk "$CT_OPS_AGENT_INSTALL_URL" -o "$tmp"; sh "$tmp"; rm -f "$tmp"' +fi + +echo "" +echo "Container: ${CONTAINER_NAME}" +echo "Status: docker exec -it ${CONTAINER_NAME} systemctl status ct-ops-agent" +echo "Logs: docker exec -it ${CONTAINER_NAME} journalctl -u ct-ops-agent -f" diff --git a/deploy/scripts/test-create-agent-dev-container.sh b/deploy/scripts/test-create-agent-dev-container.sh new file mode 100755 index 00000000..70601d21 --- /dev/null +++ b/deploy/scripts/test-create-agent-dev-container.sh @@ -0,0 +1,131 @@ +#!/usr/bin/env bash +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +REPO_ROOT="$(cd "${SCRIPT_DIR}/../.." && pwd)" +SCRIPT="${REPO_ROOT}/deploy/scripts/create-agent-dev-container.sh" + +make_mock_docker() { + local dir="$1" + + cat > "${dir}/docker" <<'EOF' +#!/usr/bin/env bash +set -euo pipefail + +log="${DOCKER_MOCK_LOG:?DOCKER_MOCK_LOG is required}" +printf '%q ' "$@" >> "$log" +printf '\n' >> "$log" + +case "${1:-}" in + network) + if [ "${2:-}" = "inspect" ]; then + exit 0 + fi + ;; + ps) + exit 0 + ;; + build) + exit 0 + ;; + run) + exit 0 + ;; + exec) + if printf '%s\n' "$*" | grep -q 'test -d /run/systemd/system'; then + exit 0 + fi + if printf '%s\n' "$*" | grep -q 'systemctl is-system-running'; then + echo "running" + exit 0 + fi + exit 0 + ;; + logs) + exit 0 + ;; +esac + +echo "unexpected docker command: $*" >&2 +exit 99 +EOF + + chmod +x "${dir}/docker" +} + +main() { + local tmpdir mockbin log output dev_env + tmpdir="$(mktemp -d)" + trap 'rm -rf "'"$tmpdir"'"' EXIT + + mockbin="${tmpdir}/mockbin" + log="${tmpdir}/docker.log" + dev_env="${tmpdir}/dev.env" + mkdir -p "$mockbin" + make_mock_docker "$mockbin" + + cat > "$dev_env" <<'EOF' +AGENT_DOWNLOAD_BASE_URL=http://dev-env-host:3000 +CT_OPS_AGENT_CONTAINER_INGEST_ADDRESS=dev-env-host:9443 +CT_OPS_ENROLMENT_TOKEN=dev_env_token +COMPOSE_PROJECT_NAME=ct-ops-dev-test +EOF + + output="$( + DOCKER_MOCK_LOG="$log" \ + CT_OPS_DEV_ENV_FILE="$dev_env" \ + PATH="${mockbin}:/usr/bin:/bin:/usr/sbin:/sbin" \ + "$SCRIPT" \ + --name ctops-agent-test + )" + + if ! grep -q -- "build -t ct-ops-agent-dev:ubuntu-24.04-systemd" "$log"; then + echo "expected docker build command" >&2 + cat "$log" >&2 + exit 1 + fi + if ! grep -q -- "--name ctops-agent-test" "$log"; then + echo "expected named docker run command" >&2 + cat "$log" >&2 + exit 1 + fi + if ! grep -q -- "--add-host host.docker.internal:host-gateway" "$log"; then + echo "expected host gateway mapping" >&2 + cat "$log" >&2 + exit 1 + fi + if ! grep -q -- "--cgroupns=host" "$log"; then + echo "expected host cgroup namespace for systemd" >&2 + cat "$log" >&2 + exit 1 + fi + if ! grep -q -- "--network ct-ops-dev-test_default" "$log"; then + echo "expected dev stack Docker network" >&2 + cat "$log" >&2 + exit 1 + fi + if ! grep -Fq -- "CT_OPS_ENROLMENT_TOKEN=dev_env_token" "$log"; then + echo "expected enrolment token from dev env" >&2 + cat "$log" >&2 + exit 1 + fi + if ! grep -Fq -- "CT_OPS_AGENT_INSTALL_URL=http://dev-env-host:3000/api/agent/install\\?ingest=dev-env-host%3A9443\\&skip_verify=true" "$log"; then + echo "expected install URL with encoded ingest address and skip_verify" >&2 + cat "$log" >&2 + exit 1 + fi + if ! grep -Fq -- "sh -euc tmp=\\\"\\\$\\(mktemp\\)\\\"\\;\\ curl\\ -fsSLk\\ \\\"\\\$CT_OPS_AGENT_INSTALL_URL\\\"\\ -o\\ \\\"\\\$tmp\\\"\\;\\ sh\\ \\\"\\\$tmp\\\"\\;\\ rm\\ -f\\ \\\"\\\$tmp\\\"" "$log"; then + echo "expected non-pipeline installer execution" >&2 + cat "$log" >&2 + exit 1 + fi + if [[ "$output" != *"docker exec -it ctops-agent-test journalctl -u ct-ops-agent -f"* ]]; then + echo "expected log follow hint" >&2 + echo "$output" >&2 + exit 1 + fi + + echo "create-agent-dev-container.sh command assembly test passed" +} + +main "$@" diff --git a/deploy/scripts/test-dev-stack.sh b/deploy/scripts/test-dev-stack.sh index d7b99c89..ee754324 100755 --- a/deploy/scripts/test-dev-stack.sh +++ b/deploy/scripts/test-dev-stack.sh @@ -65,6 +65,9 @@ EOF expected_uid="$(id -u)" expected_gid="$(id -g)" assert_contains "${repo_dir}/.dev/dev.env" "BETTER_AUTH_URL=http://localhost:3000" + assert_contains "${repo_dir}/.dev/dev.env" "AGENT_DOWNLOAD_BASE_URL=http://dev-proxy" + assert_contains "${repo_dir}/.dev/dev.env" "CT_OPS_AGENT_CONTAINER_INGEST_ADDRESS=ingest-dev:9443" + assert_contains "${repo_dir}/.dev/dev.env" "CT_OPS_ENROLMENT_TOKEN=" assert_contains "${repo_dir}/.dev/dev.env" "CT_OPS_DEV_UID=${expected_uid}" assert_contains "${repo_dir}/.dev/dev.env" "CT_OPS_DEV_GID=${expected_gid}" assert_contains "${repo_dir}/.dev/dev.env" "PASSWORD_MANAGER_SESSION_COOKIE_SECURE=false" @@ -122,6 +125,7 @@ run_public_config_generation_test() { assert_contains "${repo_dir}/.dev/dev.env" "BETTER_AUTH_URL=http://192.0.2.10:3000" assert_contains "${repo_dir}/.dev/dev.env" "AGENT_DOWNLOAD_BASE_URL=http://192.0.2.10:3000" assert_contains "${repo_dir}/.dev/dev.env" "CT_OPS_DEV_AGENT_INGEST_ADDRESS=192.0.2.10:9443" + assert_contains "${repo_dir}/.dev/dev.env" "CT_OPS_AGENT_CONTAINER_INGEST_ADDRESS=192.0.2.10:9443" assert_contains "${repo_dir}/.dev/dev.env" "BETTER_AUTH_TRUSTED_ORIGINS=http://localhost:3000,http://127.0.0.1:3000,http://192.0.2.10:3000" assert_contains "${repo_dir}/apps/web/.env.local" "BETTER_AUTH_URL=http://192.0.2.10:3000" @@ -133,7 +137,9 @@ run_public_config_generation_test() { assert_contains "${repo_dir}/.dev/dev.env" "CT_OPS_DEV_BIND_ADDR=127.0.0.1" assert_contains "${repo_dir}/.dev/dev.env" "CT_OPS_DEV_PUBLIC_HOST=" assert_contains "${repo_dir}/.dev/dev.env" "BETTER_AUTH_URL=http://localhost:3000" + assert_contains "${repo_dir}/.dev/dev.env" "AGENT_DOWNLOAD_BASE_URL=http://dev-proxy" assert_contains "${repo_dir}/.dev/dev.env" "CT_OPS_DEV_AGENT_INGEST_ADDRESS=localhost:9443" + assert_contains "${repo_dir}/.dev/dev.env" "CT_OPS_AGENT_CONTAINER_INGEST_ADDRESS=ingest-dev:9443" } run_static_wiring_test() { diff --git a/dev-stack.sh b/dev-stack.sh index 8166d60e..d7d15eed 100755 --- a/dev-stack.sh +++ b/dev-stack.sh @@ -261,7 +261,7 @@ EOF } ensure_dev_env() { - local proxy_port next_port postgres_port ingest_http_port ingest_grpc_port app_url pm_image + local proxy_port next_port postgres_port ingest_http_port ingest_grpc_port app_url agent_download_base_url agent_container_ingest_address pm_image local bind_addr public_host public_enabled trusted_origins local private_key public_key keypair_output repaired_public_key @@ -309,11 +309,15 @@ ensure_dev_env() { fi fi app_url="http://${public_host}:${proxy_port}" + agent_download_base_url="$app_url" + agent_container_ingest_address="${public_host}:${ingest_grpc_port}" trusted_origins="http://localhost:${proxy_port},http://127.0.0.1:${proxy_port},${app_url}" else bind_addr="127.0.0.1" public_host="" app_url="http://localhost:${proxy_port}" + agent_download_base_url="http://dev-proxy" + agent_container_ingest_address="ingest-dev:9443" trusted_origins="$app_url" fi @@ -326,6 +330,7 @@ ensure_dev_env() { upsert_env_var "$DEV_ENV" CT_OPS_DEV_INGEST_GRPC_PORT "$ingest_grpc_port" upsert_env_var "$DEV_ENV" CT_OPS_DEV_APP_URL "$app_url" upsert_env_var "$DEV_ENV" CT_OPS_DEV_AGENT_INGEST_ADDRESS "${public_host:-localhost}:${ingest_grpc_port}" + upsert_env_var "$DEV_ENV" CT_OPS_AGENT_CONTAINER_INGEST_ADDRESS "$agent_container_ingest_address" upsert_env_var "$DEV_ENV" CT_OPS_DEV_UID "$(id -u)" upsert_env_var "$DEV_ENV" CT_OPS_DEV_GID "$(id -g)" if [ -z "$(read_env_var "$DEV_ENV" CT_OPS_DEV_NEXT_FLAGS)" ]; then @@ -350,7 +355,10 @@ ensure_dev_env() { upsert_env_var "$DEV_ENV" BETTER_AUTH_TRUSTED_ORIGINS "$trusted_origins" upsert_env_var "$DEV_ENV" REQUIRE_EMAIL_VERIFICATION "false" upsert_env_var "$DEV_ENV" CT_OPS_TRUST_PROXY_HEADERS "true" - upsert_env_var "$DEV_ENV" AGENT_DOWNLOAD_BASE_URL "$app_url" + upsert_env_var "$DEV_ENV" AGENT_DOWNLOAD_BASE_URL "$agent_download_base_url" + if [ -z "$(read_env_var "$DEV_ENV" CT_OPS_ENROLMENT_TOKEN)" ]; then + upsert_env_var "$DEV_ENV" CT_OPS_ENROLMENT_TOKEN "" + fi upsert_env_var "$DEV_ENV" INGEST_WS_URL "" if [ -z "$(read_env_var "$DEV_ENV" PASSWORD_MANAGER_DB_PASSWORD)" ]; then