From f23af109eac754542bc904a1913191868ea518b5 Mon Sep 17 00:00:00 2001 From: Codex Date: Fri, 15 May 2026 08:11:26 +0000 Subject: [PATCH 1/4] fix(agent): include token in enrollment view command --- .../settings/agents/agents-client.tsx | 21 ++++++++++--------- apps/web/lib/actions/agents-authz.test.mjs | 13 ++++++++++++ apps/web/lib/actions/agents-core.ts | 10 ++++----- apps/web/lib/actions/agents.ts | 4 ++-- .../e2e/settings/agent-enrolment.spec.ts | 2 ++ 5 files changed, 33 insertions(+), 17 deletions(-) diff --git a/apps/web/app/(dashboard)/settings/agents/agents-client.tsx b/apps/web/app/(dashboard)/settings/agents/agents-client.tsx index 53ac0f48..a1c8a328 100644 --- a/apps/web/app/(dashboard)/settings/agents/agents-client.tsx +++ b/apps/web/app/(dashboard)/settings/agents/agents-client.tsx @@ -41,7 +41,7 @@ import { revokeEnrolmentToken, } from '@/lib/actions/agents' import type { AgentEnrolmentToken } from '@/lib/db/schema' -import type { EnrolmentTokenSafe } from '@/lib/actions/agents-core' +import type { EnrolmentTokenForAdmin } from '@/lib/actions/agents-core' import { TagEditor, type EditorTag } from '@/components/shared/tag-editor' import { DEFAULT_ENROLMENT_TOKEN_EXPIRY_DAYS, @@ -60,7 +60,7 @@ const createTokenSchema = z.object({ type CreateTokenForm = z.infer interface AgentsSettingsClientProps { - initialTokens: EnrolmentTokenSafe[] + initialTokens: EnrolmentTokenForAdmin[] appUrl: string } @@ -84,7 +84,7 @@ function CopyButton({ text }: { text: string }) { ) } -function tokenStatus(token: EnrolmentTokenSafe): { label: string; className: string } { +function tokenStatus(token: EnrolmentTokenForAdmin): { label: string; className: string } { if (token.deletedAt) { return { label: 'Revoked', className: 'bg-red-100 text-red-800 border-red-200' } } @@ -105,7 +105,7 @@ export function AgentsSettingsClient({ const [showCreateDialog, setShowCreateDialog] = useState(false) const [newTokenValue, setNewTokenValue] = useState(null) const [newInstallCommand, setNewInstallCommand] = useState(null) - const [viewToken, setViewToken] = useState(null) + const [viewToken, setViewToken] = useState(null) const [showBundleDialog, setShowBundleDialog] = useState(false) const [tokenTags, setTokenTags] = useState([]) const [browserOrigin, setBrowserOrigin] = useState('') @@ -170,7 +170,7 @@ export function AgentsSettingsClient({ const onSubmit = handleSubmit((data) => createMutation.mutate(data)) const installCommandBaseUrl = appUrl || browserOrigin const viewInstallCommand = viewToken && installCommandBaseUrl - ? buildAgentInstallCommand(installCommandBaseUrl, viewToken.skipVerify) + ? buildAgentInstallCommand(installCommandBaseUrl, viewToken.skipVerify, viewToken.token) : null useEffect(() => { @@ -310,8 +310,8 @@ export function AgentsSettingsClient({ {viewToken && (

- The full token was shown only once when it was created. To use this token, - generate a new install bundle from the Download Agent Bundle dialog. + Use this command on each server you want to enrol. The token is passed through + the environment instead of being placed in the installer URL.

{viewInstallCommand && (
@@ -325,9 +325,10 @@ export function AgentsSettingsClient({
)}
-

Token hint

+

Raw token

- ········…{viewToken.tokenHint} + {viewToken.token} +
@@ -527,7 +528,7 @@ type TokenMode = 'create' | 'existing' | 'none' interface BundleDialogProps { open: boolean onOpenChange: (open: boolean) => void - activeTokens: EnrolmentTokenSafe[] + activeTokens: EnrolmentTokenForAdmin[] } function BundleDialog({ open, onOpenChange, activeTokens }: BundleDialogProps) { diff --git a/apps/web/lib/actions/agents-authz.test.mjs b/apps/web/lib/actions/agents-authz.test.mjs index 5048aceb..a8d4c889 100644 --- a/apps/web/lib/actions/agents-authz.test.mjs +++ b/apps/web/lib/actions/agents-authz.test.mjs @@ -30,3 +30,16 @@ test('privileged agent and host mutations require tooling access', () => { ) } }) + +test('listing enrolment tokens requires privileged access because plaintext tokens are returned', () => { + const start = agentsSource.lastIndexOf('export async function listEnrolmentTokens') + assert.notEqual(start, -1, 'expected listEnrolmentTokens to exist in agents-core.ts') + const next = agentsSource.indexOf('\nexport async function ', start + 1) + const segment = agentsSource.slice(start, next === -1 ? undefined : next) + + assert.match( + segment, + /requireInstance(?:Admin|Tooling)Access\(instanceId\)/, + 'listEnrolmentTokens must require privileged access before returning enrolment token secrets', + ) +}) diff --git a/apps/web/lib/actions/agents-core.ts b/apps/web/lib/actions/agents-core.ts index 0557df02..f45eedd2 100644 --- a/apps/web/lib/actions/agents-core.ts +++ b/apps/web/lib/actions/agents-core.ts @@ -645,21 +645,21 @@ export async function createEnrolmentToken( } } -export type EnrolmentTokenSafe = Omit & { +export type EnrolmentTokenForAdmin = Omit & { tokenHint: string } -export async function listEnrolmentTokens(instanceId: string): Promise { - await requireInstanceAccess(instanceId) +export async function listEnrolmentTokens(instanceId: string): Promise { + await requireInstanceAdminAccess(instanceId) const rows = await db.query.agentEnrolmentTokens.findMany({ where: and( eq(agentEnrolmentTokens.instanceId, instanceId), isNull(agentEnrolmentTokens.deletedAt), ), }) - return rows.map(({ token, tokenHash: _hash, ...rest }) => ({ + return rows.map(({ tokenHash: _hash, ...rest }) => ({ ...rest, - tokenHint: token.slice(-4), + tokenHint: rest.token.slice(-4), })) } diff --git a/apps/web/lib/actions/agents.ts b/apps/web/lib/actions/agents.ts index 94a0c828..42c13832 100644 --- a/apps/web/lib/actions/agents.ts +++ b/apps/web/lib/actions/agents.ts @@ -18,7 +18,7 @@ import { listEnrolmentTokens as listEnrolmentTokensCore, revokeEnrolmentToken as revokeEnrolmentTokenCore, uninstallAndDeleteHost as uninstallAndDeleteHostCore, - type EnrolmentTokenSafe, + type EnrolmentTokenForAdmin, type HeartbeatPoint, type HostListParams, type HostListResult, @@ -114,7 +114,7 @@ export async function createEnrolmentToken( return createEnrolmentTokenCore(resolveCurrentActionScope(session), input) } -export async function listEnrolmentTokens(): Promise { +export async function listEnrolmentTokens(): Promise { const session = await getRequiredSession() const currentScope = resolveOptionalActionScope(session) if (!currentScope) return [] diff --git a/apps/web/tests/e2e/settings/agent-enrolment.spec.ts b/apps/web/tests/e2e/settings/agent-enrolment.spec.ts index e9d7b898..62061575 100644 --- a/apps/web/tests/e2e/settings/agent-enrolment.spec.ts +++ b/apps/web/tests/e2e/settings/agent-enrolment.spec.ts @@ -148,6 +148,8 @@ test('admin can view the install URL for an existing enrolment token', async ({ const dialog = page.getByRole('dialog') await expect(dialog).toContainText('View Existing Token') await expect(page.getByTestId('agent-enrolment-view-install-command')).toContainText('/api/agent/install') + await expect(page.getByTestId('agent-enrolment-view-install-command')).toContainText('CT_OPS_ENROLMENT_TOKEN=') + await expect(page.getByTestId('agent-enrolment-view-install-command')).toContainText('view-existing-token-value') await expect(page.getByTestId('agent-enrolment-view-install-command')).not.toContainText('token=') }) From cfd13c73e318a91124f1663a38f2565a9de03685 Mon Sep 17 00:00:00 2001 From: Codex Date: Fri, 15 May 2026 09:52:32 +0000 Subject: [PATCH 2/4] chore(agent): add dev container installer helper --- deploy/docker/agent-dev-container/Dockerfile | 17 ++ deploy/scripts/create-agent-dev-container.sh | 176 ++++++++++++++++++ .../test-create-agent-dev-container.sh | 105 +++++++++++ 3 files changed, 298 insertions(+) create mode 100644 deploy/docker/agent-dev-container/Dockerfile create mode 100755 deploy/scripts/create-agent-dev-container.sh create mode 100755 deploy/scripts/test-create-agent-dev-container.sh diff --git a/deploy/docker/agent-dev-container/Dockerfile b/deploy/docker/agent-dev-container/Dockerfile new file mode 100644 index 00000000..1ae95fde --- /dev/null +++ b/deploy/docker/agent-dev-container/Dockerfile @@ -0,0 +1,17 @@ +FROM ubuntu:24.04 + +ENV container=docker + +RUN apt-get update \ + && apt-get install -y --no-install-recommends \ + ca-certificates \ + curl \ + sudo \ + systemd \ + systemd-sysv \ + && apt-get clean \ + && rm -rf /var/lib/apt/lists/* + +STOPSIGNAL SIGRTMIN+3 + +CMD ["/sbin/init"] diff --git a/deploy/scripts/create-agent-dev-container.sh b/deploy/scripts/create-agent-dev-container.sh new file mode 100755 index 00000000..ae7799c5 --- /dev/null +++ b/deploy/scripts/create-agent-dev-container.sh @@ -0,0 +1,176 @@ +#!/usr/bin/env bash +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +REPO_ROOT="$(cd "${SCRIPT_DIR}/../.." && pwd)" +DOCKERFILE="${REPO_ROOT}/deploy/docker/agent-dev-container/Dockerfile" + +IMAGE_TAG="${CT_OPS_AGENT_DEV_IMAGE:-ct-ops-agent-dev:ubuntu-24.04-systemd}" +APP_URL="${CT_OPS_AGENT_APP_URL:-http://host.docker.internal:3000}" +INGEST_ADDRESS="${CT_OPS_AGENT_INGEST_ADDRESS:-host.docker.internal:9443}" +CONTAINER_NAME="${CT_OPS_AGENT_CONTAINER_NAME:-ctops-agent-$(date -u +%Y%m%d%H%M%S)-${RANDOM}}" +ENROLMENT_TOKEN="${CT_OPS_ENROLMENT_TOKEN:-}" +SKIP_VERIFY="true" +INSTALL_AGENT="true" + +usage() { + cat <&2 + exit 1 +} + +require_command() { + command -v "$1" >/dev/null 2>&1 || die "$1 is required" +} + +urlencode() { + local value="$1" + local i char out="" + for ((i = 0; i < ${#value}; i++)); do + char="${value:i:1}" + case "$char" in + [a-zA-Z0-9.~_-]) out+="$char" ;; + *) printf -v out '%s%%%02X' "$out" "'$char" ;; + esac + done + printf '%s' "$out" +} + +wait_for_systemd() { + local name="$1" + local attempt state + + for attempt in {1..30}; do + if docker exec "$name" test -d /run/systemd/system >/dev/null 2>&1; then + state="$(docker exec "$name" systemctl is-system-running 2>/dev/null || true)" + case "$state" in + running|degraded) + return 0 + ;; + esac + fi + sleep 1 + done + + docker logs "$name" >&2 || true + die "systemd did not become ready in container ${name}" +} + +while [ "$#" -gt 0 ]; do + case "$1" in + --token) + [ "$#" -ge 2 ] || die "--token requires a value" + ENROLMENT_TOKEN="$2" + shift 2 + ;; + --name) + [ "$#" -ge 2 ] || die "--name requires a value" + CONTAINER_NAME="$2" + shift 2 + ;; + --app-url) + [ "$#" -ge 2 ] || die "--app-url requires a value" + APP_URL="${2%/}" + shift 2 + ;; + --ingest) + [ "$#" -ge 2 ] || die "--ingest requires a value" + INGEST_ADDRESS="$2" + shift 2 + ;; + --image) + [ "$#" -ge 2 ] || die "--image requires a value" + IMAGE_TAG="$2" + shift 2 + ;; + --no-skip-verify) + SKIP_VERIFY="false" + shift + ;; + --no-install) + INSTALL_AGENT="false" + shift + ;; + -h|--help) + usage + exit 0 + ;; + *) + die "unknown option: $1" + ;; + esac +done + +require_command docker + +if [ "$INSTALL_AGENT" = "true" ] && [ -z "$ENROLMENT_TOKEN" ]; then + die "provide an enrolment token with --token or CT_OPS_ENROLMENT_TOKEN" +fi + +if docker ps -a --format '{{.Names}}' | grep -Fxq "$CONTAINER_NAME"; then + die "container already exists: ${CONTAINER_NAME}" +fi + +echo "Building ${IMAGE_TAG}..." +docker build -t "$IMAGE_TAG" -f "$DOCKERFILE" "$REPO_ROOT" + +echo "Starting ${CONTAINER_NAME}..." +docker run -d \ + --name "$CONTAINER_NAME" \ + --hostname "$CONTAINER_NAME" \ + --privileged \ + --cgroupns=host \ + --restart unless-stopped \ + --add-host host.docker.internal:host-gateway \ + --tmpfs /run \ + --tmpfs /run/lock \ + --volume /sys/fs/cgroup:/sys/fs/cgroup:rw \ + "$IMAGE_TAG" >/dev/null + +wait_for_systemd "$CONTAINER_NAME" + +if [ "$INSTALL_AGENT" = "true" ]; then + install_url="${APP_URL}/api/agent/install?ingest=$(urlencode "$INGEST_ADDRESS")" + if [ "$SKIP_VERIFY" = "true" ]; then + install_url="${install_url}&skip_verify=true" + fi + + echo "Installing CT-Ops agent in ${CONTAINER_NAME}..." + echo "Tip: the CT-Ops web app should have AGENT_DOWNLOAD_BASE_URL=${APP_URL} for container installs." + docker exec \ + -e CT_OPS_ENROLMENT_TOKEN="$ENROLMENT_TOKEN" \ + -e CT_OPS_AGENT_INSTALL_URL="$install_url" \ + "$CONTAINER_NAME" \ + sh -lc 'curl -fsSLk "$CT_OPS_AGENT_INSTALL_URL" | sh' +fi + +echo "" +echo "Container: ${CONTAINER_NAME}" +echo "Status: docker exec -it ${CONTAINER_NAME} systemctl status ct-ops-agent" +echo "Logs: docker exec -it ${CONTAINER_NAME} journalctl -u ct-ops-agent -f" diff --git a/deploy/scripts/test-create-agent-dev-container.sh b/deploy/scripts/test-create-agent-dev-container.sh new file mode 100755 index 00000000..9ed0e1a5 --- /dev/null +++ b/deploy/scripts/test-create-agent-dev-container.sh @@ -0,0 +1,105 @@ +#!/usr/bin/env bash +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +REPO_ROOT="$(cd "${SCRIPT_DIR}/../.." && pwd)" +SCRIPT="${REPO_ROOT}/deploy/scripts/create-agent-dev-container.sh" + +make_mock_docker() { + local dir="$1" + + cat > "${dir}/docker" <<'EOF' +#!/usr/bin/env bash +set -euo pipefail + +log="${DOCKER_MOCK_LOG:?DOCKER_MOCK_LOG is required}" +printf '%q ' "$@" >> "$log" +printf '\n' >> "$log" + +case "${1:-}" in + ps) + exit 0 + ;; + build) + exit 0 + ;; + run) + exit 0 + ;; + exec) + if printf '%s\n' "$*" | grep -q 'test -d /run/systemd/system'; then + exit 0 + fi + if printf '%s\n' "$*" | grep -q 'systemctl is-system-running'; then + echo "running" + exit 0 + fi + exit 0 + ;; + logs) + exit 0 + ;; +esac + +echo "unexpected docker command: $*" >&2 +exit 99 +EOF + + chmod +x "${dir}/docker" +} + +main() { + local tmpdir mockbin log output + tmpdir="$(mktemp -d)" + trap 'rm -rf "'"$tmpdir"'"' EXIT + + mockbin="${tmpdir}/mockbin" + log="${tmpdir}/docker.log" + mkdir -p "$mockbin" + make_mock_docker "$mockbin" + + output="$( + DOCKER_MOCK_LOG="$log" \ + PATH="${mockbin}:/usr/bin:/bin:/usr/sbin:/sbin" \ + "$SCRIPT" \ + --token tok_test \ + --name ctops-agent-test \ + --app-url http://host.docker.internal:3000 \ + --ingest host.docker.internal:9443 + )" + + if ! grep -q -- "build -t ct-ops-agent-dev:ubuntu-24.04-systemd" "$log"; then + echo "expected docker build command" >&2 + cat "$log" >&2 + exit 1 + fi + if ! grep -q -- "--name ctops-agent-test" "$log"; then + echo "expected named docker run command" >&2 + cat "$log" >&2 + exit 1 + fi + if ! grep -q -- "--add-host host.docker.internal:host-gateway" "$log"; then + echo "expected host gateway mapping" >&2 + cat "$log" >&2 + exit 1 + fi + if ! grep -q -- "--cgroupns=host" "$log"; then + echo "expected host cgroup namespace for systemd" >&2 + cat "$log" >&2 + exit 1 + fi + if ! grep -Fq -- "CT_OPS_AGENT_INSTALL_URL=http://host.docker.internal:3000/api/agent/install\\?ingest=host.docker.internal%3A9443\\&skip_verify=true" "$log"; then + echo "expected install URL with encoded ingest address and skip_verify" >&2 + cat "$log" >&2 + exit 1 + fi + if [[ "$output" != *"docker exec -it ctops-agent-test journalctl -u ct-ops-agent -f"* ]]; then + echo "expected log follow hint" >&2 + echo "$output" >&2 + exit 1 + fi + + echo "create-agent-dev-container.sh command assembly test passed" +} + +main "$@" From 594b4fd7af630ff64621b20274cca7af00933268 Mon Sep 17 00:00:00 2001 From: Codex Date: Fri, 15 May 2026 10:02:00 +0000 Subject: [PATCH 3/4] chore(dev): seed agent container env defaults --- apps/docs/docs/development/dev-stack.md | 14 +++++++ deploy/scripts/create-agent-dev-container.sh | 37 ++++++++++++++++++- .../test-create-agent-dev-container.sh | 22 ++++++++--- deploy/scripts/test-dev-stack.sh | 6 +++ dev-stack.sh | 12 +++++- 5 files changed, 82 insertions(+), 9 deletions(-) diff --git a/apps/docs/docs/development/dev-stack.md b/apps/docs/docs/development/dev-stack.md index 1b78a3a0..2f4f9040 100644 --- a/apps/docs/docs/development/dev-stack.md +++ b/apps/docs/docs/development/dev-stack.md @@ -27,6 +27,11 @@ The first run creates `.dev/dev.env` and `apps/web/.env.local` with local-only secrets and URLs. The root `.env` used by the release/test-server path is not modified. +For local Docker agent containers, the generated config sets +`AGENT_DOWNLOAD_BASE_URL=http://host.docker.internal:3000` and adds an empty +`CT_OPS_ENROLMENT_TOKEN=` placeholder. Paste a valid token into that placeholder +before running `deploy/scripts/create-agent-dev-container.sh`. + The `web-migrate` and `web-dev` containers run `pnpm install --frozen-lockfile` inside Docker using named volumes for `node_modules`, so Linux container dependencies do not overwrite host dependencies. The `ingest-dev` container uses @@ -84,6 +89,15 @@ For browser-based agent install bundles, the app will use `AGENT_DOWNLOAD_BASE_URL=http://100.x.y.z:3000`, so the remote host must also be able to reach port `3000`. +For same-machine Docker agent containers, leave the stack in local mode and run: + +```bash +deploy/scripts/create-agent-dev-container.sh +``` + +The helper reads `.dev/dev.env`, creates one long-lived systemd Ubuntu +container, and runs the normal CT-Ops agent install script inside it. + Switch back to loopback-only mode with: ```bash diff --git a/deploy/scripts/create-agent-dev-container.sh b/deploy/scripts/create-agent-dev-container.sh index ae7799c5..3bc70c0b 100755 --- a/deploy/scripts/create-agent-dev-container.sh +++ b/deploy/scripts/create-agent-dev-container.sh @@ -4,6 +4,7 @@ set -euo pipefail SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" REPO_ROOT="$(cd "${SCRIPT_DIR}/../.." && pwd)" DOCKERFILE="${REPO_ROOT}/deploy/docker/agent-dev-container/Dockerfile" +DEV_ENV="${CT_OPS_DEV_ENV_FILE:-${REPO_ROOT}/.dev/dev.env}" IMAGE_TAG="${CT_OPS_AGENT_DEV_IMAGE:-ct-ops-agent-dev:ubuntu-24.04-systemd}" APP_URL="${CT_OPS_AGENT_APP_URL:-http://host.docker.internal:3000}" @@ -49,6 +50,38 @@ require_command() { command -v "$1" >/dev/null 2>&1 || die "$1 is required" } +read_env_var() { + local file="$1" + local key="$2" + [ -f "$file" ] || return 0 + sed -n "s/^${key}=//p" "$file" | tail -n1 +} + +load_dev_env_defaults() { + local value + + if [ -z "${CT_OPS_AGENT_APP_URL:-}" ]; then + value="$(read_env_var "$DEV_ENV" AGENT_DOWNLOAD_BASE_URL)" + if [ -n "$value" ]; then + APP_URL="${value%/}" + fi + fi + + if [ -z "${CT_OPS_AGENT_INGEST_ADDRESS:-}" ]; then + value="$(read_env_var "$DEV_ENV" CT_OPS_AGENT_CONTAINER_INGEST_ADDRESS)" + if [ -n "$value" ]; then + INGEST_ADDRESS="$value" + fi + fi + + if [ -z "${CT_OPS_ENROLMENT_TOKEN:-}" ]; then + value="$(read_env_var "$DEV_ENV" CT_OPS_ENROLMENT_TOKEN)" + if [ -n "$value" ]; then + ENROLMENT_TOKEN="$value" + fi + fi +} + urlencode() { local value="$1" local i char out="" @@ -82,6 +115,8 @@ wait_for_systemd() { die "systemd did not become ready in container ${name}" } +load_dev_env_defaults + while [ "$#" -gt 0 ]; do case "$1" in --token) @@ -130,7 +165,7 @@ done require_command docker if [ "$INSTALL_AGENT" = "true" ] && [ -z "$ENROLMENT_TOKEN" ]; then - die "provide an enrolment token with --token or CT_OPS_ENROLMENT_TOKEN" + die "provide an enrolment token with --token, CT_OPS_ENROLMENT_TOKEN, or ${DEV_ENV}" fi if docker ps -a --format '{{.Names}}' | grep -Fxq "$CONTAINER_NAME"; then diff --git a/deploy/scripts/test-create-agent-dev-container.sh b/deploy/scripts/test-create-agent-dev-container.sh index 9ed0e1a5..65bc7085 100755 --- a/deploy/scripts/test-create-agent-dev-container.sh +++ b/deploy/scripts/test-create-agent-dev-container.sh @@ -49,23 +49,28 @@ EOF } main() { - local tmpdir mockbin log output + local tmpdir mockbin log output dev_env tmpdir="$(mktemp -d)" trap 'rm -rf "'"$tmpdir"'"' EXIT mockbin="${tmpdir}/mockbin" log="${tmpdir}/docker.log" + dev_env="${tmpdir}/dev.env" mkdir -p "$mockbin" make_mock_docker "$mockbin" + cat > "$dev_env" <<'EOF' +AGENT_DOWNLOAD_BASE_URL=http://dev-env-host:3000 +CT_OPS_AGENT_CONTAINER_INGEST_ADDRESS=dev-env-host:9443 +CT_OPS_ENROLMENT_TOKEN=dev_env_token +EOF + output="$( DOCKER_MOCK_LOG="$log" \ + CT_OPS_DEV_ENV_FILE="$dev_env" \ PATH="${mockbin}:/usr/bin:/bin:/usr/sbin:/sbin" \ "$SCRIPT" \ - --token tok_test \ - --name ctops-agent-test \ - --app-url http://host.docker.internal:3000 \ - --ingest host.docker.internal:9443 + --name ctops-agent-test )" if ! grep -q -- "build -t ct-ops-agent-dev:ubuntu-24.04-systemd" "$log"; then @@ -88,7 +93,12 @@ main() { cat "$log" >&2 exit 1 fi - if ! grep -Fq -- "CT_OPS_AGENT_INSTALL_URL=http://host.docker.internal:3000/api/agent/install\\?ingest=host.docker.internal%3A9443\\&skip_verify=true" "$log"; then + if ! grep -Fq -- "CT_OPS_ENROLMENT_TOKEN=dev_env_token" "$log"; then + echo "expected enrolment token from dev env" >&2 + cat "$log" >&2 + exit 1 + fi + if ! grep -Fq -- "CT_OPS_AGENT_INSTALL_URL=http://dev-env-host:3000/api/agent/install\\?ingest=dev-env-host%3A9443\\&skip_verify=true" "$log"; then echo "expected install URL with encoded ingest address and skip_verify" >&2 cat "$log" >&2 exit 1 diff --git a/deploy/scripts/test-dev-stack.sh b/deploy/scripts/test-dev-stack.sh index d7b99c89..77fe6f6c 100755 --- a/deploy/scripts/test-dev-stack.sh +++ b/deploy/scripts/test-dev-stack.sh @@ -65,6 +65,9 @@ EOF expected_uid="$(id -u)" expected_gid="$(id -g)" assert_contains "${repo_dir}/.dev/dev.env" "BETTER_AUTH_URL=http://localhost:3000" + assert_contains "${repo_dir}/.dev/dev.env" "AGENT_DOWNLOAD_BASE_URL=http://host.docker.internal:3000" + assert_contains "${repo_dir}/.dev/dev.env" "CT_OPS_AGENT_CONTAINER_INGEST_ADDRESS=host.docker.internal:9443" + assert_contains "${repo_dir}/.dev/dev.env" "CT_OPS_ENROLMENT_TOKEN=" assert_contains "${repo_dir}/.dev/dev.env" "CT_OPS_DEV_UID=${expected_uid}" assert_contains "${repo_dir}/.dev/dev.env" "CT_OPS_DEV_GID=${expected_gid}" assert_contains "${repo_dir}/.dev/dev.env" "PASSWORD_MANAGER_SESSION_COOKIE_SECURE=false" @@ -122,6 +125,7 @@ run_public_config_generation_test() { assert_contains "${repo_dir}/.dev/dev.env" "BETTER_AUTH_URL=http://192.0.2.10:3000" assert_contains "${repo_dir}/.dev/dev.env" "AGENT_DOWNLOAD_BASE_URL=http://192.0.2.10:3000" assert_contains "${repo_dir}/.dev/dev.env" "CT_OPS_DEV_AGENT_INGEST_ADDRESS=192.0.2.10:9443" + assert_contains "${repo_dir}/.dev/dev.env" "CT_OPS_AGENT_CONTAINER_INGEST_ADDRESS=192.0.2.10:9443" assert_contains "${repo_dir}/.dev/dev.env" "BETTER_AUTH_TRUSTED_ORIGINS=http://localhost:3000,http://127.0.0.1:3000,http://192.0.2.10:3000" assert_contains "${repo_dir}/apps/web/.env.local" "BETTER_AUTH_URL=http://192.0.2.10:3000" @@ -133,7 +137,9 @@ run_public_config_generation_test() { assert_contains "${repo_dir}/.dev/dev.env" "CT_OPS_DEV_BIND_ADDR=127.0.0.1" assert_contains "${repo_dir}/.dev/dev.env" "CT_OPS_DEV_PUBLIC_HOST=" assert_contains "${repo_dir}/.dev/dev.env" "BETTER_AUTH_URL=http://localhost:3000" + assert_contains "${repo_dir}/.dev/dev.env" "AGENT_DOWNLOAD_BASE_URL=http://host.docker.internal:3000" assert_contains "${repo_dir}/.dev/dev.env" "CT_OPS_DEV_AGENT_INGEST_ADDRESS=localhost:9443" + assert_contains "${repo_dir}/.dev/dev.env" "CT_OPS_AGENT_CONTAINER_INGEST_ADDRESS=host.docker.internal:9443" } run_static_wiring_test() { diff --git a/dev-stack.sh b/dev-stack.sh index 8166d60e..36448d09 100755 --- a/dev-stack.sh +++ b/dev-stack.sh @@ -261,7 +261,7 @@ EOF } ensure_dev_env() { - local proxy_port next_port postgres_port ingest_http_port ingest_grpc_port app_url pm_image + local proxy_port next_port postgres_port ingest_http_port ingest_grpc_port app_url agent_download_base_url agent_container_ingest_address pm_image local bind_addr public_host public_enabled trusted_origins local private_key public_key keypair_output repaired_public_key @@ -309,11 +309,15 @@ ensure_dev_env() { fi fi app_url="http://${public_host}:${proxy_port}" + agent_download_base_url="$app_url" + agent_container_ingest_address="${public_host}:${ingest_grpc_port}" trusted_origins="http://localhost:${proxy_port},http://127.0.0.1:${proxy_port},${app_url}" else bind_addr="127.0.0.1" public_host="" app_url="http://localhost:${proxy_port}" + agent_download_base_url="http://host.docker.internal:${proxy_port}" + agent_container_ingest_address="host.docker.internal:${ingest_grpc_port}" trusted_origins="$app_url" fi @@ -326,6 +330,7 @@ ensure_dev_env() { upsert_env_var "$DEV_ENV" CT_OPS_DEV_INGEST_GRPC_PORT "$ingest_grpc_port" upsert_env_var "$DEV_ENV" CT_OPS_DEV_APP_URL "$app_url" upsert_env_var "$DEV_ENV" CT_OPS_DEV_AGENT_INGEST_ADDRESS "${public_host:-localhost}:${ingest_grpc_port}" + upsert_env_var "$DEV_ENV" CT_OPS_AGENT_CONTAINER_INGEST_ADDRESS "$agent_container_ingest_address" upsert_env_var "$DEV_ENV" CT_OPS_DEV_UID "$(id -u)" upsert_env_var "$DEV_ENV" CT_OPS_DEV_GID "$(id -g)" if [ -z "$(read_env_var "$DEV_ENV" CT_OPS_DEV_NEXT_FLAGS)" ]; then @@ -350,7 +355,10 @@ ensure_dev_env() { upsert_env_var "$DEV_ENV" BETTER_AUTH_TRUSTED_ORIGINS "$trusted_origins" upsert_env_var "$DEV_ENV" REQUIRE_EMAIL_VERIFICATION "false" upsert_env_var "$DEV_ENV" CT_OPS_TRUST_PROXY_HEADERS "true" - upsert_env_var "$DEV_ENV" AGENT_DOWNLOAD_BASE_URL "$app_url" + upsert_env_var "$DEV_ENV" AGENT_DOWNLOAD_BASE_URL "$agent_download_base_url" + if [ -z "$(read_env_var "$DEV_ENV" CT_OPS_ENROLMENT_TOKEN)" ]; then + upsert_env_var "$DEV_ENV" CT_OPS_ENROLMENT_TOKEN "" + fi upsert_env_var "$DEV_ENV" INGEST_WS_URL "" if [ -z "$(read_env_var "$DEV_ENV" PASSWORD_MANAGER_DB_PASSWORD)" ]; then From 60f7abb807482928605395d16a10a3d37a5343a8 Mon Sep 17 00:00:00 2001 From: Codex Date: Fri, 15 May 2026 10:29:40 +0000 Subject: [PATCH 4/4] fix(dev): connect agent containers to dev network --- apps/docs/docs/development/dev-stack.md | 7 ++- deploy/scripts/create-agent-dev-container.sh | 57 +++++++++++++------ .../test-create-agent-dev-container.sh | 16 ++++++ deploy/scripts/test-dev-stack.sh | 8 +-- dev-stack.sh | 4 +- 5 files changed, 68 insertions(+), 24 deletions(-) diff --git a/apps/docs/docs/development/dev-stack.md b/apps/docs/docs/development/dev-stack.md index 2f4f9040..1937ca9c 100644 --- a/apps/docs/docs/development/dev-stack.md +++ b/apps/docs/docs/development/dev-stack.md @@ -28,9 +28,12 @@ secrets and URLs. The root `.env` used by the release/test-server path is not modified. For local Docker agent containers, the generated config sets -`AGENT_DOWNLOAD_BASE_URL=http://host.docker.internal:3000` and adds an empty +`AGENT_DOWNLOAD_BASE_URL=http://dev-proxy`, +`CT_OPS_AGENT_CONTAINER_INGEST_ADDRESS=ingest-dev:9443`, and an empty `CT_OPS_ENROLMENT_TOKEN=` placeholder. Paste a valid token into that placeholder -before running `deploy/scripts/create-agent-dev-container.sh`. +before running `deploy/scripts/create-agent-dev-container.sh`. The helper joins +the dev stack Docker network so these service names resolve inside the test +container without exposing the dev stack on your LAN. The `web-migrate` and `web-dev` containers run `pnpm install --frozen-lockfile` inside Docker using named volumes for `node_modules`, so Linux container diff --git a/deploy/scripts/create-agent-dev-container.sh b/deploy/scripts/create-agent-dev-container.sh index 3bc70c0b..0b737ff9 100755 --- a/deploy/scripts/create-agent-dev-container.sh +++ b/deploy/scripts/create-agent-dev-container.sh @@ -7,10 +7,11 @@ DOCKERFILE="${REPO_ROOT}/deploy/docker/agent-dev-container/Dockerfile" DEV_ENV="${CT_OPS_DEV_ENV_FILE:-${REPO_ROOT}/.dev/dev.env}" IMAGE_TAG="${CT_OPS_AGENT_DEV_IMAGE:-ct-ops-agent-dev:ubuntu-24.04-systemd}" -APP_URL="${CT_OPS_AGENT_APP_URL:-http://host.docker.internal:3000}" -INGEST_ADDRESS="${CT_OPS_AGENT_INGEST_ADDRESS:-host.docker.internal:9443}" +APP_URL="${CT_OPS_AGENT_APP_URL:-http://dev-proxy}" +INGEST_ADDRESS="${CT_OPS_AGENT_INGEST_ADDRESS:-ingest-dev:9443}" CONTAINER_NAME="${CT_OPS_AGENT_CONTAINER_NAME:-ctops-agent-$(date -u +%Y%m%d%H%M%S)-${RANDOM}}" ENROLMENT_TOKEN="${CT_OPS_ENROLMENT_TOKEN:-}" +DOCKER_NETWORK="${CT_OPS_AGENT_DOCKER_NETWORK:-}" SKIP_VERIFY="true" INSTALL_AGENT="true" @@ -30,6 +31,8 @@ Options: Default: ${INGEST_ADDRESS} --image TAG Local image tag to build/use. Default: ${IMAGE_TAG} + --network NAME Docker network containing the dev stack. + Default: COMPOSE_PROJECT_NAME_default from ${DEV_ENV} --no-skip-verify Do not request TLS skip verification in the install script. --no-install Create the container but do not run the agent installer. -h, --help Show this help. @@ -37,7 +40,7 @@ Options: Examples: CT_OPS_ENROLMENT_TOKEN=tok_123 $0 $0 --token tok_123 --name ctops-agent-a - $0 --token tok_123 --app-url http://host.docker.internal:3000 --ingest host.docker.internal:9443 + $0 --token tok_123 --app-url http://dev-proxy --ingest ingest-dev:9443 EOF } @@ -58,7 +61,7 @@ read_env_var() { } load_dev_env_defaults() { - local value + local project value if [ -z "${CT_OPS_AGENT_APP_URL:-}" ]; then value="$(read_env_var "$DEV_ENV" AGENT_DOWNLOAD_BASE_URL)" @@ -80,6 +83,13 @@ load_dev_env_defaults() { ENROLMENT_TOKEN="$value" fi fi + + if [ -z "${CT_OPS_AGENT_DOCKER_NETWORK:-}" ]; then + project="$(read_env_var "$DEV_ENV" COMPOSE_PROJECT_NAME)" + if [ -n "$project" ]; then + DOCKER_NETWORK="${project}_default" + fi + fi } urlencode() { @@ -144,6 +154,11 @@ while [ "$#" -gt 0 ]; do IMAGE_TAG="$2" shift 2 ;; + --network) + [ "$#" -ge 2 ] || die "--network requires a value" + DOCKER_NETWORK="$2" + shift 2 + ;; --no-skip-verify) SKIP_VERIFY="false" shift @@ -168,6 +183,10 @@ if [ "$INSTALL_AGENT" = "true" ] && [ -z "$ENROLMENT_TOKEN" ]; then die "provide an enrolment token with --token, CT_OPS_ENROLMENT_TOKEN, or ${DEV_ENV}" fi +if [ -n "$DOCKER_NETWORK" ] && ! docker network inspect "$DOCKER_NETWORK" >/dev/null 2>&1; then + die "Docker network '${DOCKER_NETWORK}' does not exist. Start the dev stack first with ./dev-stack.sh." +fi + if docker ps -a --format '{{.Names}}' | grep -Fxq "$CONTAINER_NAME"; then die "container already exists: ${CONTAINER_NAME}" fi @@ -176,17 +195,23 @@ echo "Building ${IMAGE_TAG}..." docker build -t "$IMAGE_TAG" -f "$DOCKERFILE" "$REPO_ROOT" echo "Starting ${CONTAINER_NAME}..." -docker run -d \ - --name "$CONTAINER_NAME" \ - --hostname "$CONTAINER_NAME" \ - --privileged \ - --cgroupns=host \ - --restart unless-stopped \ - --add-host host.docker.internal:host-gateway \ - --tmpfs /run \ - --tmpfs /run/lock \ - --volume /sys/fs/cgroup:/sys/fs/cgroup:rw \ - "$IMAGE_TAG" >/dev/null +docker_run_args=( + -d + --name "$CONTAINER_NAME" + --hostname "$CONTAINER_NAME" + --privileged + --cgroupns=host + --restart unless-stopped + --add-host host.docker.internal:host-gateway + --tmpfs /run + --tmpfs /run/lock + --volume /sys/fs/cgroup:/sys/fs/cgroup:rw +) +if [ -n "$DOCKER_NETWORK" ]; then + docker_run_args+=(--network "$DOCKER_NETWORK") +fi +docker_run_args+=("$IMAGE_TAG") +docker run "${docker_run_args[@]}" >/dev/null wait_for_systemd "$CONTAINER_NAME" @@ -202,7 +227,7 @@ if [ "$INSTALL_AGENT" = "true" ]; then -e CT_OPS_ENROLMENT_TOKEN="$ENROLMENT_TOKEN" \ -e CT_OPS_AGENT_INSTALL_URL="$install_url" \ "$CONTAINER_NAME" \ - sh -lc 'curl -fsSLk "$CT_OPS_AGENT_INSTALL_URL" | sh' + sh -euc 'tmp="$(mktemp)"; curl -fsSLk "$CT_OPS_AGENT_INSTALL_URL" -o "$tmp"; sh "$tmp"; rm -f "$tmp"' fi echo "" diff --git a/deploy/scripts/test-create-agent-dev-container.sh b/deploy/scripts/test-create-agent-dev-container.sh index 65bc7085..70601d21 100755 --- a/deploy/scripts/test-create-agent-dev-container.sh +++ b/deploy/scripts/test-create-agent-dev-container.sh @@ -17,6 +17,11 @@ printf '%q ' "$@" >> "$log" printf '\n' >> "$log" case "${1:-}" in + network) + if [ "${2:-}" = "inspect" ]; then + exit 0 + fi + ;; ps) exit 0 ;; @@ -63,6 +68,7 @@ main() { AGENT_DOWNLOAD_BASE_URL=http://dev-env-host:3000 CT_OPS_AGENT_CONTAINER_INGEST_ADDRESS=dev-env-host:9443 CT_OPS_ENROLMENT_TOKEN=dev_env_token +COMPOSE_PROJECT_NAME=ct-ops-dev-test EOF output="$( @@ -93,6 +99,11 @@ EOF cat "$log" >&2 exit 1 fi + if ! grep -q -- "--network ct-ops-dev-test_default" "$log"; then + echo "expected dev stack Docker network" >&2 + cat "$log" >&2 + exit 1 + fi if ! grep -Fq -- "CT_OPS_ENROLMENT_TOKEN=dev_env_token" "$log"; then echo "expected enrolment token from dev env" >&2 cat "$log" >&2 @@ -103,6 +114,11 @@ EOF cat "$log" >&2 exit 1 fi + if ! grep -Fq -- "sh -euc tmp=\\\"\\\$\\(mktemp\\)\\\"\\;\\ curl\\ -fsSLk\\ \\\"\\\$CT_OPS_AGENT_INSTALL_URL\\\"\\ -o\\ \\\"\\\$tmp\\\"\\;\\ sh\\ \\\"\\\$tmp\\\"\\;\\ rm\\ -f\\ \\\"\\\$tmp\\\"" "$log"; then + echo "expected non-pipeline installer execution" >&2 + cat "$log" >&2 + exit 1 + fi if [[ "$output" != *"docker exec -it ctops-agent-test journalctl -u ct-ops-agent -f"* ]]; then echo "expected log follow hint" >&2 echo "$output" >&2 diff --git a/deploy/scripts/test-dev-stack.sh b/deploy/scripts/test-dev-stack.sh index 77fe6f6c..ee754324 100755 --- a/deploy/scripts/test-dev-stack.sh +++ b/deploy/scripts/test-dev-stack.sh @@ -65,8 +65,8 @@ EOF expected_uid="$(id -u)" expected_gid="$(id -g)" assert_contains "${repo_dir}/.dev/dev.env" "BETTER_AUTH_URL=http://localhost:3000" - assert_contains "${repo_dir}/.dev/dev.env" "AGENT_DOWNLOAD_BASE_URL=http://host.docker.internal:3000" - assert_contains "${repo_dir}/.dev/dev.env" "CT_OPS_AGENT_CONTAINER_INGEST_ADDRESS=host.docker.internal:9443" + assert_contains "${repo_dir}/.dev/dev.env" "AGENT_DOWNLOAD_BASE_URL=http://dev-proxy" + assert_contains "${repo_dir}/.dev/dev.env" "CT_OPS_AGENT_CONTAINER_INGEST_ADDRESS=ingest-dev:9443" assert_contains "${repo_dir}/.dev/dev.env" "CT_OPS_ENROLMENT_TOKEN=" assert_contains "${repo_dir}/.dev/dev.env" "CT_OPS_DEV_UID=${expected_uid}" assert_contains "${repo_dir}/.dev/dev.env" "CT_OPS_DEV_GID=${expected_gid}" @@ -137,9 +137,9 @@ run_public_config_generation_test() { assert_contains "${repo_dir}/.dev/dev.env" "CT_OPS_DEV_BIND_ADDR=127.0.0.1" assert_contains "${repo_dir}/.dev/dev.env" "CT_OPS_DEV_PUBLIC_HOST=" assert_contains "${repo_dir}/.dev/dev.env" "BETTER_AUTH_URL=http://localhost:3000" - assert_contains "${repo_dir}/.dev/dev.env" "AGENT_DOWNLOAD_BASE_URL=http://host.docker.internal:3000" + assert_contains "${repo_dir}/.dev/dev.env" "AGENT_DOWNLOAD_BASE_URL=http://dev-proxy" assert_contains "${repo_dir}/.dev/dev.env" "CT_OPS_DEV_AGENT_INGEST_ADDRESS=localhost:9443" - assert_contains "${repo_dir}/.dev/dev.env" "CT_OPS_AGENT_CONTAINER_INGEST_ADDRESS=host.docker.internal:9443" + assert_contains "${repo_dir}/.dev/dev.env" "CT_OPS_AGENT_CONTAINER_INGEST_ADDRESS=ingest-dev:9443" } run_static_wiring_test() { diff --git a/dev-stack.sh b/dev-stack.sh index 36448d09..d7d15eed 100755 --- a/dev-stack.sh +++ b/dev-stack.sh @@ -316,8 +316,8 @@ ensure_dev_env() { bind_addr="127.0.0.1" public_host="" app_url="http://localhost:${proxy_port}" - agent_download_base_url="http://host.docker.internal:${proxy_port}" - agent_container_ingest_address="host.docker.internal:${ingest_grpc_port}" + agent_download_base_url="http://dev-proxy" + agent_container_ingest_address="ingest-dev:9443" trusted_origins="$app_url" fi