From ee6bc0ec6dd989598a40844c017607e9238255ea Mon Sep 17 00:00:00 2001 From: Lftobs Date: Thu, 8 Oct 2026 05:46:01 +0100 Subject: [PATCH 01/17] fix(failover): live-probe targets, rate-limit auto moves, clean old containers Abort failover when the chosen target does not answer its SSH port, refuse automatic failovers within FAILOVER_MIN_INTERVAL_MS (default 10m), allow disabling the monitor with FAILOVER_DISABLED, and remove the previous deployment's containers from the old server behind a 30s cap so a dead host cannot hang the failover. --- .../orchestrator/__tests__/failover-runner.ts | 92 +++++++++++++++++- .../orchestrator/__tests__/failover.test.ts | 18 ++++ apps/api/src/orchestrator/failover.ts | 94 ++++++++++++++++++- apps/api/src/utils/config.ts | 2 + 4 files changed, 201 insertions(+), 5 deletions(-) diff --git a/apps/api/src/orchestrator/__tests__/failover-runner.ts b/apps/api/src/orchestrator/__tests__/failover-runner.ts index b500c01..ce1d535 100644 --- a/apps/api/src/orchestrator/__tests__/failover-runner.ts +++ b/apps/api/src/orchestrator/__tests__/failover-runner.ts @@ -1,4 +1,5 @@ import { mock } from "bun:test"; +import net from "node:net"; const fileUrl = (relPath: string) => new URL(relPath, import.meta.url).toString(); @@ -7,6 +8,22 @@ let servers: any[] = []; let projects: any[] = []; let deploymentsByProject: Record = {}; +const dockerCalls: any[][] = []; + +mock.module(fileUrl("../../utils/docker-run"), () => ({ + getDockerTargetArgs: mock(() => []), + dockerRun: mock(() => Promise.resolve("")), + dockerRunTry: mock((...args: any[]) => { + dockerCalls.push(args); + return Promise.resolve(""); + }), + dockerExecStream: mock(() => null), + dockerExec: mock(() => Promise.resolve({ code: 0, stdout: "", stderr: "" })), + dockerExecWithStdin: mock(() => Promise.resolve({ code: 0, stdout: "", stderr: "" })), + ensureContainerRemoved: mock(() => Promise.resolve()), + ensureVolumeRemoved: mock(() => Promise.resolve()), +})); + mock.module(fileUrl("../../db/repo"), () => ({ getPlatformSettings: mock(() => Promise.resolve(platformSettings)), getProjectById: mock((id: string) => Promise.resolve(projects.find((p) => p.id === id) ?? null)), @@ -47,6 +64,10 @@ mock.module(fileUrl("../../executors/dispatch"), () => ({ })), })); +const probeListener = net.createServer(); +await new Promise((resolve) => probeListener.listen(0, "127.0.0.1", resolve)); +const probePort = (probeListener.address() as net.AddressInfo).port; + const { failoverProject } = await import("../failover"); const results: any = {}; @@ -115,12 +136,20 @@ try { platformSettings = { ingressServerId: "ing" }; servers = [ { id: "ing", name: "Ingress", mode: "ssh" }, - { id: "a", name: "Current", mode: "ssh", status: "connected" }, - { id: "b", name: "Target", mode: "ssh", status: "connected" }, + { id: "a", name: "Current", mode: "ssh", status: "connected", host: "127.0.0.1", port: probePort }, + { id: "b", name: "Target", mode: "ssh", status: "connected", host: "127.0.0.1", port: probePort }, ]; projects = [{ id: "p1", name: "proj-1", serverId: "a" }]; deploymentsByProject = { - p1: [{ id: "dep-1", sourceType: "git", sourceRef: "https://github.com/x/y.git", branch: "main" }], + p1: [ + { + id: "dep-1", + sourceType: "git", + sourceRef: "https://github.com/x/y.git", + branch: "main", + containerName: "app-old", + }, + ], }; const deployment = await failoverProject("p1"); @@ -133,4 +162,61 @@ try { results.test5 = { ok: false, error: e.message }; } +// Test 6: aborts when the chosen target is not actually reachable +let createDeploymentCallsBefore6 = 0; +try { + platformSettings = { ingressServerId: "ing" }; + servers = [ + { id: "ing", name: "Ingress", mode: "ssh" }, + { id: "a", name: "Current", mode: "ssh", status: "connected", host: "127.0.0.1", port: probePort }, + { id: "b", name: "DeadTarget", mode: "ssh", status: "connected", host: "127.0.0.1", port: 1 }, + ]; + projects = [{ id: "p1", name: "proj-1", serverId: "a" }]; + deploymentsByProject = { + p1: [ + { + id: "dep-1", + sourceType: "git", + sourceRef: "https://github.com/x/y.git", + branch: "main", + containerName: "app-old", + }, + ], + }; + const repo6 = await import("../../db/repo"); + createDeploymentCallsBefore6 = (repo6.createDeployment as any).mock.calls.length; + await failoverProject("p1"); + results.test6 = { ok: false, error: "should have thrown" }; +} catch (e: any) { + const repo = await import("../../db/repo"); + const created = (repo.createDeployment as any).mock.calls.length - createDeploymentCallsBefore6; + results.test6 = { + ok: e.message?.includes("not reachable") && created === 0, + error: e.message, + }; +} + +// Test 7: stops the previous deployment's containers on the old server +try { + const primaryRm = dockerCalls.some( + (args) => args[1]?.[0] === "rm" && args[1]?.includes("app-old") && args[2]?.id === "a", + ); + const replicaList = dockerCalls.some( + (args) => + args[1]?.[0] === "ps" && args[1]?.some((a: string) => a.includes("deploy-dep-1-replica-")) && args[2]?.id === "a", + ); + results.test7 = { ok: primaryRm && replicaList }; +} catch (e: any) { + results.test7 = { ok: false, error: e.message }; +} + +// Test 8: refuses a second automatic failover within the minimum interval +try { + await failoverProject("p1", { trigger: "auto" }); + results.test8 = { ok: false, error: "should have thrown" }; +} catch (e: any) { + results.test8 = { ok: e.message?.includes("before another failover"), error: e.message }; +} + +probeListener.close(); console.log(JSON.stringify(results)); diff --git a/apps/api/src/orchestrator/__tests__/failover.test.ts b/apps/api/src/orchestrator/__tests__/failover.test.ts index 5e442b1..a74b356 100644 --- a/apps/api/src/orchestrator/__tests__/failover.test.ts +++ b/apps/api/src/orchestrator/__tests__/failover.test.ts @@ -45,4 +45,22 @@ describe("failoverProject", () => { const r = parse(stdout); expect(r.test5.ok).toBe(true); }); + + it("aborts when the chosen target server is not reachable", async () => { + const { stdout } = await run(); + const r = parse(stdout); + expect(r.test6.ok).toBe(true); + }); + + it("removes the previous deployment's containers on the old server", async () => { + const { stdout } = await run(); + const r = parse(stdout); + expect(r.test7.ok).toBe(true); + }); + + it("refuses a second automatic failover within the minimum interval", async () => { + const { stdout } = await run(); + const r = parse(stdout); + expect(r.test8.ok).toBe(true); + }); }); diff --git a/apps/api/src/orchestrator/failover.ts b/apps/api/src/orchestrator/failover.ts index 5b37cbb..4bf8c1e 100644 --- a/apps/api/src/orchestrator/failover.ts +++ b/apps/api/src/orchestrator/failover.ts @@ -11,16 +11,24 @@ import { updateProject, updateRouteStatus, } from "../db/repo"; +import type { Server } from "../types"; +import { config } from "../utils/config"; +import { dockerBin } from "../utils/docker-bin"; +import { dockerRunTry } from "../utils/docker-run"; import { getIngressServer, removeIngressRouteFile } from "../utils/ingress"; import { pickBestServer } from "../utils/server-default"; +import { getContainerName } from "./runtime"; const CHECK_INTERVAL_MS = 30_000; const GRACE_MS = 180_000; const CONNECT_TIMEOUT_MS = 5_000; +const OLD_CONTAINER_CLEANUP_TIMEOUT_MS = 30_000; const unreachableSince = new Map(); const failingOver = new Set(); const previouslyUnreachableServers = new Set(); +const lastFailoverAt = new Map(); +const rateLimitWarned = new Set(); export const isServerReachable = (host: string, port: number = 22): Promise => new Promise((resolve) => { @@ -34,11 +42,69 @@ export const isServerReachable = (host: string, port: number = 22): Promise finish(false)); }); -export const failoverProject = async (projectId: string) => { +const isFailoverDisabled = () => /^(1|true|yes)$/i.test(config.failoverDisabled); + +const bestEffort = (work: Promise, label: string): Promise => + Promise.race([ + work, + new Promise((_resolve, reject) => { + const timer = setTimeout(() => reject(new Error("timed out")), OLD_CONTAINER_CLEANUP_TIMEOUT_MS); + timer.unref?.(); + }), + ]) + .then(() => undefined) + .catch((error) => { + console.error(`[Failover] ${label} failed:`, error instanceof Error ? error.message : error); + }); + +const removeOldServerContainers = ( + server: Server, + deployment: { id: string; containerName?: string | null }, + project: { name?: string | null; id: string }, +): Promise => { + const primaryName = + deployment.containerName || getContainerName(deployment.id, project.name ?? undefined, project.id); + const remove = async () => { + const listed = await dockerRunTry( + dockerBin, + ["ps", "-a", "--format", "{{.Names}}", "--filter", `name=deploy-${deployment.id}-replica-`], + server, + ); + const replicas = (listed ?? "") + .split("\n") + .map((line) => line.trim()) + .filter(Boolean); + const names = [primaryName, ...replicas]; + let removed = 0; + for (const name of names) { + await dockerRunTry(dockerBin, ["stop", "-t", "10", name], server); + const result = await dockerRunTry(dockerBin, ["rm", "-f", name], server); + if (result !== undefined) removed++; + } + console.log( + `[Failover] old-container cleanup on ${server.name}: removed ${removed}/${names.length} (${names.join(", ")})`, + ); + }; + return bestEffort(remove(), `old-container cleanup on ${server.name}`); +}; + +export const failoverProject = async ( + projectId: string, + opts: { trigger?: "manual" | "auto" } = {}, +): Promise>> => { if (failingOver.has(projectId)) throw new Error("Failover already in progress for this project"); const project = await getProjectById(projectId); if (!project) throw new Error("Project not found"); if (!project.serverId) throw new Error("Project has no server assigned"); + if ((opts.trigger ?? "manual") === "auto") { + const last = lastFailoverAt.get(projectId); + const since = last ? Date.now() - last : Number.POSITIVE_INFINITY; + if (since < config.failoverMinIntervalMs) { + throw new Error( + `Project failed over ${Math.round(since / 1000)}s ago — waiting ${config.failoverMinIntervalMs / 1000}s before another failover`, + ); + } + } const ingressServer = await getIngressServer(); if (!ingressServer) throw new Error("No ingress server configured"); const currentServer = await getServerById(project.serverId); @@ -57,6 +123,10 @@ export const failoverProject = async (projectId: string) => { } const targetServer = await getServerById(targetId); if (!targetServer) throw new Error("Target server not found"); + const targetReachable = await isServerReachable(targetServer.host, targetServer.port || 22); + if (!targetReachable) { + throw new Error(`Target server ${targetServer.name} is not reachable — failover aborted`); + } const deployment = await createDeployment({ projectId, @@ -82,11 +152,17 @@ export const failoverProject = async (projectId: string) => { await queueRemoteDeployment(deployment, project); } + await removeOldServerContainers(currentServer, latest, project); await updateProject(projectId, { serverId: targetId }); + lastFailoverAt.set(projectId, Date.now()); + rateLimitWarned.delete(projectId); unreachableSince.delete(projectId); await updateDeploymentStatus(latest.id, "inactive", { failureReason: `Superseded by failover deployment to ${targetServer.name}`, }).catch(() => {}); + console.log( + `[Failover] project ${project.name ?? projectId} moved from ${currentServer.name} to ${targetServer.name} (deployment ${deployment.id})`, + ); return deployment; } finally { failingOver.delete(projectId); @@ -148,7 +224,15 @@ export const failoverMonitorTick = async () => { console.log( `[Failover] Server ${server.name} unreachable for ${Math.round((Date.now() - firstSeen) / 1000)}s — failing over project ${project.name}`, ); - failoverProject(project.id).catch((error) => { + failoverProject(project.id, { trigger: "auto" }).catch((error) => { + const message = error instanceof Error ? error.message : String(error); + if (message.includes("before another failover")) { + if (!rateLimitWarned.has(project.id)) { + rateLimitWarned.add(project.id); + console.log(`[Failover] skipping project ${project.name}: ${message}`); + } + return; + } console.error(`[Failover] Auto-failover for project ${project.id} failed:`, error); unreachableSince.delete(project.id); }); @@ -170,14 +254,20 @@ export const failoverMonitorTick = async () => { }; export const startFailoverMonitor = () => { + if (isFailoverDisabled()) { + console.log("[Failover] monitor disabled via FAILOVER_DISABLED"); + return null; + } const handle = setInterval(failoverMonitorTick, CHECK_INTERVAL_MS); failoverMonitorTick(); return handle; }; export const failoverState = () => ({ + disabled: isFailoverDisabled(), unreachable: [...unreachableSince.entries()].map(([projectId, since]) => ({ projectId, unreachableForMs: Date.now() - since, })), + minIntervalMs: config.failoverMinIntervalMs, }); diff --git a/apps/api/src/utils/config.ts b/apps/api/src/utils/config.ts index b54ddc0..8588e89 100644 --- a/apps/api/src/utils/config.ts +++ b/apps/api/src/utils/config.ts @@ -50,4 +50,6 @@ export const config = { wireguardServerIp: withFile("WIREGUARD_SERVER_IP", "10.200.0.1"), wireguardPeerCidr: withFile("WIREGUARD_PEER_CIDR", "10.200.0.0/24"), gatewayCertDir: withFile("GATEWAY_CERT_DIR", "/app/data"), + failoverDisabled: withFile("FAILOVER_DISABLED", ""), + failoverMinIntervalMs: withFile("FAILOVER_MIN_INTERVAL_MS", "600000", Number), }; From 850bf5b66aacd7e267d9a4de52e8c3fd407d62d0 Mon Sep 17 00:00:00 2001 From: Lftobs Date: Thu, 8 Oct 2026 08:32:44 +0100 Subject: [PATCH 02/17] fix(ssh): run the railpack config generator on remote hosts Remote builds cloned the repo and ran `railpack build .` without ever consulting the project's sourceDir or projectType, so any project whose code lives in a subfolder failed with "Railpack could not determine how to build the app" while the same project deployed fine locally. Send the local railpack-config-utils.ts source to the target host, write it into the clone through a quoted heredoc, and run it there with bun, which is installed unzip-free via python3 when the host has neither bun nor unzip. Remote failures now read stdout before stderr, so failureReason no longer reports git's "Cloning into '.'..." instead of the real error. --- .../src/executors/__tests__/dispatch.test.ts | 42 +++++++++++ apps/api/src/executors/ssh-build-script.ts | 70 +++++++++++++++++++ apps/api/src/executors/ssh.ts | 17 ++++- .../__tests__/deployment-errors.test.ts | 23 +++++- .../api/src/orchestrator/deployment-errors.ts | 3 + 5 files changed, 151 insertions(+), 4 deletions(-) diff --git a/apps/api/src/executors/__tests__/dispatch.test.ts b/apps/api/src/executors/__tests__/dispatch.test.ts index ab8d3bb..3ed6ebb 100644 --- a/apps/api/src/executors/__tests__/dispatch.test.ts +++ b/apps/api/src/executors/__tests__/dispatch.test.ts @@ -2,6 +2,8 @@ import { describe, expect, it } from "bun:test"; import { executorFor } from "../dispatch"; import { buildRemoteDeployScript, parseRemoteBuildResult } from "../ssh-build-script"; +const generator = "export const generateDynamicRailpackJson = async () => {};\n"; + const input = { deploymentId: "deployment-1", workspaceRoot: "/var/lib/dequel/workspace", @@ -11,6 +13,7 @@ const input = { imageTag: "example-api-deploym:latest", clearCache: false, environmentVariables: [{ key: "NODE_ENV", value: "production" }], + railpackGenerator: generator, }; describe("remote SSH build script", () => { @@ -47,6 +50,45 @@ describe("remote SSH build script", () => { const script = buildRemoteDeployScript(input); expect(script).toContain("curl -fsSL https://railpack.com/install.sh"); }); + + it("ships the railpack generator into the workspace after the clone", () => { + const script = buildRemoteDeployScript(input); + expect(script).toContain("cat > \"$PROJECT_DIR/.dequel-railpack-gen.ts\" <<'DEQUEL_RAILPACK_GEN_EOF'"); + expect(script).toContain(generator); + expect(script).toContain("await generateDynamicRailpackJson("); + expect(script.indexOf("git clone")).toBeLessThan(script.indexOf("DEQUEL_RAILPACK_GEN_EOF")); + expect(script).toContain('bun "$PROJECT_DIR/.dequel-railpack-gen.ts" "$PROJECT_DIR"'); + expect(script).toContain("DEQUEL_RAILPACK_GEN_EOF\nbun "); + }); + + it("installs bun before generating the config", () => { + const script = buildRemoteDeployScript(input); + expect(script).toContain("bun-linux-${BUN_ARCH}.zip"); + expect(script).toContain("python3 -m zipfile -e"); + expect(script).toContain("bun.sh/install"); + expect(script).toContain('export PATH="$HOME/.bun/bin:$PATH"'); + expect(script.indexOf("bun-linux-${BUN_ARCH}.zip")).toBeLessThan(script.indexOf(".dequel-railpack-gen.ts")); + }); + + it("passes source dir and project type to the generator", () => { + const script = buildRemoteDeployScript({ + ...input, + sourceDir: "client", + projectType: "static", + buildCommand: "npm run build", + startCommand: "npm start", + installCommand: null, + outputDir: null, + }); + expect(script).toContain( + `bun "$PROJECT_DIR/.dequel-railpack-gen.ts" "$PROJECT_DIR" 'client' 'static' 'npm run build' 'npm start' '' ''`, + ); + }); + + it("escapes single quotes in the source dir", () => { + const script = buildRemoteDeployScript({ ...input, sourceDir: "it's" }); + expect(script).toContain(`"$PROJECT_DIR" 'it'\\''s'`); + }); }); describe("remote build result parsing", () => { diff --git a/apps/api/src/executors/ssh-build-script.ts b/apps/api/src/executors/ssh-build-script.ts index 6b94e4c..8b138cd 100644 --- a/apps/api/src/executors/ssh-build-script.ts +++ b/apps/api/src/executors/ssh-build-script.ts @@ -7,10 +7,32 @@ export interface SshBuildScriptInput { imageTag: string; clearCache?: boolean; environmentVariables: { key: string; value: string }[]; + sourceDir?: string | null; + projectType?: string | null; + buildCommand?: string | null; + installCommand?: string | null; + outputDir?: string | null; + startCommand?: string | null; + railpackGenerator: string; } const sh = (value: string) => `'${value.replace(/'/g, `'\\''`)}'`; +const GENERATOR_EOF = "DEQUEL_RAILPACK_GEN_EOF"; + +const GENERATOR_CLI = `const [workspace, sourceDir, projectType, buildCommand, startCommand, installCommand, outputDir] = + process.argv.slice(2); +await generateDynamicRailpackJson( + workspace, + sourceDir || null, + projectType || null, + buildCommand || null, + startCommand || null, + async (line) => console.log(line), + installCommand || null, + outputDir || null, +);`; + export const buildRemoteDeployScript = (input: SshBuildScriptInput): string => { const cacheKey = input.imageTag .split(":")[0] @@ -28,6 +50,16 @@ export const buildRemoteDeployScript = (input: SshBuildScriptInput): string => { const remoteWorkspace = input.workspaceRoot.startsWith("/app") ? "$HOME/.dequel/workspace" : input.workspaceRoot; + const generatorArgs = [ + '"$PROJECT_DIR"', + sh(input.sourceDir ?? ""), + sh(input.projectType ?? ""), + sh(input.buildCommand ?? ""), + sh(input.startCommand ?? ""), + sh(input.installCommand ?? ""), + sh(input.outputDir ?? ""), + ].join(" "); + return [ "set -euo pipefail", "", @@ -55,6 +87,44 @@ export const buildRemoteDeployScript = (input: SshBuildScriptInput): string => { `git clone --depth 1 ${sh(input.gitUrl)} .`, checkoutSha, "", + 'if command -v bun >/dev/null 2>&1 || [ -x "$HOME/.bun/bin/bun" ]; then', + ' echo "[build] bun already installed"', + "else", + ' echo "[build] Installing bun (required to generate railpack.json)"', + ' ARCH="$(uname -m)"', + ' case "$ARCH" in', + " x86_64) BUN_ARCH=x64 ;;", + " aarch64|arm64) BUN_ARCH=aarch64 ;;", + ' *) echo "[build] unsupported architecture for bun: $ARCH" >&2; exit 1 ;;', + " esac", + " if command -v python3 >/dev/null 2>&1; then", + ' mkdir -p "$HOME/.bun/bin" /tmp/dequel-bun', + ' curl -fsSL -o /tmp/dequel-bun.zip "https://github.com/oven-sh/bun/releases/latest/download/bun-linux-${BUN_ARCH}.zip"', + " python3 -m zipfile -e /tmp/dequel-bun.zip /tmp/dequel-bun", + ' mv "/tmp/dequel-bun/bun-linux-${BUN_ARCH}/bun" "$HOME/.bun/bin/bun"', + ' chmod +x "$HOME/.bun/bin/bun"', + " rm -rf /tmp/dequel-bun.zip /tmp/dequel-bun", + " elif command -v unzip >/dev/null 2>&1; then", + ' curl -fsSL https://bun.sh/install | BUN_INSTALL="$HOME/.bun" bash', + " else", + ' echo "[build] bun install requires python3 or unzip on this machine" >&2', + " exit 1", + " fi", + "fi", + 'export PATH="$HOME/.bun/bin:$PATH"', + "if ! command -v bun >/dev/null 2>&1; then", + ' echo "[build] bun is required to generate railpack.json but is not installed" >&2', + " exit 1", + "fi", + "", + 'echo "[build] Generating dynamic railpack.json for caching and monorepo resolution"', + `cat > "$PROJECT_DIR/.dequel-railpack-gen.ts" <<'${GENERATOR_EOF}'`, + input.railpackGenerator, + GENERATOR_CLI, + GENERATOR_EOF, + `bun "$PROJECT_DIR/.dequel-railpack-gen.ts" ${generatorArgs}`, + 'rm -f "$PROJECT_DIR/.dequel-railpack-gen.ts"', + "", 'SHA="$(git rev-parse HEAD)"', `echo "[build] Building image ${input.imageTag} with Railpack"`, `railpack build --name ${sh(input.imageTag)} --progress plain --cache-key ${sh(effectiveCacheKey)} \\`, diff --git a/apps/api/src/executors/ssh.ts b/apps/api/src/executors/ssh.ts index 2c2eff5..f5911c3 100644 --- a/apps/api/src/executors/ssh.ts +++ b/apps/api/src/executors/ssh.ts @@ -1,4 +1,6 @@ -import { summarizeDeploymentError } from "../orchestrator/deployment-errors"; +import { readFileSync } from "node:fs"; +import { join } from "node:path"; +import { remoteScriptFailure, summarizeDeploymentError } from "../orchestrator/deployment-errors"; import type { Deployment, Project, Server } from "../types"; import { config } from "../utils/config"; import { CANCELLED_FAILURE_REASON } from "../utils/failure-outcome"; @@ -15,6 +17,8 @@ import type { DeploymentExecutor, ExecutorCancelInput, ExecutorDeployInput, Exec let repoModule: typeof import("../db/repo") | null = null; let runtimeModule: typeof import("../orchestrator/runtime") | null = null; +const railpackGenerator = readFileSync(join(import.meta.dir, "../orchestrator/railpack-config-utils.ts"), "utf8"); + const getRepo = async () => (repoModule ??= await import("../db/repo")); const getRuntime = async () => (runtimeModule ??= await import("../orchestrator/runtime")); @@ -108,7 +112,7 @@ const deployComposeRemote = async (deployment: Deployment, project: Project, ser await emitLog(deployment.id, "build", line); }, }); - if (result.code !== 0) throw new Error(result.stderr || result.stdout || "Remote compose build failed"); + if (result.code !== 0) throw new Error(remoteScriptFailure(result, "Remote compose build failed")); const composeResult = parseRemoteComposeResult(result.stdout); if (!composeResult) throw new Error("Remote compose completed without a result marker"); @@ -297,6 +301,13 @@ export const sshExecutor: DeploymentExecutor = { imageTag, clearCache: deployment.clearCache ?? false, environmentVariables: envVars, + sourceDir: project.sourceDir, + projectType: project.projectType, + buildCommand: project.buildCommand, + installCommand: project.installCommand, + outputDir: project.outputDir, + startCommand: project.startCommand, + railpackGenerator, }); try { @@ -305,7 +316,7 @@ export const sshExecutor: DeploymentExecutor = { await emitLog(deployment.id, "build", line); }, }); - if (result.code !== 0) throw new Error(result.stderr || result.stdout || "Remote build failed"); + if (result.code !== 0) throw new Error(remoteScriptFailure(result, "Remote build failed")); const buildResult = parseRemoteBuildResult(result.stdout); if (!buildResult) throw new Error("Remote build completed without a result marker"); diff --git a/apps/api/src/orchestrator/__tests__/deployment-errors.test.ts b/apps/api/src/orchestrator/__tests__/deployment-errors.test.ts index fdbf786..4b0a5bd 100644 --- a/apps/api/src/orchestrator/__tests__/deployment-errors.test.ts +++ b/apps/api/src/orchestrator/__tests__/deployment-errors.test.ts @@ -1,5 +1,5 @@ import { describe, expect, test } from "bun:test"; -import { summarizeDeploymentError } from "../deployment-errors"; +import { remoteScriptFailure, summarizeDeploymentError } from "../deployment-errors"; describe("summarizeDeploymentError", () => { test("extracts an actionable runtime requirement from noisy build output", () => { @@ -33,3 +33,24 @@ unrecognized image format`); ); }); }); + +describe("remoteScriptFailure", () => { + test("prefers stdout so git progress on stderr cannot become the failure reason", () => { + const message = remoteScriptFailure( + { stdout: "✖ Railpack could not determine how to build the app.", stderr: "Cloning into '.'..." }, + "Remote build failed", + ); + expect(message).toContain("Railpack could not determine"); + expect(message).toContain("Cloning into"); + }); + + test("keeps stderr when stdout is empty", () => { + expect(remoteScriptFailure({ stdout: "", stderr: "Permission denied (publickey)." }, "fallback")).toBe( + "Permission denied (publickey).", + ); + }); + + test("falls back when both streams are empty", () => { + expect(remoteScriptFailure({ stdout: "", stderr: "" }, "Remote build failed")).toBe("Remote build failed"); + }); +}); diff --git a/apps/api/src/orchestrator/deployment-errors.ts b/apps/api/src/orchestrator/deployment-errors.ts index 8f9bbf1..770a3d2 100644 --- a/apps/api/src/orchestrator/deployment-errors.ts +++ b/apps/api/src/orchestrator/deployment-errors.ts @@ -29,3 +29,6 @@ export const summarizeDeploymentError = (error: unknown): string => { return "Deployment failed. Check the deployment logs for details."; }; + +export const remoteScriptFailure = (result: { stdout: string; stderr: string }, fallback: string): string => + [result.stdout, result.stderr].filter(Boolean).join("\n") || fallback; From 34a863854e580fd48f58d77b778e29d38da2b425 Mon Sep 17 00:00:00 2001 From: Lftobs Date: Thu, 8 Oct 2026 21:47:06 +0100 Subject: [PATCH 03/17] fix(logs): access-log ingress routes and target runtime logs at the deployment's server ingressSite (the route writer for cross-server projects behind the local ingress) never emitted CADDY_ACCESS_LOG_BLOCK, so project requests were never access-logged and request-logs/metrics stayed empty. Runtime logs ran 'docker logs' against the local node for every deployment (the SSE stream used a raw spawn, bypassing ssh targeting), returning [] for anything on a remote server; both handlers now resolve the deployment's server behind orchestrator/runtime-logs.ts. --- apps/api/src/api/deployments/index.ts | 28 ++++---- .../__tests__/runtime-logs-runner.ts | 67 +++++++++++++++++++ .../__tests__/runtime-logs.test.ts | 46 +++++++++++++ apps/api/src/orchestrator/runtime-logs.ts | 41 ++++++++++++ apps/api/src/utils/__tests__/ingress.test.ts | 2 + apps/api/src/utils/ingress.ts | 4 +- 6 files changed, 171 insertions(+), 17 deletions(-) create mode 100644 apps/api/src/orchestrator/__tests__/runtime-logs-runner.ts create mode 100644 apps/api/src/orchestrator/__tests__/runtime-logs.test.ts create mode 100644 apps/api/src/orchestrator/runtime-logs.ts diff --git a/apps/api/src/api/deployments/index.ts b/apps/api/src/api/deployments/index.ts index 138e5c8..2da9526 100644 --- a/apps/api/src/api/deployments/index.ts +++ b/apps/api/src/api/deployments/index.ts @@ -16,6 +16,7 @@ import { executorFor } from "../../executors/dispatch"; import { orchestrator } from "../../orchestrator"; import { summarizeDeploymentError } from "../../orchestrator/deployment-errors"; import { logBus } from "../../orchestrator/log-bus"; +import { readRuntimeLogs, runtimeLogFollowerArgs } from "../../orchestrator/runtime-logs"; import { config } from "../../utils/config"; import { isPrivateGitUrl } from "../../utils/validate"; import { created, fail, ok } from "../response"; @@ -329,20 +330,8 @@ export const deploymentsRoutes = new Elysia() set.status = 404; return fail("Deployment not found"); } - const { run } = await import("../../orchestrator/runtime"); - const containerName = deployment.containerName || `deploy-${id}`; try { - const output = await run("docker", ["logs", "--tail", "200", containerName]); - const lines = output - .split("\n") - .filter(Boolean) - .map((line, i) => ({ - sequence: i + 1, - message: line, - timestamp: new Date().toISOString(), - stage: "runtime" as const, - })); - return ok(lines); + return ok(await readRuntimeLogs(deployment)); } catch { return ok([]); } @@ -354,7 +343,6 @@ export const deploymentsRoutes = new Elysia() return fail("Deployment not found"); } const encoder = new TextEncoder(); - const containerName = deployment.containerName || `deploy-${id}`; let closed = false; const stop = () => { closed = true; @@ -369,9 +357,19 @@ export const deploymentsRoutes = new Elysia() controller.enqueue(encoder.encode(`event: ${eventName}\ndata: ${JSON.stringify(payload)}\n\n`)); }; const { spawn } = await import("node:child_process"); - const child = spawn("docker", ["logs", "--tail", "100", "--follow", containerName], { + const child = spawn("docker", await runtimeLogFollowerArgs(deployment), { stdio: ["ignore", "pipe", "pipe"], }); + child.on("error", (err) => { + send("log", { + sequence: 0, + message: `docker logs failed: ${err.message}`, + timestamp: new Date().toISOString(), + stage: "runtime", + }); + send("close", { reason: "spawn failed" }); + stop(); + }); let seq = 0; child.stdout.on("data", (chunk: Buffer) => { const lines = chunk.toString().split("\n").filter(Boolean); diff --git a/apps/api/src/orchestrator/__tests__/runtime-logs-runner.ts b/apps/api/src/orchestrator/__tests__/runtime-logs-runner.ts new file mode 100644 index 0000000..2159e72 --- /dev/null +++ b/apps/api/src/orchestrator/__tests__/runtime-logs-runner.ts @@ -0,0 +1,67 @@ +import { mock } from "bun:test"; + +const fileUrl = (relPath: string) => new URL(relPath, import.meta.url).toString(); + +const servers: Record = { + "srv-worker": { id: "srv-worker", mode: "ssh", host: "1.2.3.4", port: 22 }, +}; + +const dockerCalls: any[][] = []; +const targetArgsCalls: any[] = []; + +mock.module(fileUrl("../../db/repo"), () => ({ + getServerById: mock((id: string) => Promise.resolve(servers[id] ?? null)), +})); + +mock.module(fileUrl("../../utils/docker-run"), () => ({ + dockerRun: mock((...args: any[]) => { + dockerCalls.push(args); + return Promise.resolve("line one\nline two"); + }), + getDockerTargetArgs: mock((server: any) => { + targetArgsCalls.push(server); + return server?.mode === "ssh" ? ["-H", `ssh://root@${server.host}:${server.port}`] : []; + }), +})); + +try { + const { readRuntimeLogs, runtimeLogFollowerArgs } = await import("../runtime-logs"); + + const results: any = {}; + + const lines = await readRuntimeLogs({ id: "dep-1", containerName: "app-old", serverId: "srv-worker" }); + results.readOnRemote = { + count: lines.length, + first: { sequence: lines[0].sequence, message: lines[0].message, stage: lines[0].stage }, + serverPassed: dockerCalls[0][2]?.id === "srv-worker", + command: dockerCalls[0].slice(0, 2), + }; + + dockerCalls.length = 0; + const localLines = await readRuntimeLogs({ id: "dep-2", containerName: null, serverId: null }); + results.readLocal = { + count: localLines.length, + command: dockerCalls[0].slice(0, 2), + serverNull: dockerCalls[0][2] === null, + }; + + targetArgsCalls.length = 0; + const followerArgs = await runtimeLogFollowerArgs({ id: "dep-3", containerName: "app-1", serverId: "srv-worker" }); + results.follower = { + args: followerArgs, + serverPassed: targetArgsCalls[0]?.id === "srv-worker", + }; + + targetArgsCalls.length = 0; + const localArgs = await runtimeLogFollowerArgs({ id: "dep-4", containerName: null, serverId: null }); + results.followerLocal = { + args: localArgs, + serverNull: targetArgsCalls[0] === null, + }; + + console.log(JSON.stringify(results)); +} catch (err: any) { + console.error("Runner error:", err?.message ?? String(err)); + console.error(err?.stack ?? err); + process.exit(1); +} diff --git a/apps/api/src/orchestrator/__tests__/runtime-logs.test.ts b/apps/api/src/orchestrator/__tests__/runtime-logs.test.ts new file mode 100644 index 0000000..a2cdfa4 --- /dev/null +++ b/apps/api/src/orchestrator/__tests__/runtime-logs.test.ts @@ -0,0 +1,46 @@ +import { describe, expect, it } from "bun:test"; +import { execFile } from "node:child_process"; +import { promisify } from "node:util"; + +const exec = promisify(execFile); + +const run = () => + exec("bun", ["run", "src/orchestrator/__tests__/runtime-logs-runner.ts"], { + cwd: `${import.meta.dir}/../../..`, + timeout: 30000, + }); + +const parse = (stdout: string) => { + const lines = stdout.trim().split("\n"); + return JSON.parse(lines[lines.length - 1]); +}; + +describe("runtime logs target the deployment's server", () => { + it("reads logs through the remote server and maps lines", async () => { + const { stdout } = await run(); + const results = parse(stdout); + + expect(results.readOnRemote.count).toBe(2); + expect(results.readOnRemote.first).toEqual({ sequence: 1, message: "line one", stage: "runtime" }); + expect(results.readOnRemote.serverPassed).toBe(true); + expect(results.readOnRemote.command).toEqual(["docker", ["logs", "--tail", "200", "app-old"]]); + + expect(results.readLocal.count).toBe(2); + expect(results.readLocal.command).toEqual(["docker", ["logs", "--tail", "200", "deploy-dep-2"]]); + expect(results.readLocal.serverNull).toBe(true); + + expect(results.follower.serverPassed).toBe(true); + expect(results.follower.args).toEqual([ + "-H", + "ssh://root@1.2.3.4:22", + "logs", + "--tail", + "100", + "--follow", + "app-1", + ]); + + expect(results.followerLocal.serverNull).toBe(true); + expect(results.followerLocal.args).toEqual(["logs", "--tail", "100", "--follow", "deploy-dep-4"]); + }, 30_000); +}); diff --git a/apps/api/src/orchestrator/runtime-logs.ts b/apps/api/src/orchestrator/runtime-logs.ts new file mode 100644 index 0000000..76f31e9 --- /dev/null +++ b/apps/api/src/orchestrator/runtime-logs.ts @@ -0,0 +1,41 @@ +import { getServerById } from "../db/repo"; +import type { Server } from "../types"; +import { dockerRun, getDockerTargetArgs } from "../utils/docker-run"; + +export interface RuntimeLogLine { + sequence: number; + message: string; + timestamp: string; + stage: "runtime"; +} + +interface DeploymentTarget { + id: string; + containerName?: string | null; + serverId?: string | null; +} + +const resolveServer = async (deployment: DeploymentTarget): Promise => + deployment.serverId ? await getServerById(deployment.serverId) : null; + +const containerNameFor = (deployment: DeploymentTarget): string => + deployment.containerName || `deploy-${deployment.id}`; + +export const readRuntimeLogs = async (deployment: DeploymentTarget): Promise => { + const server = await resolveServer(deployment); + const output = await dockerRun("docker", ["logs", "--tail", "200", containerNameFor(deployment)], server); + return output + .split("\n") + .filter(Boolean) + .map((line, i) => ({ + sequence: i + 1, + message: line, + timestamp: new Date().toISOString(), + stage: "runtime" as const, + })); +}; + +export const runtimeLogFollowerArgs = async (deployment: DeploymentTarget): Promise => { + const server = await resolveServer(deployment); + return [...getDockerTargetArgs(server), "logs", "--tail", "100", "--follow", containerNameFor(deployment)]; +}; diff --git a/apps/api/src/utils/__tests__/ingress.test.ts b/apps/api/src/utils/__tests__/ingress.test.ts index 433f6e3..2e2c425 100644 --- a/apps/api/src/utils/__tests__/ingress.test.ts +++ b/apps/api/src/utils/__tests__/ingress.test.ts @@ -19,6 +19,8 @@ describe("ingress route shapes", () => { const snippet = ingressSite("app.example.com", "203.0.113.10"); expect(snippet).toContain("reverse_proxy 203.0.113.10:80"); expect(snippet).not.toContain("header_up Host"); + expect(snippet).toContain("log {"); + expect(snippet).toContain("output stdout"); }); }); diff --git a/apps/api/src/utils/ingress.ts b/apps/api/src/utils/ingress.ts index cdb3936..22e9b0a 100644 --- a/apps/api/src/utils/ingress.ts +++ b/apps/api/src/utils/ingress.ts @@ -1,7 +1,7 @@ import { rm, writeFile } from "node:fs/promises"; import { join } from "node:path"; import { createAgentJob, getPlatformSettings, getServerById, upsertRoute } from "../db/repo"; -import { caddyReverseProxy, caddySite } from "./caddy-site"; +import { CADDY_ACCESS_LOG_BLOCK, caddyReverseProxy, caddySite } from "./caddy-site"; import { config } from "./config"; import { removeRemoteCaddyRoute, syncRemoteCaddyRoute } from "./ssh"; @@ -47,7 +47,7 @@ export const projectServerSite = ( }; export const ingressSite = (hostname: string, upstreamHost: string): string => - `${hostname} {\n reverse_proxy ${upstreamHost}:80\n}\n`; + `${hostname} {\n${CADDY_ACCESS_LOG_BLOCK}\n reverse_proxy ${upstreamHost}:80\n}\n`; export const syncIngressRoute = async ( ingressServer: { id: string; mode: string }, From 31cbea0fcf6091ea4b1a991e23a0a7cec34812d4 Mon Sep 17 00:00:00 2001 From: Lftobs Date: Fri, 9 Oct 2026 02:06:38 +0100 Subject: [PATCH 04/17] fix: bug-sheet fixes, test-suite repair, and Let's Encrypt certs for database hosts MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Close out the open rows from artifacts/verify-dequel/BUGS.md (005-018, 027, 028, 030-033, 039, 040), live-verified on the primary host. BUG-027: cancelling a deployment now kills the build's whole process group (detached local spawns with pgid tracking, DEQUEL_PGID echoed by the remote script and parsed on abort), and the cancel handler reports the abort instead of a generic failure. BUG-039: pam-auth mounts host /etc as a directory and copies passwd/shadow/group into the container in place every 2s, so useradd/chpasswd/usermod become visible to PAM immediately — file binds froze the mount-time inode so logins kept failing until a restart. BUG-040: Caddy now provisions real Let's Encrypt certificates for db-* hostnames (site route files database-.caddy written on create, patch, and delete trigger auto-HTTPS), and the gateway serves those certs from the shared caddy-data volume (mtime-cached, SAN and expiry checked, self-signed fallback kept). Live-verified: openssl -verify_return_error returns 0 for all three public databases and the node pg driver's verify-full semantics now connect out of the box. BUG-016: the credentials endpoint probes external reachability and returns a warning the dashboard surfaces when the endpoint is down. Also extract authMiddleware into its own module, add lazy repo imports and orchestrator mocks so bun test runs green again (440 pass, 1 pre-existing DiagRunMachine failure), and correct the database TLS docs note to describe the new certificate path. --- apps/api/src/api/auth-middleware.ts | 53 +++++++ apps/api/src/api/databases/index.ts | 58 +++++++- apps/api/src/api/deployments/index.ts | 10 +- apps/api/src/api/index.ts | 47 +------ apps/api/src/api/projects/index.ts | 23 ++- apps/api/src/api/scaling/index.ts | 49 +++++++ apps/api/src/api/ssh-keys/index.ts | 65 +++++++-- apps/api/src/db/repo/projects.ts | 50 +++++-- apps/api/src/db/repo/ssh-keys.ts | 10 +- apps/api/src/executors/ssh.ts | 64 +++++++-- apps/api/src/fixdiag/provider-models.ts | 21 +-- apps/api/src/index.ts | 11 +- .../orchestrator/__tests__/failover-runner.ts | 24 +++- .../orchestrator/__tests__/failover.test.ts | 6 + apps/api/src/orchestrator/compose.ts | 17 ++- apps/api/src/orchestrator/failover.ts | 21 +-- apps/api/src/orchestrator/project-cleanup.ts | 51 +++++++ apps/api/src/orchestrator/railpack.ts | 12 +- apps/api/src/scaling/__tests__/engine.test.ts | 46 ++++++ apps/api/src/servers/prepare.ts | 131 ++++++++++++++++-- apps/api/src/tunnels/cert-store.ts | 56 ++++++++ apps/api/src/tunnels/gateway.ts | 12 +- apps/api/src/utils/compose-ingress.ts | 10 +- apps/api/src/utils/db-caddy-route.ts | 68 +++++++++ apps/api/src/utils/permissions.ts | 13 ++ apps/api/src/utils/proc-group.ts | 12 ++ apps/api/src/utils/server-default.ts | 6 +- apps/api/src/utils/ssh.ts | 9 +- apps/api/src/utils/tcp-probe.ts | 16 +++ apps/docs/src/content/docs/databases.md | 2 +- apps/web/src/api/client.ts | 13 +- apps/web/src/api/databases.ts | 2 + .../src/components/databases/DatabaseCard.tsx | 6 + .../components/databases/DatabaseOverview.tsx | 11 +- .../components/settings/LlmKeysSection.tsx | 14 +- .../components/settings/ServersSection.tsx | 22 +-- .../components/settings/SshKeyPoolSection.tsx | 56 ++++++-- apps/web/src/routes/CreateProjectPage.tsx | 52 +++++-- apps/web/src/routes/Login.tsx | 4 +- docker-compose.yml | 18 ++- 40 files changed, 963 insertions(+), 208 deletions(-) create mode 100644 apps/api/src/api/auth-middleware.ts create mode 100644 apps/api/src/orchestrator/project-cleanup.ts create mode 100644 apps/api/src/tunnels/cert-store.ts create mode 100644 apps/api/src/utils/db-caddy-route.ts create mode 100644 apps/api/src/utils/proc-group.ts create mode 100644 apps/api/src/utils/tcp-probe.ts diff --git a/apps/api/src/api/auth-middleware.ts b/apps/api/src/api/auth-middleware.ts new file mode 100644 index 0000000..2ba3a07 --- /dev/null +++ b/apps/api/src/api/auth-middleware.ts @@ -0,0 +1,53 @@ +import type { Elysia } from "elysia"; +import { fail } from "./response"; + +const BYPASS_PATHS = new Set([ + "/api/auth/login", + "/api/auth/logout", + "/api/auth/refresh", + "/api/auth/me", + "/api/health", + "/api/github/callback", + "/api/github/webhook", + "/api/agents/register", + "/api/agents/p2p-sync", +]); + +export const authMiddleware = (app: Elysia) => + app.onBeforeHandle(async ({ request, set, path }) => { + if (BYPASS_PATHS.has(path)) return; + + const cookie = request.headers.get("cookie") || ""; + const match = cookie.match(/(?:^|;\s*)dequel_session=([^;]+)/); + if (match) { + const { verifyAccessToken } = await import("../utils/auth"); + const payload = await verifyAccessToken(match[1]); + if (payload) return; + set.status = 401; + return fail("Invalid session"); + } + + const authHeader = request.headers.get("authorization"); + if (authHeader?.startsWith("Bearer ")) { + const token = authHeader.slice(7); + if (token) { + const { validateApiKey } = await import("../db/repo"); + const { canPerformMethod, parsePermissions, requiredPermissionForMethod } = await import( + "../utils/permissions" + ); + const key = await validateApiKey(token); + if (!key) { + set.status = 401; + return fail("Invalid API key"); + } + if (!canPerformMethod(parsePermissions(key.permissions), request.method)) { + set.status = 403; + return fail(`API key is missing the ${requiredPermissionForMethod(request.method)} scope`); + } + return; + } + } + + set.status = 401; + return fail("Authentication required"); + }); diff --git a/apps/api/src/api/databases/index.ts b/apps/api/src/api/databases/index.ts index fdd2526..39acbc5 100644 --- a/apps/api/src/api/databases/index.ts +++ b/apps/api/src/api/databases/index.ts @@ -13,7 +13,9 @@ import { } from "../../db/repo"; import type { Database } from "../../types"; import { config } from "../../utils/config"; +import { removeDbGatewayRoute, syncDbGatewayRoute } from "../../utils/db-caddy-route"; import { resolveServerIp } from "../../utils/dns"; +import { probeTcp } from "../../utils/tcp-probe"; import { created, fail, ok } from "../response"; import { executeDatabaseQuery, getDatabaseTables } from "./query"; @@ -76,6 +78,7 @@ const createManagedDatabase = async (body: any, projectId: string | null, set: a return fail("A database with this name already exists"); } const dbRecord = await createDbRecord({ ...validation.input, projectId }); + void syncDbGatewayRoute(dbRecord); const { provisionDatabase } = await import("../../databases/manager"); provisionDatabase(dbRecord).catch((err: Error) => console.error("DB provision failed", err)); return created(sanitizeDatabase(dbRecord)); @@ -118,27 +121,66 @@ export const databasesRoutes = new Elysia() ? `${dbRecord.internalHost}.${baseDomain}` : null; if (gatewayHost) { + const reachable = await probeTcp(gatewayHost, 443); + if (reachable) { + return ok({ + username: dbRecord.username, + password: dbRecord.password, + internalConnectionString: dbRecord.connectionString, + externalConnectionString: buildGatewayConnectionString(dbRecord, gatewayHost), + externalHost: gatewayHost, + externalPort: 443, + externalReachable: true, + warning: null, + }); + } return ok({ username: dbRecord.username, password: dbRecord.password, internalConnectionString: dbRecord.connectionString, - externalConnectionString: buildGatewayConnectionString(dbRecord, gatewayHost), + externalConnectionString: null, externalHost: gatewayHost, externalPort: 443, + externalReachable: false, + warning: `Gateway endpoint ${gatewayHost}:443 is not reachable from the network — check the gateway service and DNS.`, }); } const externalHost = dbRecord.publicAccess && dbRecord.externalPort ? await resolveServerIp() : null; const usableHost = externalHost && isNonLoopbackIp(externalHost) ? externalHost : null; - const externalConnectionString = usableHost - ? buildConnectionString(dbRecord, usableHost, dbRecord.externalPort!) - : null; + if (usableHost && dbRecord.externalPort) { + const reachable = await probeTcp(usableHost, dbRecord.externalPort); + if (reachable) { + return ok({ + username: dbRecord.username, + password: dbRecord.password, + internalConnectionString: dbRecord.connectionString, + externalConnectionString: buildConnectionString(dbRecord, usableHost, dbRecord.externalPort), + externalHost: usableHost, + externalPort: dbRecord.externalPort, + externalReachable: true, + warning: null, + }); + } + return ok({ + username: dbRecord.username, + password: dbRecord.password, + internalConnectionString: dbRecord.connectionString, + externalConnectionString: null, + externalHost: usableHost, + externalPort: dbRecord.externalPort, + externalReachable: false, + warning: `Public endpoint ${usableHost}:${dbRecord.externalPort} is not reachable — open the port in your cloud security group or firewall to allow external clients.`, + }); + } return ok({ username: dbRecord.username, password: dbRecord.password, internalConnectionString: dbRecord.connectionString, - externalConnectionString, - externalHost: usableHost, - externalPort: usableHost ? dbRecord.externalPort : null, + externalConnectionString: null, + externalHost: null, + externalPort: null, + externalReachable: null, + warning: null, }); }) .post("/databases/:id/start", async ({ params: { id }, set }) => { @@ -235,6 +277,7 @@ export const databasesRoutes = new Elysia() backupSchedule: body?.backupSchedule, backupRetention: body?.backupRetention, }); + void syncDbGatewayRoute(updated!); return ok(sanitizeDatabase(updated!)); }) .delete("/databases/:id", async ({ params: { id }, set }) => { @@ -259,6 +302,7 @@ export const databasesRoutes = new Elysia() return fail("Database resources could not be deleted; cleanup will be retried automatically"); } await deleteDatabase(id); + void removeDbGatewayRoute(current); return ok(null, "Database deleted"); }); }); diff --git a/apps/api/src/api/deployments/index.ts b/apps/api/src/api/deployments/index.ts index 2da9526..ee97b0f 100644 --- a/apps/api/src/api/deployments/index.ts +++ b/apps/api/src/api/deployments/index.ts @@ -142,9 +142,17 @@ export const deploymentsRoutes = new Elysia() set.status = 400; return fail("archive file is required for upload source"); } + const safeName = basename(file.name || "project.zip").replace(/[^a-zA-Z0-9._-]/g, "_"); + if (![".zip", ".tar", ".tar.gz", ".tgz"].some((ext) => safeName.toLowerCase().endsWith(ext))) { + set.status = 400; + return fail("Unsupported archive format. Use .zip, .tar, .tar.gz, or .tgz"); + } + if (file.size === 0) { + set.status = 400; + return fail("Archive file is empty"); + } const uploadsDir = join(config.workspaceRoot, "uploads"); await mkdir(uploadsDir, { recursive: true }); - const safeName = basename(file.name || "project.zip").replace(/[^a-zA-Z0-9._-]/g, "_"); const uploadPath = join(uploadsDir, `${Date.now()}-${safeName}`); const bytes = new Uint8Array(await file.arrayBuffer()); await writeFile(uploadPath, bytes); diff --git a/apps/api/src/api/index.ts b/apps/api/src/api/index.ts index 571912f..42708fc 100644 --- a/apps/api/src/api/index.ts +++ b/apps/api/src/api/index.ts @@ -1,9 +1,10 @@ import { Elysia } from "elysia"; -import { fail } from "./response"; +import { fixdiagRoutes } from "../fixdiag/routes"; import { agentRoutes } from "./agents"; import { alertsRoutes } from "./alerts"; import { apiKeysRoutes } from "./api-keys"; import { authRoutes } from "./auth"; +import { authMiddleware } from "./auth-middleware"; import { backupRoutes } from "./backups"; import { databasesRoutes } from "./databases"; import { deploymentsRoutes } from "./deployments"; @@ -14,59 +15,17 @@ import { healthRoutes } from "./health"; import { projectsRoutes } from "./projects"; import { projectStatusRoutes } from "./projects/status"; import { prometheusRoutes } from "./prometheus"; +import { fail } from "./response"; import { routesRoutes } from "./routes"; import { scalingRoutes } from "./scaling"; import { serverInfoRoutes } from "./server-info"; import { serversRoutes } from "./servers"; import { settingsRoutes } from "./settings"; import { llmSettingsRoutes } from "./settings/llm"; -import { fixdiagRoutes } from "../fixdiag/routes"; import { sharedEnvLinksRoutes, sharedEnvVarsRoutes } from "./shared-env-vars"; import { sshKeysRoutes } from "./ssh-keys"; import { volumesRoutes } from "./volumes"; -const BYPASS_PATHS = new Set([ - "/api/auth/login", - "/api/auth/logout", - "/api/auth/refresh", - "/api/auth/me", - "/api/health", - "/api/github/callback", - "/api/github/webhook", - "/api/agents/register", - "/api/agents/p2p-sync", -]); - -const authMiddleware = (app: Elysia) => - app.onBeforeHandle(async ({ request, set, path }) => { - if (BYPASS_PATHS.has(path)) return; - - const cookie = request.headers.get("cookie") || ""; - const match = cookie.match(/(?:^|;\s*)dequel_session=([^;]+)/); - if (match) { - const { verifyAccessToken } = await import("../utils/auth"); - const payload = await verifyAccessToken(match[1]); - if (payload) return; - set.status = 401; - return fail("Invalid session"); - } - - const authHeader = request.headers.get("authorization"); - if (authHeader?.startsWith("Bearer ")) { - const token = authHeader.slice(7); - if (token) { - const { validateApiKey } = await import("../db/repo"); - const key = await validateApiKey(token); - if (key) return; - set.status = 401; - return fail("Invalid API key"); - } - } - - set.status = 401; - return fail("Authentication required"); - }); - const INTERNAL_ERROR = /Failed query:|params:|getaddrinfo|ECONNREFUSED|ETIMEDOUT|EAI_AGAIN|EHOSTUNREACH|timeout exceeded|node:internal|Cannot read propert|is not a function|is not a constructor|Unexpected token/i; diff --git a/apps/api/src/api/projects/index.ts b/apps/api/src/api/projects/index.ts index fe698b2..06f377d 100644 --- a/apps/api/src/api/projects/index.ts +++ b/apps/api/src/api/projects/index.ts @@ -9,14 +9,17 @@ import { listProjects, updateProject, } from "../../db/repo"; +import { destroyComposeStack } from "../../orchestrator/compose"; import { failoverProject } from "../../orchestrator/failover"; +import { cleanupProjectContainers } from "../../orchestrator/project-cleanup"; import { reloadCaddy, tryRun } from "../../orchestrator/runtime"; import { config } from "../../utils/config"; import { dockerBin } from "../../utils/docker-bin"; +import { dockerRunTry, ensureContainerRemoved } from "../../utils/docker-run"; import { removeFromCaddyRoute } from "../../utils/domain-verifier"; import { buildProjectRequestHostRegex, caddyRequestLogSelector } from "../../utils/loki"; -import { isPort, isPrivateGitUrl, SERVICE_NAME_RE, validateComposeServices } from "../../utils/validate"; import { captureTelemetry } from "../../utils/telemetry"; +import { isPort, isPrivateGitUrl, SERVICE_NAME_RE, validateComposeServices } from "../../utils/validate"; import { created, fail, ok } from "../response"; const validateComposeFields = (body: any): string | null => { @@ -168,23 +171,17 @@ export const projectsRoutes = new Elysia() return fail("Project not found"); } - for (const name of info.deploymentContainerNames) { - await tryRun(dockerBin, ["stop", "-t", "5", name]); - await tryRun(dockerBin, ["rm", "-f", name]); - } - for (const name of info.databaseContainerNames) { - await tryRun(dockerBin, ["stop", "-t", "5", name]); - await tryRun(dockerBin, ["rm", "-f", name]); - } + await cleanupProjectContainers(info, { + getServerById, + ensureContainerRemoved, + destroyComposeStack, + dockerRunTry, + }); for (const name of [...info.databaseVolumeNames, ...info.volumeDockerNames]) { await tryRun(dockerBin, ["volume", "rm", "-f", name]); } - for (const tag of info.deploymentImageTags) { - if (tag) await tryRun(dockerBin, ["rmi", "-f", tag]); - } - for (const { domain, projectName } of info.domains) { await removeFromCaddyRoute(domain, id, projectName); } diff --git a/apps/api/src/api/scaling/index.ts b/apps/api/src/api/scaling/index.ts index ef1c7b6..419afcb 100644 --- a/apps/api/src/api/scaling/index.ts +++ b/apps/api/src/api/scaling/index.ts @@ -2,6 +2,45 @@ import { Elysia } from "elysia"; import { deleteScalingPolicy, getProjectById, getScalingPolicy, upsertScalingPolicy } from "../../db/repo"; import { fail, ok } from "../response"; +const isPercent = (value: unknown): value is number => + typeof value === "number" && Number.isInteger(value) && value >= 1 && value <= 100; + +const isNonNegativeInt = (value: unknown): value is number => + typeof value === "number" && Number.isInteger(value) && value >= 0; + +const isCooldown = (value: unknown): value is number => + typeof value === "number" && Number.isInteger(value) && value >= 1 && value <= 86_400; + +const validatePolicyBody = (body: any): string | null => { + if (body.cpuThresholdPercent !== undefined && !isPercent(body.cpuThresholdPercent)) { + return "cpuThresholdPercent must be an integer between 1 and 100"; + } + if (body.memoryThresholdPercent !== undefined && !isPercent(body.memoryThresholdPercent)) { + return "memoryThresholdPercent must be an integer between 1 and 100"; + } + if (body.minReplicas !== undefined && !isNonNegativeInt(body.minReplicas)) { + return "minReplicas must be an integer of 0 or greater"; + } + if (body.maxReplicas !== undefined && !isNonNegativeInt(body.maxReplicas)) { + return "maxReplicas must be an integer of 0 or greater"; + } + if (body.cooldownSeconds !== undefined && !isCooldown(body.cooldownSeconds)) { + return "cooldownSeconds must be an integer between 1 and 86400"; + } + if (body.enabled !== undefined && typeof body.enabled !== "boolean") { + return "enabled must be a boolean"; + } + return null; +}; + +const validateReplicaRange = async (projectId: string, body: any): Promise => { + if (body.minReplicas === undefined && body.maxReplicas === undefined) return null; + const existing = await getScalingPolicy(projectId); + const min = body.minReplicas ?? existing?.minReplicas ?? 1; + const max = body.maxReplicas ?? existing?.maxReplicas ?? 5; + return max < min ? "maxReplicas must be greater than or equal to minReplicas" : null; +}; + export const scalingRoutes = new Elysia() .get("/projects/:id/scaling", async ({ params, set }) => { const policy = await getScalingPolicy(params.id); @@ -16,11 +55,21 @@ export const scalingRoutes = new Elysia() set.status = 400; return fail("body is required"); } + const bodyError = validatePolicyBody(body); + if (bodyError) { + set.status = 400; + return fail(bodyError); + } const project = await getProjectById(params.id); if (!project) { set.status = 404; return fail("Project not found"); } + const rangeError = await validateReplicaRange(params.id, body); + if (rangeError) { + set.status = 400; + return fail(rangeError); + } if (body.enabled !== false && (!project.cpuLimit || project.cpuLimit <= 0)) { set.status = 400; return fail("Cannot enable autoscaling on a project without CPU resource limits configured."); diff --git a/apps/api/src/api/ssh-keys/index.ts b/apps/api/src/api/ssh-keys/index.ts index bd0cec4..dd6f03f 100644 --- a/apps/api/src/api/ssh-keys/index.ts +++ b/apps/api/src/api/ssh-keys/index.ts @@ -1,7 +1,41 @@ import { Elysia } from "elysia"; -import { createSshKey, deleteSshKey, getSshKeyById, listSshKeys } from "../../db/repo/ssh-keys"; +import { + computeSshKeyFingerprint, + createSshKey, + deleteSshKey, + getSshKeyByFingerprint, + getSshKeyById, + listSshKeys, +} from "../../db/repo/ssh-keys"; import { fail, ok } from "../response"; +const PRIVATE_KEY_PATTERN = + /^-----BEGIN ((?:[A-Z0-9]+ )*?)PRIVATE KEY-----\r?\n([\s\S]*?)-----END \1PRIVATE KEY-----\s*$/; +const BASE64_PATTERN = /^(?:[A-Za-z0-9+/]{4})*(?:[A-Za-z0-9+/]{2}==|[A-Za-z0-9+/]{3}=)?$/; + +const isValidPrivateKey = (value: string): boolean => { + const match = PRIVATE_KEY_PATTERN.exec(value.trim()); + if (!match) return false; + const body = match[2] + .split(/\r?\n/) + .filter((line) => !line.includes(":")) + .join("") + .replace(/\s/g, ""); + return body.length >= 16 && BASE64_PATTERN.test(body); +}; + +const isUniqueViolation = (err: unknown): boolean => { + const e = err as { code?: string; cause?: { code?: string } }; + return e?.code === "23505" || e?.cause?.code === "23505"; +}; + +const duplicateKey = async (fingerprint: string, set: any) => { + const existing = await getSshKeyByFingerprint(fingerprint); + if (!existing) return null; + set.status = 409; + return fail(`SSH key already exists in the pool as "${existing.name}"`); +}; + export const sshKeysRoutes = new Elysia() .get("/ssh-keys", async () => { return ok(await listSshKeys()); @@ -11,13 +45,28 @@ export const sshKeysRoutes = new Elysia() set.status = 400; return fail("name and privateKey are required"); } - const created = await createSshKey({ - name: body.name, - privateKey: body.privateKey, - tags: body.tags, - }); - set.status = 201; - return ok(created); + if (!isValidPrivateKey(body.privateKey)) { + set.status = 400; + return fail("privateKey must be a valid PEM or OpenSSH private key"); + } + const fingerprint = computeSshKeyFingerprint(body.privateKey); + const existing = await duplicateKey(fingerprint, set); + if (existing) return existing; + try { + const created = await createSshKey({ + name: body.name, + privateKey: body.privateKey, + tags: body.tags, + }); + set.status = 201; + return ok(created); + } catch (err) { + if (isUniqueViolation(err)) { + const raced = await duplicateKey(fingerprint, set); + if (raced) return raced; + } + throw err; + } }) .get("/ssh-keys/:id", async ({ params, set }: any) => { const key = await getSshKeyById(params.id); diff --git a/apps/api/src/db/repo/projects.ts b/apps/api/src/db/repo/projects.ts index 34d53c1..5504870 100644 --- a/apps/api/src/db/repo/projects.ts +++ b/apps/api/src/db/repo/projects.ts @@ -16,9 +16,18 @@ import { import { formatTimestamp, getRowsAffected, now } from "./helpers"; export interface ProjectCleanupInfo { - deploymentContainerNames: string[]; - deploymentImageTags: string[]; - databaseContainerNames: string[]; + deploymentTargets: { + id: string; + sourceType: string; + containerName: string | null; + imageTag: string | null; + serverId: string | null; + }[]; + databaseTargets: { + containerName: string | null; + proxyContainerName: string | null; + serverId: string | null; + }[]; databaseVolumeNames: string[]; volumeDockerNames: string[]; domains: { domain: string; projectName: string }[]; @@ -163,12 +172,23 @@ export const deleteProjectCascade = async (id: string): Promise d.containerName).map((d) => d.containerName!); - const deploymentImageTags = depRows.filter((d) => d.imageTag).map((d) => d.imageTag!); + const deploymentTargets = depRows.map((d) => ({ + id: d.id, + sourceType: d.sourceType, + containerName: d.containerName, + imageTag: d.imageTag, + serverId: d.serverId, + })); for (const dep of depRows) { await db.delete(deploymentLogs).where(eq(deploymentLogs.deploymentId, dep.id)).execute(); @@ -185,11 +205,20 @@ export const deleteProjectCascade = async (id: string): Promise d.containerName).map((d) => d.containerName!); + const databaseTargets = dbRows.map((d) => ({ + containerName: d.containerName, + proxyContainerName: d.proxyContainerName, + serverId: d.serverId, + })); const databaseVolumeNames = dbRows.filter((d) => d.volumeName).map((d) => d.volumeName!); await db.update(databases).set({ projectId: null, updatedAt: now() }).where(eq(databases.projectId, id)).execute(); @@ -206,9 +235,8 @@ export const deleteProjectCascade = async (id: string): Promise { +export const computeSshKeyFingerprint = (privateKey: string): string => { return createHash("sha256").update(privateKey).digest("hex").slice(0, 32); }; @@ -24,7 +24,7 @@ const mapSshKey = (row: typeof sshKeys.$inferSelect): SshKey => ({ export const createSshKey = async (input: CreateSshKeyInput): Promise => { const id = randomUUID(); const timestamp = now(); - const fingerprint = computeFingerprint(input.privateKey); + const fingerprint = computeSshKeyFingerprint(input.privateKey); const encrypted = encryptValue(input.privateKey, config.envEncryptionKey); const db = await getDb(); await db @@ -57,6 +57,12 @@ export const getSshKeyById = async (id: string): Promise => { return row ? mapSshKey(row) : null; }; +export const getSshKeyByFingerprint = async (fingerprint: string): Promise => { + const db = await getDb(); + const [row] = await db.select().from(sshKeys).where(eq(sshKeys.fingerprint, fingerprint)).execute(); + return row ? mapSshKey(row) : null; +}; + export const getSshKeyPrivateKey = async (id: string): Promise => { const db = await getDb(); const [row] = await db diff --git a/apps/api/src/executors/ssh.ts b/apps/api/src/executors/ssh.ts index f5911c3..1546502 100644 --- a/apps/api/src/executors/ssh.ts +++ b/apps/api/src/executors/ssh.ts @@ -4,7 +4,7 @@ import { remoteScriptFailure, summarizeDeploymentError } from "../orchestrator/d import type { Deployment, Project, Server } from "../types"; import { config } from "../utils/config"; import { CANCELLED_FAILURE_REASON } from "../utils/failure-outcome"; -import { removeRemoteCaddyRoute, runRemoteScript, syncRemoteCaddyRoute } from "../utils/ssh"; +import { RemoteScriptAbortedError, removeRemoteCaddyRoute, runRemoteScript, syncRemoteCaddyRoute } from "../utils/ssh"; import { emitLog } from "./logging"; import { buildRemoteDeployScript, parseRemoteBuildResult } from "./ssh-build-script"; import { @@ -175,10 +175,10 @@ const deployComposeRemote = async (deployment: Deployment, project: Project, ser const domain = rawBaseDomain === "localhost" ? `${fallbackDomain}:80` : fallbackDomain; snippet = `${domain} {\n log {\n output stdout\n format json\n }\n reverse_proxy ${primary.container}:${primary.port} {\n header_up Host {upstream_hostport}\n }\n}\n`; } - const { DB_SERVICE_NAMES } = await import("../utils/compose-ingress"); + const { isDbServiceName } = await import("../utils/compose-ingress"); for (const svc of webServices) { if (svc.name === primaryServiceName) continue; - if (DB_SERVICE_NAMES.has(svc.name)) continue; + if (isDbServiceName(svc.name)) continue; const customMatch = customMappings.find((c) => c.serviceName === svc.name); const domains: string[] = []; if (customMatch?.subdomain?.trim()) { @@ -266,6 +266,26 @@ const markFailed = async (deploymentId: string, error: unknown) => { await recordDeploymentFailure({ deploymentId, reason: message, source: "ssh" }); }; +const remoteBuilds = new Map(); + +export const parseRemoteBuildPgid = (line: string): string | undefined => { + const match = line.match(/DEQUEL_PGID:\s*(\d+)/); + return match?.[1]; +}; + +export const killRemoteBuildGroup = async ( + server: Parameters[0], + pgid: string, + run: typeof runRemoteScript = runRemoteScript, +): Promise => { + await run( + server, + `kill -TERM -- -${pgid} 2>/dev/null || true +sleep 3 +kill -KILL -- -${pgid} 2>/dev/null || true`, + ).catch(() => {}); +}; + export const sshExecutor: DeploymentExecutor = { mode: "ssh", @@ -292,7 +312,8 @@ export const sshExecutor: DeploymentExecutor = { const imageTag = `${slugify(project.name)}-${deployment.id.slice(0, 8)}:latest`; const envVars = await listEnvironmentVariablesForDeploy(project.id, deployment.environment ?? undefined); - const script = buildRemoteDeployScript({ + const pgidEcho = `echo "DEQUEL_PGID:$(ps -o pgid= -p $$ | tr -d ' ')"`; + const script = `${pgidEcho}\n${buildRemoteDeployScript({ deploymentId: deployment.id, workspaceRoot: config.workspaceRoot, gitUrl: deployment.sourceRef, @@ -308,11 +329,19 @@ export const sshExecutor: DeploymentExecutor = { outputDir: project.outputDir, startCommand: project.startCommand, railpackGenerator, - }); + })}`; + const buildEntry = { controller: new AbortController() }; + remoteBuilds.set(deployment.id, buildEntry); try { const result = await runRemoteScript(server, script, { + signal: buildEntry.controller.signal, onLog: async (line) => { + const pgid = parseRemoteBuildPgid(line); + if (pgid) { + buildEntry.pgid = pgid; + return; + } await emitLog(deployment.id, "build", line); }, }); @@ -336,7 +365,11 @@ export const sshExecutor: DeploymentExecutor = { await emitLog(current.id, "system", `Marked inactive (superseded by ${deployment.id.slice(0, 8)})`); } } catch (error) { - await markFailed(deployment.id, error); + if (!(error instanceof RemoteScriptAbortedError)) { + await markFailed(deployment.id, error); + } + } finally { + remoteBuilds.delete(deployment.id); } }, @@ -412,7 +445,7 @@ export const sshExecutor: DeploymentExecutor = { await deleteDeploymentAndLogs(deployment.id); }, - async cancel({ deployment }: ExecutorCancelInput) { + async cancel({ deployment, server }: ExecutorCancelInput) { const { recordDeploymentCancellation } = await getRepo(); if (deployment.status !== "pending" && deployment.status !== "building") return; await recordDeploymentCancellation({ @@ -420,6 +453,21 @@ export const sshExecutor: DeploymentExecutor = { reason: CANCELLED_FAILURE_REASON, source: "ssh", }); - await emitLog(deployment.id, "system", "Deployment cancelled by user (remote build may continue on the server)"); + const entry = remoteBuilds.get(deployment.id); + let remoteStopped = false; + if (entry) { + entry.controller.abort(); + if (entry.pgid) { + remoteStopped = true; + killRemoteBuildGroup(server, entry.pgid).catch(() => {}); + } + } + await emitLog( + deployment.id, + "system", + remoteStopped + ? "Deployment cancelled by user — remote build processes stopped" + : "Deployment cancelled by user (remote build may continue on the server)", + ); }, }; diff --git a/apps/api/src/fixdiag/provider-models.ts b/apps/api/src/fixdiag/provider-models.ts index 308efa0..b66b674 100644 --- a/apps/api/src/fixdiag/provider-models.ts +++ b/apps/api/src/fixdiag/provider-models.ts @@ -6,22 +6,9 @@ export interface FetchModelsOptions { } export const DEFAULT_PROVIDER_MODELS: Record = { - openai: ["gpt-4o", "gpt-4o-mini", "o1", "o3-mini", "gpt-4-turbo", "gpt-6.1-sol", "gpt-6-astra", "gpt-5.6-sol"], - anthropic: [ - "claude-3-5-sonnet-latest", - "claude-3-5-haiku-latest", - "claude-3-opus-latest", - "claude-sonnet-5", - "claude-opus-5", - ], - gemini: [ - "gemini-2.5-flash", - "gemini-2.5-pro", - "gemini-1.5-pro", - "gemini-1.5-flash", - "gemini-3.8-flash", - "gemini-3.7-flash", - ], + openai: ["gpt-4o", "gpt-4o-mini", "o1", "o3-mini", "gpt-4-turbo"], + anthropic: ["claude-3-5-sonnet-latest", "claude-3-5-haiku-latest", "claude-3-opus-latest"], + gemini: ["gemini-2.5-flash", "gemini-2.5-pro", "gemini-1.5-pro", "gemini-1.5-flash"], groq: [ "llama-3.3-70b-versatile", "llama-3.1-8b-instant", @@ -29,7 +16,7 @@ export const DEFAULT_PROVIDER_MODELS: Record = { "moonshotai/kimi-k2-instruct", "openai/gpt-oss-120b", ], - ollama: ["llama3:latest", "qwen2.5-coder:latest", "mistral:latest", "deepseek-r1:latest", "qwen3.5:0.8b"], + ollama: ["llama3:latest", "qwen2.5-coder:latest", "mistral:latest", "deepseek-r1:latest"], custom: [], }; diff --git a/apps/api/src/index.ts b/apps/api/src/index.ts index 24f8a7d..1f42f18 100644 --- a/apps/api/src/index.ts +++ b/apps/api/src/index.ts @@ -9,7 +9,7 @@ import { getDb } from "./db/db-provider"; import { migrate } from "./db/migrate"; import { ensureLocalServer } from "./db/repo"; import { markInterruptedDiagRuns } from "./db/repo/diag-runs"; -import { deployments } from "./db/schema"; +import { deployments, projects } from "./db/schema"; import { alertEvaluator } from "./monitoring/evaluator"; import { startFailureNotifier } from "./monitoring/failure-notifier"; import { orchestrator } from "./orchestrator"; @@ -22,8 +22,6 @@ import { cleanupExpiredTokens, initAuth } from "./utils/auth"; import { config } from "./utils/config"; import { startDomainPolling } from "./utils/domain-verifier"; import { loadOrCreateJwtSecret } from "./utils/secrets"; - -import { projects } from "./db/schema"; import { captureTelemetry } from "./utils/telemetry"; const bootstrap = async () => { @@ -97,7 +95,12 @@ dequel_uptime_seconds ${uptimeSec} }; const app = new Elysia() - .use(cors()) + .use( + cors({ + origin: () => true, + credentials: false, + }), + ) .onBeforeHandle(() => { metrics.requestsTotal++; }) diff --git a/apps/api/src/orchestrator/__tests__/failover-runner.ts b/apps/api/src/orchestrator/__tests__/failover-runner.ts index ce1d535..0e9728a 100644 --- a/apps/api/src/orchestrator/__tests__/failover-runner.ts +++ b/apps/api/src/orchestrator/__tests__/failover-runner.ts @@ -83,7 +83,10 @@ try { await failoverProject("p1"); results.test1 = { ok: false, error: "should have thrown" }; } catch (e: any) { - results.test1 = { ok: e.message?.includes("No other healthy server available") }; + results.test1 = { + ok: e.message?.includes("No other healthy server available") && e.message?.includes("p1"), + error: e.message, + }; } // Test 2: rejects non-ssh project servers @@ -218,5 +221,24 @@ try { results.test8 = { ok: e.message?.includes("before another failover"), error: e.message }; } +// Test 9: no-deployment error carries project context on a single line +try { + platformSettings = { ingressServerId: "ing" }; + servers = [ + { id: "ing", name: "Ingress", mode: "ssh" }, + { id: "a", name: "Current", mode: "ssh", status: "connected", host: "127.0.0.1", port: probePort }, + ]; + projects = [{ id: "p1", serverId: "a" }]; + deploymentsByProject = { p1: [] }; + await failoverProject("p1"); + results.test9 = { ok: false, error: "should have thrown" }; +} catch (e: any) { + const message = e.message ?? ""; + results.test9 = { + ok: message.includes("no deployments to fail over") && message.includes("p1") && !message.includes("\n"), + error: message, + }; +} + probeListener.close(); console.log(JSON.stringify(results)); diff --git a/apps/api/src/orchestrator/__tests__/failover.test.ts b/apps/api/src/orchestrator/__tests__/failover.test.ts index a74b356..890e78e 100644 --- a/apps/api/src/orchestrator/__tests__/failover.test.ts +++ b/apps/api/src/orchestrator/__tests__/failover.test.ts @@ -63,4 +63,10 @@ describe("failoverProject", () => { const r = parse(stdout); expect(r.test8.ok).toBe(true); }); + + it("includes project context in the no-deployments error", async () => { + const { stdout } = await run(); + const r = parse(stdout); + expect(r.test9.ok).toBe(true); + }); }); diff --git a/apps/api/src/orchestrator/compose.ts b/apps/api/src/orchestrator/compose.ts index d25df56..0ec1efd 100644 --- a/apps/api/src/orchestrator/compose.ts +++ b/apps/api/src/orchestrator/compose.ts @@ -1,7 +1,10 @@ import { spawn } from "node:child_process"; import { existsSync, readFileSync, realpathSync } from "node:fs"; import { dirname, join, resolve, sep } from "node:path"; +import type { Server } from "../types"; import { dockerBin } from "../utils/docker-bin"; +import { dockerRun } from "../utils/docker-run"; +import { killChildTree } from "../utils/proc-group"; import { CancelledError } from "./railpack"; export interface ComposeTarget { @@ -203,6 +206,7 @@ const spawnComposeCommand = ( cwd, env: envVars || {}, stdio: ["ignore", "pipe", "pipe"], + detached: true, }); let stdout = ""; @@ -223,7 +227,8 @@ const spawnComposeCommand = ( }; const onAbort = () => { - child.kill("SIGTERM"); + killChildTree(child, "SIGTERM"); + setTimeout(() => killChildTree(child, "SIGKILL"), 5000); finish(undefined, new CancelledError()); }; @@ -322,14 +327,14 @@ export const deployWithCompose = async ( await onLog("Docker Compose stack started."); }; -export const destroyComposeStack = async (projectName: string): Promise => { +export const destroyComposeStack = async (projectName: string, server?: Server | null): Promise => { const filter = `label=com.docker.compose.project=${projectName}`; - const ps = await runDocker(["ps", "-aq", "--filter", filter]); - const ids = ps.stdout.split(/\s+/).filter(Boolean); + const ps = await dockerRun(dockerBin, ["ps", "-aq", "--filter", filter], server).catch(() => ""); + const ids = ps.split(/\s+/).filter((s) => /^[0-9a-f]{12,}$/.test(s)); for (const id of ids) { - await runDocker(["rm", "-f", id]); + await dockerRun(dockerBin, ["rm", "-f", id], server).catch(() => {}); } - await runDocker(["network", "rm", `${projectName}_default`]).catch(() => {}); + await dockerRun(dockerBin, ["network", "rm", `${projectName}_default`], server).catch(() => {}); }; export const getComposeContainerNames = async (projectName: string): Promise> => { diff --git a/apps/api/src/orchestrator/failover.ts b/apps/api/src/orchestrator/failover.ts index 4bf8c1e..d38b2d6 100644 --- a/apps/api/src/orchestrator/failover.ts +++ b/apps/api/src/orchestrator/failover.ts @@ -44,6 +44,11 @@ export const isServerReachable = (host: string, port: number = 22): Promise /^(1|true|yes)$/i.test(config.failoverDisabled); +const formatError = (error: unknown): string => { + const message = error instanceof Error ? error.message : String(error); + return message.trim() || "unknown error"; +}; + const bestEffort = (work: Promise, label: string): Promise => Promise.race([ work, @@ -54,7 +59,7 @@ const bestEffort = (work: Promise, label: string): Promise => ]) .then(() => undefined) .catch((error) => { - console.error(`[Failover] ${label} failed:`, error instanceof Error ? error.message : error); + console.error(`[Failover] ${label} failed: ${formatError(error)}`); }); const removeOldServerContainers = ( @@ -112,14 +117,14 @@ export const failoverProject = async ( const deployments = await listDeployments(projectId, 0, 1); const latest = deployments[0]; - if (!latest) throw new Error("Project has no deployments to fail over"); + if (!latest) throw new Error(`Project ${project.name ?? projectId} has no deployments to fail over`); if (latest.sourceType !== "git") throw new Error("Failover requires a Git deployment"); failingOver.add(projectId); try { const targetId = await pickBestServer(null, project.serverId, ["ssh"]); if (!targetId || targetId === project.serverId || targetId === "local") { - throw new Error("No other healthy server available for failover"); + throw new Error(`No other healthy server available for failover of project ${project.name ?? projectId}`); } const targetServer = await getServerById(targetId); if (!targetServer) throw new Error("Target server not found"); @@ -145,7 +150,7 @@ export const failoverProject = async ( void executorFor("ssh") .deploy({ deployment, project, server: targetServer }) .catch((error) => { - console.error(`[Failover] Deployment ${deployment.id} failed:`, error); + console.error(`[Failover] Deployment ${deployment.id} failed: ${formatError(error)}`); }); } else { const { queueRemoteDeployment } = await import("../agents/deployments"); @@ -225,7 +230,7 @@ export const failoverMonitorTick = async () => { `[Failover] Server ${server.name} unreachable for ${Math.round((Date.now() - firstSeen) / 1000)}s — failing over project ${project.name}`, ); failoverProject(project.id, { trigger: "auto" }).catch((error) => { - const message = error instanceof Error ? error.message : String(error); + const message = formatError(error); if (message.includes("before another failover")) { if (!rateLimitWarned.has(project.id)) { rateLimitWarned.add(project.id); @@ -233,7 +238,7 @@ export const failoverMonitorTick = async () => { } return; } - console.error(`[Failover] Auto-failover for project ${project.id} failed:`, error); + console.error(`[Failover] Auto-failover for project ${project.id} failed: ${message}`); unreachableSince.delete(project.id); }); }); @@ -243,13 +248,13 @@ export const failoverMonitorTick = async () => { for (const serverId of recoveredServers) { console.log(`[Failover] Server ${serverId} recovered — cleaning up stale routes`); await cleanupStaleRoutes(ingressServer, serverId).catch((error) => { - console.error(`[Failover] Stale route cleanup for server ${serverId} failed:`, error); + console.error(`[Failover] Stale route cleanup for server ${serverId} failed: ${formatError(error)}`); }); } previouslyUnreachableServers.clear(); for (const id of currentUnreachableServers) previouslyUnreachableServers.add(id); } catch (error) { - console.error("[Failover] Monitor tick failed:", error); + console.error(`[Failover] Monitor tick failed: ${formatError(error)}`); } }; diff --git a/apps/api/src/orchestrator/project-cleanup.ts b/apps/api/src/orchestrator/project-cleanup.ts new file mode 100644 index 0000000..db9f392 --- /dev/null +++ b/apps/api/src/orchestrator/project-cleanup.ts @@ -0,0 +1,51 @@ +import type { ProjectCleanupInfo } from "../db/repo"; +import type { Server } from "../types"; + +const CONTAINER_ID = /^[0-9a-f]{12,}$/; + +export interface ProjectCleanupRunner { + getServerById: (id: string) => Promise; + ensureContainerRemoved: (name: string, server?: Server | null) => Promise; + destroyComposeStack: (projectName: string, server?: Server | null) => Promise; + dockerRunTry: (cmd: string, args: string[], server?: Server | null) => Promise; +} + +export type ProjectCleanupTarget = Pick; + +export const cleanupProjectContainers = async ( + info: ProjectCleanupTarget, + runner: ProjectCleanupRunner, +): Promise => { + const serverCache = new Map(); + const serverFor = async (serverId: string | null): Promise => { + if (!serverId) return null; + if (!serverCache.has(serverId)) { + serverCache.set(serverId, await runner.getServerById(serverId).catch(() => null)); + } + return serverCache.get(serverId) ?? null; + }; + + for (const dep of info.deploymentTargets) { + const server = await serverFor(dep.serverId); + if (dep.sourceType === "compose") { + await runner.destroyComposeStack(dep.containerName || `deploy-${dep.id}`, server).catch(() => {}); + } else { + if (dep.containerName) await runner.ensureContainerRemoved(dep.containerName, server).catch(() => {}); + const replicas = await runner + .dockerRunTry("docker", ["ps", "-aq", "--filter", `name=deploy-${dep.id}-replica-`], server) + .catch(() => undefined); + for (const cid of (replicas ?? "").split(/\s+/).filter((s) => CONTAINER_ID.test(s))) { + await runner.dockerRunTry("docker", ["rm", "-f", cid], server).catch(() => {}); + } + } + if (dep.imageTag) await runner.dockerRunTry("docker", ["rmi", "-f", dep.imageTag], server).catch(() => {}); + } + + for (const dbTarget of info.databaseTargets) { + const server = await serverFor(dbTarget.serverId); + if (dbTarget.containerName) await runner.ensureContainerRemoved(dbTarget.containerName, server).catch(() => {}); + if (dbTarget.proxyContainerName) { + await runner.ensureContainerRemoved(dbTarget.proxyContainerName, server).catch(() => {}); + } + } +}; diff --git a/apps/api/src/orchestrator/railpack.ts b/apps/api/src/orchestrator/railpack.ts index b629c3c..5e24e37 100644 --- a/apps/api/src/orchestrator/railpack.ts +++ b/apps/api/src/orchestrator/railpack.ts @@ -1,6 +1,7 @@ import { spawn } from "node:child_process"; import type { Server } from "../types"; import { dockerBin } from "../utils/docker-bin"; +import { killChildTree } from "../utils/proc-group"; import { generateDynamicRailpackJson } from "./railpack-config-utils"; export interface RailpackBuildResult { @@ -66,7 +67,7 @@ export class CancelledError extends Error { } } -const spawnAsync = ( +export const spawnAsync = ( cmd: string, args: string[], opts?: { @@ -87,6 +88,7 @@ const spawnAsync = ( stdio: ["ignore", "pipe", "pipe"], env: opts?.env ?? process.env, cwd: opts?.cwd, + detached: true, }); let stdout = ""; let stderr = ""; @@ -94,10 +96,10 @@ const spawnAsync = ( const onAbort = () => { if (settled) return; - child.kill("SIGTERM"); + killChildTree(child, "SIGTERM"); setTimeout(() => { if (!settled) { - child.kill("SIGKILL"); + killChildTree(child, "SIGKILL"); } }, 5000); finish(undefined, new CancelledError()); @@ -116,10 +118,10 @@ const spawnAsync = ( ? setTimeout(() => { if (settled) return; opts.onTimeout?.(); - child.kill("SIGTERM"); + killChildTree(child, "SIGTERM"); setTimeout(() => { if (!settled) { - child.kill("SIGKILL"); + killChildTree(child, "SIGKILL"); } }, 5000); finish(undefined, new Error(`${cmd} timed out after ${opts.timeoutMs}ms`)); diff --git a/apps/api/src/scaling/__tests__/engine.test.ts b/apps/api/src/scaling/__tests__/engine.test.ts index 6ddb3c8..4273c98 100644 --- a/apps/api/src/scaling/__tests__/engine.test.ts +++ b/apps/api/src/scaling/__tests__/engine.test.ts @@ -311,6 +311,52 @@ describe("evaluate", () => { expect(mockWriteFilePath).toBeNull(); }); + it("scales down at idle even when cpuThresholdPercent is 0", async () => { + mockGetScalingPolicyResult = () => ({ + ...TEST_POLICY, + cpuThresholdPercent: 0, + minReplicas: 1, + cooldownSeconds: 0, + }); + mockRedisData = { lowCpuSince: String(Date.now() - 310_000), lastScaleDown: "0" }; + mockReadFileContent = "proj-1.localhost:80 {\n reverse_proxy deploy-dep-1:3000 deploy-dep-1-replica-2:3000\n}\n"; + mockTryRunImpl = async () => ""; + mockRunImpl = async (_cmd: string, args: string[]) => { + if (args.includes("stats")) return '{"CPUPerc":"0%","MemUsage":"64MiB / 512MiB"}'; + if (args[0] === "stop") return ""; + if (args[0] === "rm") return ""; + if (args[0] === "ps") return "caddy-abc123\n"; + if (args.includes("reload")) return ""; + return ""; + }; + + await (scalingEngine as any).evaluate(TEST_DEPLOYMENT); + expect(mockWriteFilePath).toBeTruthy(); + expect(mockWriteContent).toContain("deploy-dep-1"); + expect(mockWriteContent).not.toContain("replica-2"); + }); + + it("scales up on any observed CPU when cpuThresholdPercent is 0", async () => { + mockGetScalingPolicyResult = () => ({ ...TEST_POLICY, cpuThresholdPercent: 0, cooldownSeconds: 0 }); + mockRedisData = { highCpuSince: String(Date.now() - 10_000), lastScaleUp: "0" }; + mockReadFileContent = "proj-1.localhost:80 {\n reverse_proxy deploy-dep-1:3000\n}\n"; + + mockRunImpl = async (_cmd: string, args: string[]) => { + if (args.some((a) => a.includes("stats"))) return '{"CPUPerc":"1%","MemUsage":"128MiB / 512MiB"}'; + if (args.some((a) => a.includes("Image"))) return "my-app:latest"; + if (args.some((a) => a.includes("Env"))) return '["PORT=3000"]'; + if (args.some((a) => a.includes("Mounts"))) return "[]"; + if (args[0] === "run") return "new-container-id"; + if (args[0] === "ps") return "caddy-abc123\n"; + if (args.some((a) => a.includes("reload"))) return ""; + return ""; + }; + + await (scalingEngine as any).evaluate(TEST_DEPLOYMENT); + expect(mockWriteFilePath).toBeTruthy(); + expect(mockWriteContent).toContain("deploy-dep-1-replica-2"); + }); + it("skips evaluation when the agent server is offline", async () => { mockGetScalingPolicyResult = () => ({ ...TEST_POLICY, cooldownSeconds: 0 }); mockRedisData = { highCpuSince: String(Date.now() - 10_000), lastScaleUp: "0" }; diff --git a/apps/api/src/servers/prepare.ts b/apps/api/src/servers/prepare.ts index cf5c3da..fea1ade 100644 --- a/apps/api/src/servers/prepare.ts +++ b/apps/api/src/servers/prepare.ts @@ -50,6 +50,125 @@ if [ -n "$SUDO" ]; then fi log "docker" "Docker ready" +find_container() { + docker ps -a --format '{{.Names}}' 2>/dev/null | grep -m1 "$1" || true +} + +find_running_container() { + docker ps --format '{{.Names}}' 2>/dev/null | grep -m1 "$1" || true +} + +remove_stopped() { + local label="$1" name="$2" + local all running + all="$(docker ps -a --format '{{.Names}}' 2>/dev/null || true)" + running="$(docker ps --format '{{.Names}}' 2>/dev/null || true)" + if grep -qxF "$name" <<< "$all" && ! grep -qxF "$name" <<< "$running"; then + if $SUDO docker rm -f "$name" >/dev/null 2>&1; then + log "$label" "Removed stopped container $name" + fi + fi +} + +equivalent_caddy() { + local name image ports mounts + while IFS='|' read -r name image ports; do + if [ -z "$name" ]; then + continue + fi + case "$name" in + *dequel-caddy*) + echo "$name" + return 0 + ;; + esac + case "$image" in + *caddy*) ;; + *) continue ;; + esac + case "$ports" in + *":80->"*|*":443->"*) ;; + *) continue ;; + esac + mounts="$(docker inspect -f '{{range .Mounts}}{{.Destination}} {{end}}' "$name" 2>/dev/null || true)" + case "$mounts" in + */etc/caddy/routes*) + echo "$name" + return 0 + ;; + esac + done <<< "$(docker ps --format '{{.Names}}|{{.Image}}|{{.Ports}}' 2>/dev/null || true)" + return 0 +} + +equivalent_buildkit() { + local name image + while IFS='|' read -r name image; do + if [ -z "$name" ]; then + continue + fi + case "$name $image" in + *buildkit*) + echo "$name" + return 0 + ;; + esac + done <<< "$(docker ps --format '{{.Names}}|{{.Image}}' 2>/dev/null || true)" + return 0 +} + +equivalent_running() { + case "$1" in + caddy) + equivalent_caddy + ;; + buildkit) + equivalent_buildkit + ;; + esac + return 0 +} + +ensure_container() { + local label="$1" name="$2" + shift 2 + local running existing equivalent err + running="$(find_running_container "$name")" + if [ -n "$running" ]; then + log "$label" "Container already running as $running" + remove_stopped "$label" "$name" + return 0 + fi + existing="$(find_container "$name")" + if [ -n "$existing" ]; then + log "$label" "Starting existing container $existing..." + if $SUDO docker start "$existing" >/dev/null 2>&1; then + log "$label" "Started $existing" + return 0 + fi + remove_stopped "$label" "$name" + log "$label" "Existing container $existing did not start" + fi + log "$label" "Starting Dequel $label container..." + err="" + if err="$($SUDO docker run "$@" 2>&1 >/dev/null)"; then + log "$label" "Started $label container" + return 0 + fi + equivalent="$(equivalent_running "$label")" + if [ -n "$equivalent" ]; then + log "$label" "Launch failed but equivalent container $equivalent is already running - reusing it" + remove_stopped "$label" "$name" + return 0 + fi + remove_stopped "$label" "$name" + if [ -z "$err" ]; then + err="unknown docker error" + fi + echo "[prepare:$label] Failed to start $label container: $err" >&2 + exit 1 +} + install_caddy() { $SUDO mkdir -p /etc/caddy/routes if [ ! -f /etc/caddy/Caddyfile ]; then @@ -60,12 +179,7 @@ install_caddy() { fi $SUDO docker network create dequel_net 2>/dev/null || true - if ! docker ps -a --format '{{.Names}}' 2>/dev/null | grep -q '^dequel-caddy$'; then - log "caddy" "Starting Dequel Caddy container..." - $SUDO docker run -d --restart unless-stopped --name dequel-caddy --network dequel_net -p 80:80 -p 443:443 -v /etc/caddy/Caddyfile:/etc/caddy/Caddyfile -v /etc/caddy/routes:/etc/caddy/routes -v caddy_data:/data caddy:alpine 2>/dev/null || true - else - $SUDO docker start dequel-caddy 2>/dev/null || true - fi + ensure_container caddy dequel-caddy -d --restart unless-stopped --name dequel-caddy --network dequel_net -p 80:80 -p 443:443 -v /etc/caddy/Caddyfile:/etc/caddy/Caddyfile -v /etc/caddy/routes:/etc/caddy/routes -v caddy_data:/data caddy:alpine } install_caddy @@ -83,10 +197,7 @@ else log "railpack" "Railpack install failed - builds will attempt auto-install" fi -if ! docker ps -a --format '{{.Names}}' 2>/dev/null | grep -q '^buildkit$'; then - log "buildkit" "Starting BuildKit container..." - $SUDO docker run -d --restart unless-stopped --name buildkit --privileged moby/buildkit:latest 2>/dev/null || true -fi +ensure_container buildkit buildkit -d --restart unless-stopped --name buildkit --privileged moby/buildkit:latest log "done" "Server preparation complete" `; diff --git a/apps/api/src/tunnels/cert-store.ts b/apps/api/src/tunnels/cert-store.ts new file mode 100644 index 0000000..ee74705 --- /dev/null +++ b/apps/api/src/tunnels/cert-store.ts @@ -0,0 +1,56 @@ +import { X509Certificate } from "node:crypto"; +import { readdir, readFile, stat } from "node:fs/promises"; +import { join } from "node:path"; + +export interface LoadedCert { + cert: string; + key: string; +} + +interface CacheEntry { + mtimeMs: number; + notAfterMs: number; + value: LoadedCert | null; +} + +const cache = new Map(); + +const findCrtPath = async (dataDir: string, host: string): Promise => { + try { + const issuers = await readdir(join(dataDir, "certificates")); + for (const issuer of issuers) { + const crtPath = join(dataDir, "certificates", issuer, host, `${host}.crt`); + const st = await stat(crtPath).catch(() => null); + if (st?.isFile()) return crtPath; + } + } catch { + return null; + } + return null; +}; + +export const loadCaddyCert = async (dataDir: string, host: string, now = Date.now()): Promise => { + const crtPath = await findCrtPath(dataDir, host); + if (!crtPath) { + cache.delete(host); + return null; + } + const st = await stat(crtPath); + const cached = cache.get(host); + if (cached && cached.mtimeMs === st.mtimeMs && (cached.value === null || cached.notAfterMs > now + 60_000)) { + return cached.value; + } + try { + const keyPath = crtPath.replace(/\.crt$/, ".key"); + const [cert, key] = await Promise.all([readFile(crtPath, "utf8"), readFile(keyPath, "utf8")]); + const x509 = new X509Certificate(cert); + const notAfterMs = Date.parse(x509.validTo); + const valid = x509.checkHost(host) && notAfterMs > now + 60_000; + const value = valid ? { cert, key } : null; + cache.set(host, { mtimeMs: st.mtimeMs, notAfterMs, value }); + return value; + } catch { + cache.delete(host); + return null; + } +}; diff --git a/apps/api/src/tunnels/gateway.ts b/apps/api/src/tunnels/gateway.ts index f25ee70..bf19e1c 100644 --- a/apps/api/src/tunnels/gateway.ts +++ b/apps/api/src/tunnels/gateway.ts @@ -5,6 +5,7 @@ import { Pool } from "pg"; import { config } from "../utils/config"; import { decryptValue } from "../utils/crypto"; import { type GatewayCert, loadOrCreateGatewayCert } from "./cert"; +import { loadCaddyCert } from "./cert-store"; import { parseClientHello, parsePostgresPreamble, readClientHello } from "./peek"; import { type GatewayRoute, type RouteRow, resolveGatewayRoute } from "./routes"; import { forwardViaSsh, type SshEngineTarget } from "./ssh-tunnel"; @@ -15,6 +16,7 @@ export interface GatewayOptions { caddyHost: string; caddyPort: number; cert: GatewayCert; + caddyDataDir?: string; lookup: (internalHost: string) => Promise; } @@ -45,12 +47,15 @@ const forwardToCaddy = (client: Socket, acc: Buffer, caddyHost: string, caddyPor link([client, upstream]); }; -const terminateForEngine = ( +const terminateForEngine = async ( client: Socket, acc: Buffer, - cert: GatewayCert, route: Extract, + sni: string | null, + options: GatewayOptions, ) => { + const caddyCert = options.caddyDataDir && sni ? await loadCaddyCert(options.caddyDataDir, sni) : null; + const cert = caddyCert ?? options.cert; const tlsOptions: ServerOptions = { key: cert.key, cert: cert.cert }; const terminator = createTlsServer(tlsOptions, (plain) => { if (route.ssh) { @@ -102,7 +107,7 @@ export const startGateway = async (options: GatewayOptions): Promise { + const lower = name.toLowerCase(); + return DB_SERVICE_NAMES.has(lower) || lower.endsWith("db") || DB_NAME_PATTERN.test(lower); +}; + export const computeComposeIngressHostnames = ( webServices: { name: string; port: number }[], primaryServiceName: string, @@ -29,7 +37,7 @@ export const computeComposeIngressHostnames = ( if (svc.name === primaryServiceName) continue; const customMatch = customMappings.find((c) => c.serviceName === svc.name); - if (!customMatch && DB_SERVICE_NAMES.has(svc.name)) continue; + if (isDbServiceName(svc.name)) continue; const subdomainPrefix = customMatch?.subdomain?.trim() || svc.name; diff --git a/apps/api/src/utils/db-caddy-route.ts b/apps/api/src/utils/db-caddy-route.ts new file mode 100644 index 0000000..93fde03 --- /dev/null +++ b/apps/api/src/utils/db-caddy-route.ts @@ -0,0 +1,68 @@ +import { existsSync, mkdirSync, readFileSync, unlinkSync, writeFileSync } from "node:fs"; +import { dirname, join } from "node:path"; +import { reloadCaddy } from "../orchestrator/runtime"; +import { config } from "./config"; + +export interface DbRouteRecord { + internalHost: string; + type: string; + publicAccess: boolean; +} + +export interface DbRouteOpts { + routesDir?: string; + reloadFn?: () => Promise; + baseDomain?: string; +} + +const gatewayBaseDomain = (opts?: DbRouteOpts): string => opts?.baseDomain ?? config.caddyBaseDomain; + +export const dbGatewayEligible = (db: DbRouteRecord, opts?: DbRouteOpts): boolean => { + const base = gatewayBaseDomain(opts); + return db.publicAccess && db.type !== "mysql" && Boolean(base) && base !== "localhost"; +}; + +export const dbGatewayHost = (db: DbRouteRecord, opts?: DbRouteOpts): string => + `${db.internalHost}.${gatewayBaseDomain(opts)}`; + +const routePath = (routesDir: string, internalHost: string): string => + join(routesDir, `database-${internalHost}.caddy`); + +const writeAndReload = async (filePath: string, content: string, reloadFn: () => Promise) => { + mkdirSync(dirname(filePath), { recursive: true }); + if (existsSync(filePath) && readFileSync(filePath, "utf8") === content) return; + writeFileSync(filePath, content, "utf8"); + await reloadFn(); +}; + +export const syncDbGatewayRoute = async (db: DbRouteRecord, opts?: DbRouteOpts): Promise => { + const routesDir = opts?.routesDir ?? config.caddyRoutesDir; + const reloadFn = opts?.reloadFn ?? reloadCaddy; + const filePath = routePath(routesDir, db.internalHost); + try { + if (!dbGatewayEligible(db, opts)) { + if (existsSync(filePath)) { + unlinkSync(filePath); + await reloadFn(); + } + return; + } + await writeAndReload(filePath, `${dbGatewayHost(db, opts)} {\n\trespond 404\n}\n`, reloadFn); + } catch (e) { + console.warn(`Could not sync Caddy route for database ${db.internalHost}:`, e instanceof Error ? e.message : e); + } +}; + +export const removeDbGatewayRoute = async (db: DbRouteRecord, opts?: DbRouteOpts): Promise => { + const routesDir = opts?.routesDir ?? config.caddyRoutesDir; + const reloadFn = opts?.reloadFn ?? reloadCaddy; + const filePath = routePath(routesDir, db.internalHost); + try { + if (existsSync(filePath)) { + unlinkSync(filePath); + await reloadFn(); + } + } catch (e) { + console.warn(`Could not remove Caddy route for database ${db.internalHost}:`, e instanceof Error ? e.message : e); + } +}; diff --git a/apps/api/src/utils/permissions.ts b/apps/api/src/utils/permissions.ts index 4dcb249..8ffd6d4 100644 --- a/apps/api/src/utils/permissions.ts +++ b/apps/api/src/utils/permissions.ts @@ -23,3 +23,16 @@ export const hasPermission = (permissions: Permission[], requiredAction: string, return false; }); }; + +const READ_METHODS = new Set(["GET", "HEAD"]); + +const isReadMethod = (method: string): boolean => READ_METHODS.has(method.toUpperCase()); + +export const requiredPermissionForMethod = (method: string): string => + isReadMethod(method) ? "deploy:read" : "deploy:write"; + +export const canPerformMethod = (permissions: Permission[], method: string): boolean => { + const canWrite = hasPermission(permissions, "deploy:write"); + if (isReadMethod(method)) return canWrite || hasPermission(permissions, "deploy:read"); + return canWrite; +}; diff --git a/apps/api/src/utils/proc-group.ts b/apps/api/src/utils/proc-group.ts new file mode 100644 index 0000000..f9f2c90 --- /dev/null +++ b/apps/api/src/utils/proc-group.ts @@ -0,0 +1,12 @@ +import type { ChildProcess } from "node:child_process"; + +export const killChildTree = (child: ChildProcess, signal: NodeJS.Signals = "SIGTERM"): void => { + if (child.pid == null) return; + try { + process.kill(-child.pid, signal); + } catch { + try { + child.kill(signal); + } catch {} + } +}; diff --git a/apps/api/src/utils/server-default.ts b/apps/api/src/utils/server-default.ts index 610aca3..9ab23ba 100644 --- a/apps/api/src/utils/server-default.ts +++ b/apps/api/src/utils/server-default.ts @@ -1,6 +1,8 @@ -import { ensureLocalServer, getServerById, listProjects, listServers } from "../db/repo"; +import type { listServers } from "../db/repo"; import { isServerPreparing } from "../servers/prepare"; +const repo = () => import("../db/repo"); + const isServerHealthy = (s: Awaited>[number]): boolean => { if (isServerPreparing(s.id)) return false; if (s.mode === "agent") { @@ -18,11 +20,13 @@ export const pickBestServer = async ( modeFilter?: string[], ): Promise => { if (preferredServerId && preferredServerId !== excludeServerId) { + const { getServerById } = await repo(); const preferred = await getServerById(preferredServerId); if (preferred && (preferred.mode === "local" || isServerHealthy(preferred))) { if (!modeFilter || modeFilter.includes(preferred.mode)) return preferredServerId; } } + const { ensureLocalServer, listProjects, listServers } = await repo(); const servers = await listServers(); const projects = await listProjects(); const projectCount = new Map(); diff --git a/apps/api/src/utils/ssh.ts b/apps/api/src/utils/ssh.ts index 1d51a7a..85eb52d 100644 --- a/apps/api/src/utils/ssh.ts +++ b/apps/api/src/utils/ssh.ts @@ -12,6 +12,13 @@ export interface SshExecutionOptions { const SSH_KEYS_DIR = join(tmpdir(), "dequel_ssh_keys"); +export class RemoteScriptAbortedError extends Error { + constructor() { + super("Remote script aborted"); + this.name = "RemoteScriptAbortedError"; + } +} + export const ensureSshKey = (server: { host: string; port?: number; @@ -393,7 +400,7 @@ export const runRemoteScript = ( if (options.signal) { options.signal.addEventListener("abort", () => { child.kill("SIGTERM"); - reject(new Error("Remote build script aborted")); + reject(new RemoteScriptAbortedError()); }); } child.stdin.write(script); diff --git a/apps/api/src/utils/tcp-probe.ts b/apps/api/src/utils/tcp-probe.ts new file mode 100644 index 0000000..189285e --- /dev/null +++ b/apps/api/src/utils/tcp-probe.ts @@ -0,0 +1,16 @@ +import { connect } from "node:net"; + +export const probeTcp = (host: string, port: number, timeoutMs = 2000): Promise => + new Promise((resolve) => { + const socket = connect({ host, port }); + let settled = false; + const finish = (ok: boolean) => { + if (settled) return; + settled = true; + socket.destroy(); + resolve(ok); + }; + socket.setTimeout(timeoutMs, () => finish(false)); + socket.once("connect", () => finish(true)); + socket.once("error", () => finish(false)); + }); diff --git a/apps/docs/src/content/docs/databases.md b/apps/docs/src/content/docs/databases.md index 6625c3d..2fbe127 100644 --- a/apps/docs/src/content/docs/databases.md +++ b/apps/docs/src/content/docs/databases.md @@ -64,7 +64,7 @@ If **Public Access** is disabled, no external endpoint is issued — the databas Notes: - MySQL is the exception: its wire protocol begins with a server greeting before the client sends anything, so there is no hostname for the gateway to route on. Exposed MySQL databases keep the direct port path (`:`); open that port in your firewall, or use the dashboard SQL console. -- The gateway terminates TLS with a platform certificate. MongoDB strings skip verification (`tlsAllowInvalidCertificates`). For PostgreSQL, `sslmode=require` skips verification in libpq/psql, but the Node `pg` driver treats it as full verification — there, either append `&uselibpqcompat=true` (Node only; psql rejects that flag) or set `ssl: { rejectUnauthorized: false }` in client code. Redis clients verify certificates by default, so set `rejectUnauthorized: false` in your client configuration. +- The gateway terminates TLS with a Let's Encrypt certificate for the database hostname — Caddy issues it within seconds of enabling Public Access and renews it automatically — so the strings above verify out of the box. In the gaps (right after enabling Public Access, or on a `localhost` setup with no base domain) the gateway falls back to a self-signed certificate: MongoDB strings already skip verification (`tlsAllowInvalidCertificates`); for PostgreSQL append `&uselibpqcompat=true` (Node only; psql rejects that flag) or set `ssl: { rejectUnauthorized: false }` in client code; Redis clients verify certificates by default, so set `rejectUnauthorized: false` in your client configuration while the fallback is in effect. - PostgreSQL clients work either way: classic clients send a plaintext `SSLRequest` first (the gateway answers it locally), and direct-TLS clients (`sslmode=direct`) announce the hostname immediately. - Every new connection is checked against the database status, the Public Access toggle, and the optional IP allowlist before any bytes are forwarded. diff --git a/apps/web/src/api/client.ts b/apps/web/src/api/client.ts index 6598ca3..9fc71a1 100644 --- a/apps/web/src/api/client.ts +++ b/apps/web/src/api/client.ts @@ -18,11 +18,11 @@ import type { SmtpSettingsStatus, Volume, } from "../types"; -import { BASE, apiFetch } from "./http"; +import { apiFetch, BASE } from "./http"; -export * from "./http"; export * from "./auth"; export * from "./databases"; +export * from "./http"; // Projects export const listProjects = () => apiFetch("/projects"); @@ -394,15 +394,6 @@ export const removeRepoHook = (owner: string, repo: string) => export const setEnvVar = (projectId: string, key: string, value: string, environment?: string) => createEnvVar(projectId, { key, value, environment }); -export const uploadSourceZip = (file: File) => { - const formData = new FormData(); - formData.append("file", file); - return apiFetch<{ filePath: string }>("/upload", { - method: "POST", - body: formData, - }); -}; - // Shared Environment Variables export const listSharedEnvVars = (environment?: string) => apiFetch(`/shared-env-vars${environment ? `?environment=${environment}` : ""}`); diff --git a/apps/web/src/api/databases.ts b/apps/web/src/api/databases.ts index 354d2ec..4bc0918 100644 --- a/apps/web/src/api/databases.ts +++ b/apps/web/src/api/databases.ts @@ -32,6 +32,8 @@ export const getDatabaseCredentials = (id: string) => externalConnectionString: string | null; externalHost: string | null; externalPort: number | null; + externalReachable: boolean | null; + warning: string | null; }>(`/databases/${id}/credentials`); export const startDatabase = (id: string) => apiFetch(`/databases/${id}/start`, { method: "POST" }); export const stopDatabase = (id: string) => apiFetch(`/databases/${id}/stop`, { method: "POST" }); diff --git a/apps/web/src/components/databases/DatabaseCard.tsx b/apps/web/src/components/databases/DatabaseCard.tsx index 823eb30..7077f1f 100644 --- a/apps/web/src/components/databases/DatabaseCard.tsx +++ b/apps/web/src/components/databases/DatabaseCard.tsx @@ -209,6 +209,12 @@ export function DatabaseCard({ database, project, onChanged, onDelete }: Databas {credentialsError}

)} + + {credentials?.warning && ( +

+ {credentials.warning} +

+ )}
diff --git a/apps/web/src/components/databases/DatabaseOverview.tsx b/apps/web/src/components/databases/DatabaseOverview.tsx index 41c6fed..a66bf3b 100644 --- a/apps/web/src/components/databases/DatabaseOverview.tsx +++ b/apps/web/src/components/databases/DatabaseOverview.tsx @@ -7,7 +7,11 @@ import { Input } from "../ui/input"; interface DatabaseOverviewProps { database: Database; - creds: { internalConnectionString?: string; externalConnectionString?: string } | null; + creds?: { + internalConnectionString?: string | null; + externalConnectionString?: string | null; + warning?: string | null; + } | null; } export function DatabaseOverview({ database, creds }: DatabaseOverviewProps) { @@ -50,6 +54,11 @@ export function DatabaseOverview({ database, creds }: DatabaseOverviewProps) { {copied ? "Copied" : "Copy"}
+ {creds?.warning && ( +

+ {creds.warning} +

+ )}
diff --git a/apps/web/src/components/settings/LlmKeysSection.tsx b/apps/web/src/components/settings/LlmKeysSection.tsx index 0ed86e4..54ea302 100644 --- a/apps/web/src/components/settings/LlmKeysSection.tsx +++ b/apps/web/src/components/settings/LlmKeysSection.tsx @@ -38,15 +38,20 @@ export function LlmKeysSection() { const configured = data?.filter((k) => k.configured) ?? []; const currentStatus = data?.find((k) => k.provider === provider); const currentMeta = PROVIDERS.find((p) => p.id === provider) ?? PROVIDERS[0]; + const missingKey = !currentStatus?.configured && provider !== "ollama" && !apiKey.trim(); useEffect(() => { setBaseURL(currentStatus?.baseUrl ?? currentMeta.defaultBaseUrl ?? ""); setApiKey(""); - setSaveResult(null); }, [provider, currentStatus?.baseUrl, currentMeta.defaultBaseUrl]); + useEffect(() => { + setSaveResult(null); + }, [provider]); + const save = async (e: React.FormEvent) => { e.preventDefault(); + if (missingKey) return; setSaveResult(null); setSaving(true); try { @@ -294,13 +299,18 @@ export function LlmKeysSection() { )} {saveResult}

+ ) : missingKey && !saving ? ( +

+ + Enter an API key to save. +

) : ( )}
@@ -140,9 +148,7 @@ export function ServersSection() { -

- {s.mode === "agent" ? `WireGuard P2P ${s.agentVersion || ""}` : `SSH (${s.sshUser || "root"}@${s.host}:${s.port})`} -

+

{connectionLabel(s)}

{s.mode !== "local" && ( )} @@ -193,9 +201,7 @@ export function ServersSection() { variant="outline" className="font-mono text-[11px] bg-black/40 text-zinc-300 border-border/60" > - {s.mode === "agent" - ? `WireGuard P2P ${s.agentVersion || ""}` - : `SSH (${s.sshUser || "root"}@${s.host}:${s.port})`} + {connectionLabel(s)} diff --git a/apps/web/src/components/settings/SshKeyPoolSection.tsx b/apps/web/src/components/settings/SshKeyPoolSection.tsx index d54ebe5..c79a955 100644 --- a/apps/web/src/components/settings/SshKeyPoolSection.tsx +++ b/apps/web/src/components/settings/SshKeyPoolSection.tsx @@ -1,5 +1,5 @@ import { useQuery } from "@tanstack/react-query"; -import { KeyRound, Plus, Shield, Trash2, Upload } from "lucide-react"; +import { KeyRound, Plus, Shield, ShieldAlert, Trash2, Upload } from "lucide-react"; import { useRef, useState } from "react"; import * as api from "../../api/client"; import { Badge } from "../ui/badge"; @@ -18,9 +18,13 @@ export function SshKeyPoolSection() { const [name, setName] = useState(""); const [privateKey, setPrivateKey] = useState(""); const [isAdding, setIsAdding] = useState(false); + const [addError, setAddError] = useState(""); + const [isSubmitting, setIsSubmitting] = useState(false); const [deletingId, setDeletingId] = useState(null); const fileInputRef = useRef(null); + const deletingKey = keys.find((k) => k.id === deletingId); + const detectedType = privateKey.includes("ED25519") ? "Ed25519" : privateKey.includes("RSA") @@ -48,11 +52,24 @@ export function SshKeyPoolSection() { const add = async (e: React.FormEvent) => { e.preventDefault(); if (!name.trim() || !privateKey.trim()) return; - await api.createSshKey({ name: name.trim(), privateKey: privateKey.trim() }); - setName(""); - setPrivateKey(""); - setIsAdding(false); - refetch(); + setAddError(""); + setIsSubmitting(true); + try { + await api.createSshKey({ name: name.trim(), privateKey: privateKey.trim() }); + setName(""); + setPrivateKey(""); + setIsAdding(false); + refetch(); + } catch (err) { + setAddError(err instanceof Error ? err.message : "Failed to add SSH key"); + } finally { + setIsSubmitting(false); + } + }; + + const toggleForm = () => { + setAddError(""); + setIsAdding((open) => !open); }; const handleDelete = async () => { @@ -78,7 +95,7 @@ export function SshKeyPoolSection() {
@@ -218,7 +246,7 @@ export function SshKeyPoolSection() { variant="outline" className="font-mono text-[11px] bg-black/40 text-zinc-400 border-border/60" > - {k.fingerprint || "SHA256:..."} + {k.fingerprint || "0123456789abcdef0123456789abcdef"} @@ -253,7 +281,7 @@ export function SshKeyPoolSection() { Key Identifier - Fingerprint + Key Hash Tags Added Date Actions @@ -273,7 +301,7 @@ export function SshKeyPoolSection() { variant="outline" className="font-mono text-[11px] bg-black/40 text-zinc-400 border-border/60" > - {k.fingerprint || "SHA256:..."} + {k.fingerprint || "0123456789abcdef0123456789abcdef"} @@ -320,8 +348,8 @@ export function SshKeyPoolSection() { Delete SSH Key - Are you sure you want to remove this key from the pool? Servers relying on this key will lose remote - management access. + Are you sure you want to remove "{deletingKey?.name ?? "this key"}" from the pool? Servers relying on this + key will lose remote management access. diff --git a/apps/web/src/routes/CreateProjectPage.tsx b/apps/web/src/routes/CreateProjectPage.tsx index a55bf67..153c21f 100644 --- a/apps/web/src/routes/CreateProjectPage.tsx +++ b/apps/web/src/routes/CreateProjectPage.tsx @@ -4,6 +4,7 @@ import { AlertCircle, ArrowLeft, Rocket, Server } from "lucide-react"; import { useEffect, useState } from "react"; import * as api from "../api/client"; import { getGithubIntegration } from "../api/client"; +import { ApiError } from "../api/http"; import { BuildStrategySection, type ComposeServiceRow } from "../components/project/create/BuildStrategySection"; import { CreationStatusOverlay } from "../components/project/create/CreationStatusOverlay"; import { DeploymentTargetSection, getDeploymentTargets } from "../components/project/create/DeploymentTargetSection"; @@ -80,6 +81,12 @@ export function CreateProjectPage() { "idle" | "creating_project" | "creating_envs" | "done" | "error" >("idle"); const [errorMessage, setErrorMessage] = useState(""); + const [errorStatus, setErrorStatus] = useState(null); + + const clearError = () => { + setErrorMessage(""); + setErrorStatus(null); + }; useEffect(() => { getGithubIntegration() @@ -124,10 +131,12 @@ export function CreateProjectPage() { const handleRetry = () => { setSubmittingStatus("idle"); - setErrorMessage(""); + clearError(); }; const handleSubmit = async () => { + clearError(); + if (!name.trim()) { setErrorMessage("Please enter a project name."); return; @@ -144,16 +153,8 @@ export function CreateProjectPage() { } setSubmittingStatus("creating_project"); - setErrorMessage(""); try { - let finalRepoUrl = repoUrl; - - if (sourceType === "upload" && zipFile) { - const uploadRes = await api.uploadSourceZip(zipFile); - finalRepoUrl = uploadRes.filePath; - } - let composeServicesPayload: string | undefined; if (buildType === "compose" && composeServicesList.length > 0) { const validServices = composeServicesList.filter((s) => s.serviceName.trim()); @@ -167,8 +168,8 @@ export function CreateProjectPage() { description: description.trim() || undefined, serverId: getDeploymentTargets(servers).some((s) => s.id === serverId) ? serverId : "local", sourceType, - repoUrl: finalRepoUrl, - repoBranch: repoBranch.trim() || undefined, + repoUrl: sourceType === "upload" ? undefined : repoUrl, + repoBranch: sourceType === "upload" ? undefined : repoBranch.trim() || undefined, buildType, projectType, sourceDir: sourceDir.trim() || undefined, @@ -206,13 +207,26 @@ export function CreateProjectPage() { await api.linkSharedEnvVars(project.id, stagedSharedVarIds); } + if (sourceType === "upload" && zipFile) { + const form = new FormData(); + form.set("sourceType", "upload"); + form.set("projectId", project.id); + form.set("archive", zipFile); + await api.createDeployment(form); + } + setSubmittingStatus("done"); setTimeout(() => { navigate({ to: "/project/$projectId", params: { projectId: project.id }, search: { tab: "deployments" } }); }, 1000); - } catch (err: any) { + } catch (err: unknown) { setSubmittingStatus("error"); - setErrorMessage(err.message || "Failed to create project."); + if (err instanceof ApiError) { + setErrorStatus(err.status); + setErrorMessage(err.message || "Failed to create project."); + } else { + setErrorMessage(err instanceof Error && err.message ? err.message : "Failed to create project."); + } } }; @@ -297,12 +311,22 @@ export function CreateProjectPage() { /> {/* Environment Variables (with File Upload, Key-Value builder, and Bulk paste) */} - + {/* Error Message Display */} {errorMessage && (
+ {errorStatus !== null && ( + + HTTP {errorStatus} + + )} {errorMessage}
)} diff --git a/apps/web/src/routes/Login.tsx b/apps/web/src/routes/Login.tsx index 6e7a379..6079aa1 100644 --- a/apps/web/src/routes/Login.tsx +++ b/apps/web/src/routes/Login.tsx @@ -135,8 +135,8 @@ export function Login() { -
- dequel.local/dashboard +
+ {typeof window !== "undefined" ? window.location.origin : "http://localhost:3000"}
diff --git a/docker-compose.yml b/docker-compose.yml index 37ec6d4..18c6c45 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -102,13 +102,19 @@ services: - sh - -c - | - apt-get update -qq > /dev/null 2>&1 && apt-get install -y -qq libpam-modules > /dev/null 2>&1 && cp /app/dequel /etc/pam.d/dequel && exec python3 -u /app/pam-server.py + apt-get update -qq > /dev/null 2>&1 && apt-get install -y -qq libpam-modules > /dev/null 2>&1 && cp /app/dequel /etc/pam.d/dequel + sync_host_etc() { + cp /host-etc/passwd /etc/passwd 2>/dev/null || true + cp /host-etc/shadow /etc/shadow 2>/dev/null || true + cp /host-etc/group /etc/group 2>/dev/null || true + } + sync_host_etc + ( while true; do sleep 2; sync_host_etc; done ) & + exec python3 -u /app/pam-server.py volumes: - ./scripts/auth/pam-server.py:/app/pam-server.py:ro - ./scripts/auth/dequel:/app/dequel:ro - - /etc/passwd:/etc/passwd:ro - - /etc/shadow:/etc/shadow:ro - - /etc/group:/etc/group:ro + - /etc:/host-etc:ro networks: dequel: aliases: @@ -161,10 +167,12 @@ services: DATABASE_URL: postgresql://${POSTGRES_USER:-dequel}:${POSTGRES_PASSWORD:-dequel}@postgres:5432/${POSTGRES_DB:-dequel} CADDY_BASE_DOMAIN: ${CADDY_BASE_DOMAIN:-localhost} GATEWAY_CERT_DIR: /app/data + CADDY_DATA_DIR: /caddy-data/caddy ports: - "443:443" volumes: - api-data:/app/data + - caddy-data:/caddy-data:ro depends_on: postgres: condition: service_healthy @@ -198,6 +206,7 @@ services: volumes: - ./infra/caddy/Caddyfile:/etc/caddy/Caddyfile:ro - ./infra/caddy/routes:/etc/caddy/routes + - caddy-data:/data depends_on: api: condition: service_healthy @@ -388,6 +397,7 @@ networks: volumes: buildkit-data: + caddy-data: diag-sandbox-data: redis-data: postgres-data: From b7581b7786dfa290217c914a24d94a69deec2e5b Mon Sep 17 00:00:00 2001 From: Lftobs Date: Fri, 9 Oct 2026 04:10:54 +0100 Subject: [PATCH 05/17] fix(databases): use real certificates and server hosts for connections Wire Caddy certificate detection into connection strings and resolve direct database endpoints from their assigned server. Also begins refactoring Railpack process cancellation. --- apps/api/src/api/databases/index.ts | 23 ++++++++++++++--- apps/api/src/orchestrator/railpack.ts | 33 ++++++------------------- apps/api/src/utils/config.ts | 1 + apps/docs/src/content/docs/databases.md | 4 +-- docker-compose.yml | 2 ++ 5 files changed, 31 insertions(+), 32 deletions(-) diff --git a/apps/api/src/api/databases/index.ts b/apps/api/src/api/databases/index.ts index 39acbc5..9157de7 100644 --- a/apps/api/src/api/databases/index.ts +++ b/apps/api/src/api/databases/index.ts @@ -6,11 +6,13 @@ import { deleteDatabase, getDatabaseById, getProjectById, + getServerById, listAllDatabases, listDatabases, updateDatabaseSettings, updateDatabaseStatus, } from "../../db/repo"; +import { loadCaddyCert } from "../../tunnels/cert-store"; import type { Database } from "../../types"; import { config } from "../../utils/config"; import { removeDbGatewayRoute, syncDbGatewayRoute } from "../../utils/db-caddy-route"; @@ -62,6 +64,14 @@ const isNonLoopbackIp = (value: string): boolean => { return !value.startsWith("127."); }; +const resolveDbExternalHost = async (serverId: string | null): Promise => { + if (serverId) { + const server = await getServerById(serverId); + if (server && isNonLoopbackIp(server.host)) return server.host; + } + return resolveServerIp(); +}; + const createManagedDatabase = async (body: any, projectId: string | null, set: any) => { if (projectId && !(await getProjectById(projectId))) { set.status = 404; @@ -123,11 +133,14 @@ export const databasesRoutes = new Elysia() if (gatewayHost) { const reachable = await probeTcp(gatewayHost, 443); if (reachable) { + const hasRealCert = config.caddyDataDir + ? (await loadCaddyCert(config.caddyDataDir, gatewayHost)) !== null + : false; return ok({ username: dbRecord.username, password: dbRecord.password, internalConnectionString: dbRecord.connectionString, - externalConnectionString: buildGatewayConnectionString(dbRecord, gatewayHost), + externalConnectionString: buildGatewayConnectionString(dbRecord, gatewayHost, hasRealCert), externalHost: gatewayHost, externalPort: 443, externalReachable: true, @@ -145,7 +158,8 @@ export const databasesRoutes = new Elysia() warning: `Gateway endpoint ${gatewayHost}:443 is not reachable from the network — check the gateway service and DNS.`, }); } - const externalHost = dbRecord.publicAccess && dbRecord.externalPort ? await resolveServerIp() : null; + const externalHost = + dbRecord.publicAccess && dbRecord.externalPort ? await resolveDbExternalHost(dbRecord.serverId ?? null) : null; const usableHost = externalHost && isNonLoopbackIp(externalHost) ? externalHost : null; if (usableHost && dbRecord.externalPort) { const reachable = await probeTcp(usableHost, dbRecord.externalPort); @@ -316,12 +330,13 @@ const buildConnectionString = (dbRecord: DatabaseRecord, host: string, port: num return `${protocol}://${dbRecord.username}:${dbRecord.password}@${host}:${port}/${dbRecord.databaseName}`; }; -const buildGatewayConnectionString = (dbRecord: DatabaseRecord, host: string) => { +const buildGatewayConnectionString = (dbRecord: DatabaseRecord, host: string, hasRealCert: boolean) => { const auth = `${dbRecord.username}:${dbRecord.password}`; const path = `/${dbRecord.databaseName}`; if (dbRecord.type === "redis") return `rediss://:${dbRecord.password}@${host}:443`; if (dbRecord.type === "mongodb") { - return `mongodb://${auth}@${host}:443${path}?authSource=admin&tls=true&tlsAllowInvalidCertificates=true`; + const tls = hasRealCert ? "tls=true" : "tls=true&tlsAllowInvalidCertificates=true"; + return `mongodb://${auth}@${host}:443${path}?authSource=admin&${tls}`; } return `postgresql://${auth}@${host}:443${path}?sslmode=require`; }; diff --git a/apps/api/src/orchestrator/railpack.ts b/apps/api/src/orchestrator/railpack.ts index ec841ae..c3c0e49 100644 --- a/apps/api/src/orchestrator/railpack.ts +++ b/apps/api/src/orchestrator/railpack.ts @@ -1,7 +1,7 @@ import { spawn } from "node:child_process"; import type { Server } from "../types"; import { dockerBin } from "../utils/docker-bin"; -import { killChildTree } from "../utils/proc-group"; +import { killProcessGroup } from "../utils/process-exec"; import { generateDynamicRailpackJson } from "./railpack-config-utils"; export interface RailpackBuildResult { @@ -67,7 +67,7 @@ export class CancelledError extends Error { } } -export const spawnAsync = ( +const spawnAsync = ( cmd: string, args: string[], opts?: { @@ -93,7 +93,7 @@ export const spawnAsync = ( ...(opts?.env ?? process.env), }, cwd: opts?.cwd, - detached: true, + detached: isPosix, }); let stdout = ""; let stderr = ""; @@ -101,13 +101,10 @@ export const spawnAsync = ( let timeoutTimer: ReturnType | null = null; let escalationTimer: ReturnType | null = null; - const onAbort = () => { - if (settled) return; - killChildTree(child, "SIGTERM"); - setTimeout(() => { - if (!settled) { - killChildTree(child, "SIGKILL"); - } + const terminate = () => { + killProcessGroup(child, "SIGTERM"); + escalationTimer = setTimeout(() => { + killProcessGroup(child, "SIGKILL"); }, 5000); if (typeof escalationTimer.unref === "function") escalationTimer.unref(); }; @@ -118,24 +115,8 @@ export const spawnAsync = ( if (opts?.signal) { opts.signal.removeEventListener("abort", onAbort); } - opts.signal.addEventListener("abort", onAbort, { once: true }); }; - const timeout = - opts?.timeoutMs && opts.timeoutMs > 0 - ? setTimeout(() => { - if (settled) return; - opts.onTimeout?.(); - killChildTree(child, "SIGTERM"); - setTimeout(() => { - if (!settled) { - killChildTree(child, "SIGKILL"); - } - }, 5000); - finish(undefined, new Error(`${cmd} timed out after ${opts.timeoutMs}ms`)); - }, opts.timeoutMs) - : null; - const finish = ( result?: { stdout: string; diff --git a/apps/api/src/utils/config.ts b/apps/api/src/utils/config.ts index 8588e89..d0d73bc 100644 --- a/apps/api/src/utils/config.ts +++ b/apps/api/src/utils/config.ts @@ -24,6 +24,7 @@ export const config = { databaseUrl: withFile("DATABASE_URL", ""), workspaceRoot: withFile("WORKSPACE_ROOT", "/app/workspace"), caddyRoutesDir: withFile("CADDY_ROUTES_DIR", "/caddy/routes"), + caddyDataDir: withFile("CADDY_DATA_DIR", ""), port: withFile("PORT", "17474", Number), caddyBaseDomain: withFile("CADDY_BASE_DOMAIN", "localhost"), controlPlaneUrl: withFile("CONTROL_PLANE_URL", ""), diff --git a/apps/docs/src/content/docs/databases.md b/apps/docs/src/content/docs/databases.md index 2fbe127..9664591 100644 --- a/apps/docs/src/content/docs/databases.md +++ b/apps/docs/src/content/docs/databases.md @@ -52,7 +52,7 @@ Exposed databases receive strings that require TLS:
- mongodb://user:password@db-1a2b3c4d.dequel.example.com:443/dbname?authSource=admin&tls=true&tlsAllowInvalidCertificates=true + mongodb://user:password@db-1a2b3c4d.dequel.example.com:443/dbname?authSource=admin&tls=true
@@ -64,7 +64,7 @@ If **Public Access** is disabled, no external endpoint is issued — the databas Notes: - MySQL is the exception: its wire protocol begins with a server greeting before the client sends anything, so there is no hostname for the gateway to route on. Exposed MySQL databases keep the direct port path (`:`); open that port in your firewall, or use the dashboard SQL console. -- The gateway terminates TLS with a Let's Encrypt certificate for the database hostname — Caddy issues it within seconds of enabling Public Access and renews it automatically — so the strings above verify out of the box. In the gaps (right after enabling Public Access, or on a `localhost` setup with no base domain) the gateway falls back to a self-signed certificate: MongoDB strings already skip verification (`tlsAllowInvalidCertificates`); for PostgreSQL append `&uselibpqcompat=true` (Node only; psql rejects that flag) or set `ssl: { rejectUnauthorized: false }` in client code; Redis clients verify certificates by default, so set `rejectUnauthorized: false` in your client configuration while the fallback is in effect. +- The gateway terminates TLS with a Let's Encrypt certificate for the database hostname — Caddy issues it within seconds of enabling Public Access and renews it automatically — so the strings above verify out of the box. In the gaps (right after enabling Public Access, or on a `localhost` setup with no base domain) the gateway falls back to a self-signed certificate: MongoDB strings carry `tlsAllowInvalidCertificates=true` (verification off) until a real certificate exists; for PostgreSQL append `&uselibpqcompat=true` (Node only; psql rejects that flag) or set `ssl: { rejectUnauthorized: false }` in client code; Redis clients verify certificates by default, so set `rejectUnauthorized: false` in your client configuration while the fallback is in effect. - PostgreSQL clients work either way: classic clients send a plaintext `SSLRequest` first (the gateway answers it locally), and direct-TLS clients (`sslmode=direct`) announce the hostname immediately. - Every new connection is checked against the database status, the Public Access toggle, and the optional IP allowlist before any bytes are forwarded. diff --git a/docker-compose.yml b/docker-compose.yml index 62e3df5..fce45c4 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -41,6 +41,7 @@ services: WORKSPACE_ROOT: /app/workspace CADDY_ROUTES_DIR: /caddy/routes CADDY_BASE_DOMAIN: ${CADDY_BASE_DOMAIN:-localhost} + CADDY_DATA_DIR: /caddy-data/caddy DOCKER_NETWORK: dequel_net APP_INTERNAL_PORT: 3000 BUILDKIT_HOST: tcp://buildkit:1234 @@ -55,6 +56,7 @@ services: - /var/run/docker.sock:/var/run/docker.sock - railpack-cache:/tmp/railpack - api-data:/app/data + - caddy-data:/caddy-data:ro - backup-data:/data/backups - /etc/passwd:/etc/passwd:ro - /etc/group:/etc/group:ro From 211d0f6b34d9a511b06f0ee27d0e130c0fab354b Mon Sep 17 00:00:00 2001 From: Lftobs Date: Fri, 9 Oct 2026 10:34:48 +0100 Subject: [PATCH 06/17] docs(changelog): include gateway, failover and database fixes in v0.4.0 --- apps/docs/src/content/changelogs/v0.4.0.md | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/apps/docs/src/content/changelogs/v0.4.0.md b/apps/docs/src/content/changelogs/v0.4.0.md index b6156a9..8cf4412 100644 --- a/apps/docs/src/content/changelogs/v0.4.0.md +++ b/apps/docs/src/content/changelogs/v0.4.0.md @@ -14,6 +14,7 @@ date: "2026-10-03" - Track anonymous usage events ([8abdf74](https://github.com/Lftobs/dequel/commit/8abdf7420877621e10ed277ccb63a70df97a3523)) - Add inline MongoDB document editing ([4ead155](https://github.com/Lftobs/dequel/commit/4ead15502a7aaa24cd09b2477409dea1c277a494)) - Route TLS database traffic on port 443 ([8111b92](https://github.com/Lftobs/dequel/commit/8111b926be66562f1c4b89757eeaea090df19a8d)) +- Forward remote database traffic over SSH ([42fef01](https://github.com/Lftobs/dequel/commit/42fef016ab9140d137835fb5a829da5ca55ea177)) ### Improvements @@ -39,3 +40,15 @@ date: "2026-10-03" - Clean stale routes from their owning server ([fd2d5e3](https://github.com/Lftobs/dequel/commit/fd2d5e30a6c93c36398404dc360277b30f75b87a)) - Stop previous containers and replicas on redeploy ([ee35155](https://github.com/Lftobs/dequel/commit/ee351553cff825c642195c107169123f34d484ec)) - Use the real container name as the route primary ([aa64121](https://github.com/Lftobs/dequel/commit/aa6412131cde9c4542d543a1609d12ac2afa5aad)) +- Resolve zombie process leaks and subprocess lifecycle management ([6284ed6](https://github.com/Lftobs/dequel/commit/6284ed6299d00fa815e912973d2ea14a1045c986)) +- Address review feedback on stream decoding, process race conditions, and duplicate types ([35ecc92](https://github.com/Lftobs/dequel/commit/35ecc924595e23ce0fc0d614c0e4efffc7b9394c)) +- Pass the project server to rollback cleanup ([aa61f92](https://github.com/Lftobs/dequel/commit/aa61f92afafa9c34b90cb757e57d13292b139777)) +- Count only replicas Docker actually removed ([511d0dc](https://github.com/Lftobs/dequel/commit/511d0dc64eee125f630b37e552049f8ead831ae6)) +- Fail closed when a database allows no CIDR ([3304b53](https://github.com/Lftobs/dequel/commit/3304b530cc7c7a26c8d4178938410cd5b8009111)) +- Let the server finish closing before exiting ([6940ea0](https://github.com/Lftobs/dequel/commit/6940ea04f8a65e5aec9abfe461f44cc48c948d8f)) +- Don't report provisioning before credentials are revealed ([8798c37](https://github.com/Lftobs/dequel/commit/8798c377be26ff2bd56c7623656c0da468108b7d)) +- Live-probe targets, rate-limit auto moves, clean old containers ([ee6bc0e](https://github.com/Lftobs/dequel/commit/ee6bc0ec6dd989598a40844c017607e9238255ea)) +- Run the railpack config generator on remote hosts ([850bf5b](https://github.com/Lftobs/dequel/commit/850bf5b66aacd7e267d9a4de52e8c3fd407d62d0)) +- Access-log ingress routes and target runtime logs at the deployment's server ([34a8638](https://github.com/Lftobs/dequel/commit/34a863854e580fd48f58d77b778e29d38da2b425)) +- Bug-sheet fixes, test-suite repair, and Let's Encrypt certs for database hosts ([31cbea0](https://github.com/Lftobs/dequel/commit/31cbea0fcf6091ea4b1a991e23a0a7cec34812d4)) +- Use real certificates and server hosts for connections ([b7581b7](https://github.com/Lftobs/dequel/commit/b7581b7786dfa290217c914a24d94a69deec2e5b)) From 91e928bd73edfabfe33e87d84da4bcea2fe87b8c Mon Sep 17 00:00:00 2001 From: Lftobs Date: Fri, 9 Oct 2026 11:08:46 +0100 Subject: [PATCH 07/17] docs: clarify database exposure and MySQL access paths --- apps/docs/src/content/docs/databases.md | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/apps/docs/src/content/docs/databases.md b/apps/docs/src/content/docs/databases.md index 9664591..1c8d61d 100644 --- a/apps/docs/src/content/docs/databases.md +++ b/apps/docs/src/content/docs/databases.md @@ -39,6 +39,14 @@ To inject this secret into your container runtime, navigate to **Environment Var When **Public Access** is enabled, Dequel issues an external connection string that travels through the shared HTTPS port (443). The gateway reads the hostname from the TLS handshake and forwards the raw bytes to the database, so only ports 22, 80, and 443 ever need to be open — no per-database firewall rules, no random high ports. +### How exposing works for PostgreSQL, MongoDB, and Redis + +PostgreSQL, MongoDB, and Redis all expose through the gateway the same way. To expose one of them: + +1. Open the database in the dashboard and turn on **Public Access**. +2. Dequel writes a gateway route for `db-.` and Caddy automatically provisions a Let's Encrypt certificate for that hostname. This takes seconds and renews on its own. +3. Connect from any machine with the external string Dequel hands out. Your client opens TLS to port 443 and announces the hostname; the gateway checks the database status, the Public Access toggle, and the optional IP allowlist, then forwards raw bytes to the database — including databases that run on a remote server in your cluster. + Setup requirements: - Set `CADDY_BASE_DOMAIN` to your domain (for example `dequel.example.com`). @@ -61,9 +69,12 @@ Exposed databases receive strings that require TLS: If **Public Access** is disabled, no external endpoint is issued — the database only receives its internal cluster string. +### MySQL — direct port instead + +MySQL is the one exception. Its wire protocol sends a server greeting before the client says anything, so there is no client-announced hostname for the gateway to route on. Exposed MySQL databases therefore keep the direct port path: Dequel publishes a random port on the database's own server, and the credentials panel shows the endpoint (`:`) along with a warning while the port is still blocked. Open that port in your server's firewall or security group, or use the dashboard SQL console instead. + Notes: -- MySQL is the exception: its wire protocol begins with a server greeting before the client sends anything, so there is no hostname for the gateway to route on. Exposed MySQL databases keep the direct port path (`:`); open that port in your firewall, or use the dashboard SQL console. - The gateway terminates TLS with a Let's Encrypt certificate for the database hostname — Caddy issues it within seconds of enabling Public Access and renews it automatically — so the strings above verify out of the box. In the gaps (right after enabling Public Access, or on a `localhost` setup with no base domain) the gateway falls back to a self-signed certificate: MongoDB strings carry `tlsAllowInvalidCertificates=true` (verification off) until a real certificate exists; for PostgreSQL append `&uselibpqcompat=true` (Node only; psql rejects that flag) or set `ssl: { rejectUnauthorized: false }` in client code; Redis clients verify certificates by default, so set `rejectUnauthorized: false` in your client configuration while the fallback is in effect. - PostgreSQL clients work either way: classic clients send a plaintext `SSLRequest` first (the gateway answers it locally), and direct-TLS clients (`sslmode=direct`) announce the hostname immediately. - Every new connection is checked against the database status, the Public Access toggle, and the optional IP allowlist before any bytes are forwarded. From e70a76c33ca2a193f8e2a0b3f3b1ee75137e625c Mon Sep 17 00:00:00 2001 From: Lftobs Date: Fri, 9 Oct 2026 12:56:14 +0100 Subject: [PATCH 08/17] refactor(config): separate fixed values from tunables Persist a generated encryption key during installation and migrate existing encrypted secrets on startup. Configure GitHub integration through the dashboard instead of config seeding. --- apps/api/src/api/github/index.ts | 3 +- apps/api/src/db/migrate.ts | 35 +---- apps/api/src/index.ts | 2 + apps/api/src/utils/config-loader.ts | 36 ++--- apps/api/src/utils/config.ts | 33 +++-- apps/api/src/utils/crypto.ts | 2 + .../src/content/docs/agent-caddy-routes.md | 18 ++- apps/docs/src/content/docs/installation.md | 13 +- apps/docs/src/content/docs/system-config.md | 133 +++++++++++------- docker-compose.yml | 3 +- scripts/dequel | 23 +++ scripts/install.sh | 4 +- 12 files changed, 168 insertions(+), 137 deletions(-) diff --git a/apps/api/src/api/github/index.ts b/apps/api/src/api/github/index.ts index b877def..91da7a3 100644 --- a/apps/api/src/api/github/index.ts +++ b/apps/api/src/api/github/index.ts @@ -9,7 +9,6 @@ import { setGithubIntegration, } from "../../db/repo"; import { orchestrator } from "../../orchestrator"; -import { config } from "../../utils/config"; import { fail, ok } from "../response"; const validateToken = async (token: string): Promise => { @@ -249,7 +248,7 @@ export const githubRoutes = new Elysia({ prefix: "/github" }) } const webhookUrl = `${publicUrl(request).origin}/api/github/webhook`; const integration = await getGithubIntegration(); - const secret = integration?.webhookSecret || config.githubWebhookSecret; + const secret = integration?.webhookSecret || ""; try { const hooks = await fetchGitHub(`/repos/${params.owner}/${params.repo}/hooks`, token); diff --git a/apps/api/src/db/migrate.ts b/apps/api/src/db/migrate.ts index 8707c6d..12d63ee 100644 --- a/apps/api/src/db/migrate.ts +++ b/apps/api/src/db/migrate.ts @@ -1,7 +1,5 @@ import { migrate as drizzleMigrate } from "drizzle-orm/node-postgres/migrator"; -import { config } from "../utils/config"; import { getDb } from "./client"; -import { getGithubIntegration, setGithubIntegration } from "./repo/github"; export const migrate = async () => { const db = await getDb(); @@ -13,41 +11,10 @@ export const migrate = async () => { const msg = err?.message ?? String(err); const innerCode = err?.cause?.code; if (err?.code === "42P07" || innerCode === "42P07" || msg.includes("already exists")) { - console.log("[Migrate] Some tables already exist, continuing to seed"); + console.log("[Migrate] Some tables already exist, continuing"); } else { console.error("[Migrate] Migration failed:", msg); throw err; } } - - await seedFromConfig(); -}; - -const seedFromConfig = async () => { - if (config.githubClientId && config.githubClientSecret) { - const existing = await getGithubIntegration(); - if (existing) { - if ( - existing.clientId !== config.githubClientId || - existing.clientSecret !== config.githubClientSecret || - (config.githubWebhookSecret && existing.webhookSecret !== config.githubWebhookSecret) - ) { - await setGithubIntegration({ - clientId: config.githubClientId, - clientSecret: config.githubClientSecret, - appName: config.githubAppName, - webhookSecret: config.githubWebhookSecret || undefined, - }); - console.log("[Config] Synced GitHub integration from config file"); - } - } else { - await setGithubIntegration({ - clientId: config.githubClientId, - clientSecret: config.githubClientSecret, - appName: config.githubAppName, - webhookSecret: config.githubWebhookSecret || undefined, - }); - console.log("[Config] Seeded GitHub integration from config file"); - } - } }; diff --git a/apps/api/src/index.ts b/apps/api/src/index.ts index 4756b84..199fd9c 100644 --- a/apps/api/src/index.ts +++ b/apps/api/src/index.ts @@ -6,6 +6,7 @@ import { apiRoutes } from "./api"; import { startBackupScheduler } from "./backup/scheduler"; import { startDatabaseMonitoring } from "./databases/manager"; import { getDb } from "./db/db-provider"; +import { migrateEncryptionKeys } from "./db/encryption-migration"; import { migrate } from "./db/migrate"; import { ensureLocalServer } from "./db/repo"; import { markInterruptedDiagRuns } from "./db/repo/diag-runs"; @@ -34,6 +35,7 @@ const bootstrap = async () => { initAuth(jwtSecret); await migrate(); + await migrateEncryptionKeys().catch((err) => console.error("[EncryptionMigration] failed:", err)); await ensureLocalServer(); const interrupted = await markInterruptedDiagRuns().catch(() => 0); if (interrupted > 0) console.log(`[Fixdiag] Marked ${interrupted} interrupted diagnosis run(s) as failed`); diff --git a/apps/api/src/utils/config-loader.ts b/apps/api/src/utils/config-loader.ts index 02a81a6..d9463f6 100644 --- a/apps/api/src/utils/config-loader.ts +++ b/apps/api/src/utils/config-loader.ts @@ -5,24 +5,24 @@ import { resolve } from "node:path"; const XDG_CONFIG_HOME = process.env.XDG_CONFIG_HOME || `${homedir()}/.config`; export interface FileConfig { - port?: number; - databaseUrl?: string; - workspaceRoot?: string; - caddyRoutesDir?: string; - caddyBaseDomain?: string; - dockerNetwork?: string; - appInternalPort?: number; - buildkitHost?: string; - envEncryptionKey?: string; - redisUrl?: string; - queueConcurrency?: number; - queueRetryMax?: number; - queueRetryBaseMs?: number; - alertEvalIntervalMs?: number; - githubClientId?: string; - githubClientSecret?: string; - githubAppName?: string; - githubWebhookSecret?: string; + CADDY_BASE_DOMAIN?: string; + CONTROL_PLANE_URL?: string; + AGENT_TUNNEL_URL?: string; + QUEUE_CONCURRENCY?: number; + QUEUE_RETRY_MAX?: number; + QUEUE_RETRY_BASE_MS?: number; + ALERT_EVAL_INTERVAL_MS?: number; + FAILURE_SWEEP_INTERVAL_MS?: number; + GRAFANA_URL?: string; + GRAFANA_USER?: string; + GRAFANA_PASS?: string; + WIREGUARD_SERVER_CONTAINER?: string; + WIREGUARD_SERVER_PUBLIC_KEY?: string; + WIREGUARD_SERVER_ENDPOINT?: string; + WIREGUARD_SERVER_IP?: string; + WIREGUARD_PEER_CIDR?: string; + FAILOVER_DISABLED?: string; + FAILOVER_MIN_INTERVAL_MS?: number; } const searchPaths = (): string[] => { diff --git a/apps/api/src/utils/config.ts b/apps/api/src/utils/config.ts index d0d73bc..7a6e3c4 100644 --- a/apps/api/src/utils/config.ts +++ b/apps/api/src/utils/config.ts @@ -1,4 +1,5 @@ import { loadFileConfig } from "./config-loader"; +import { DEFAULT_ENV_ENCRYPTION_KEY } from "./crypto"; const fileConfig = loadFileConfig(); @@ -12,36 +13,41 @@ const withFile = (key: string, envDefault: string, transform?: (v: string) => return transform ? transform(envDefault) : (envDefault as unknown as T); }; +const withEnv = (key: string, envDefault: string, transform?: (v: string) => T): T => { + const envVal = process.env[key]; + if (envVal !== undefined) { + return transform ? transform(envVal) : (envVal as unknown as T); + } + return transform ? transform(envDefault) : (envDefault as unknown as T); +}; + const SYSTEM = { dockerNetwork: "dequel_net", buildkitHost: "tcp://buildkit:1234", redisUrl: "redis://redis:6379", dequelSourceRepoUrl: "https://github.com/Lftobs/dequel.git", + workspaceRoot: "/app/workspace", + caddyRoutesDir: "/caddy/routes", + caddyDataDir: "/caddy-data/caddy", + port: 3001, + appInternalPort: 3000, + gatewayCertDir: "/app/data", + dequelSlackWebhookUrl: "", + dequelSlackChannel: "", } as const; export const config = { ...SYSTEM, - databaseUrl: withFile("DATABASE_URL", ""), - workspaceRoot: withFile("WORKSPACE_ROOT", "/app/workspace"), - caddyRoutesDir: withFile("CADDY_ROUTES_DIR", "/caddy/routes"), - caddyDataDir: withFile("CADDY_DATA_DIR", ""), - port: withFile("PORT", "17474", Number), + databaseUrl: withEnv("DATABASE_URL", ""), caddyBaseDomain: withFile("CADDY_BASE_DOMAIN", "localhost"), controlPlaneUrl: withFile("CONTROL_PLANE_URL", ""), agentTunnelUrl: withFile("AGENT_TUNNEL_URL", ""), - appInternalPort: withFile("APP_INTERNAL_PORT", "17476", Number), - envEncryptionKey: withFile("ENV_ENCRYPTION_KEY", "dev-env-key-change-me"), - dequelSlackWebhookUrl: withFile("DEQUEL_SLACK_WEBHOOK_URL", ""), - dequelSlackChannel: withFile("DEQUEL_SLACK_CHANNEL", ""), + envEncryptionKey: withEnv("ENV_ENCRYPTION_KEY", DEFAULT_ENV_ENCRYPTION_KEY), queueConcurrency: withFile("QUEUE_CONCURRENCY", "3", Number), queueRetryMax: withFile("QUEUE_RETRY_MAX", "5", Number), queueRetryBaseMs: withFile("QUEUE_RETRY_BASE_MS", "5000", Number), alertEvalIntervalMs: withFile("ALERT_EVAL_INTERVAL_MS", "60000", Number), failureSweepIntervalMs: withFile("FAILURE_SWEEP_INTERVAL_MS", "60000", Number), - githubClientId: withFile("GITHUB_CLIENT_ID", ""), - githubClientSecret: withFile("GITHUB_CLIENT_SECRET", ""), - githubAppName: withFile("GITHUB_APP_NAME", "Dequel"), - githubWebhookSecret: withFile("GITHUB_WEBHOOK_SECRET", ""), grafanaUrl: withFile("GRAFANA_URL", "http://grafana:3000"), grafanaUser: withFile("GRAFANA_USER", "admin"), grafanaPass: withFile("GRAFANA_PASS", "admin"), @@ -50,7 +56,6 @@ export const config = { wireguardServerEndpoint: withFile("WIREGUARD_SERVER_ENDPOINT", ""), wireguardServerIp: withFile("WIREGUARD_SERVER_IP", "10.200.0.1"), wireguardPeerCidr: withFile("WIREGUARD_PEER_CIDR", "10.200.0.0/24"), - gatewayCertDir: withFile("GATEWAY_CERT_DIR", "/app/data"), failoverDisabled: withFile("FAILOVER_DISABLED", ""), failoverMinIntervalMs: withFile("FAILOVER_MIN_INTERVAL_MS", "600000", Number), }; diff --git a/apps/api/src/utils/crypto.ts b/apps/api/src/utils/crypto.ts index af77208..0cafe4b 100644 --- a/apps/api/src/utils/crypto.ts +++ b/apps/api/src/utils/crypto.ts @@ -3,6 +3,8 @@ import { createCipheriv, createDecipheriv, randomBytes, scryptSync } from "node: const ALGO = "aes-256-gcm"; const IV_LEN = 12; +export const DEFAULT_ENV_ENCRYPTION_KEY = "dev-env-key-change-me"; + export const deriveKey = (secret: string) => scryptSync(secret, "dequel-env", 32); export const encryptValue = (value: string, secret: string) => { diff --git a/apps/docs/src/content/docs/agent-caddy-routes.md b/apps/docs/src/content/docs/agent-caddy-routes.md index 2b215c0..3302217 100644 --- a/apps/docs/src/content/docs/agent-caddy-routes.md +++ b/apps/docs/src/content/docs/agent-caddy-routes.md @@ -5,11 +5,15 @@ description: How the agent container manages Caddy route files for deployed appl slug: agent-caddy-routes --- -The `CADDY_ROUTES_DIR` environment variable controls where the API writes Caddy route files. On agent servers, the default is `/etc/caddy/routes`. Each deployed application gets its own `.caddy` file in this directory. +The API writes Caddy route files for local deployments to the fixed +`/caddy/routes` directory. Each deployed application gets its own `.caddy` file +in this directory. ## Route Directory -The route directory path is resolved from the `CADDY_ROUTES_DIR` environment variable. In local mode (API and Caddy on the same host), this defaults to `/caddy/routes`. On remote agent servers, the agent writes to `/etc/caddy/routes`. +The local route directory is fixed at `/caddy/routes` (the API creates it on +startup). On remote servers, routes are written to `/etc/caddy/routes` — over +SSH for SSH-mode servers, or by the agent for agent-mode servers. The directory must exist and be writable by the process managing Caddy. The API creates it automatically on startup. @@ -39,7 +43,7 @@ my-app.localhost:80 { output stdout format json } - reverse_proxy my-app-abc12345:17476 { + reverse_proxy my-app-abc12345:3000 { header_up Host {upstream_hostport} } } @@ -47,7 +51,7 @@ my-app.localhost:80 { - **Hostname:** `.` (e.g., `my-app.localhost:80` or `my-app.example.com`) - **Container name:** `-` (e.g., `my-app-abc12345`) -- **Port:** The application's internal port (default `17476`, or `PORT` env var value) +- **Port:** The application's internal port (the platform default is `3000`, or the project's configured port) For custom domains, multiple hostnames are comma-separated on the first line: @@ -57,7 +61,7 @@ my-app.localhost:80, blog.example.com:80 { output stdout format json } - reverse_proxy my-app-abc12345:17476 { + reverse_proxy my-app-abc12345:3000 { header_up Host {upstream_hostport} } } @@ -70,7 +74,7 @@ my-app.localhost:80, blog.example.com:80 { When a deployment completes successfully: 1. The API builds a Caddy snippet using `buildCaddySnippet()` from `apps/api/src/utils/domain-verifier.ts`. -2. The snippet is written to `/.caddy`. +2. The snippet is written to `/caddy/routes/.caddy`. 3. Caddy is reloaded via `caddy reload --config /etc/caddy/Caddyfile`. 4. A route record is persisted to the database with status `active`. @@ -107,7 +111,7 @@ Reload is triggered after every route file change (deploy, rollback, delete, cus ### Route files not appearing -1. Check that `CADDY_ROUTES_DIR` is set correctly and the directory exists. +1. Check that the routes directory exists (`/caddy/routes` locally, `/etc/caddy/routes` on remote servers). 2. Verify the API process has write permissions to the directory. 3. For agent servers, ensure the agent is running and reachable (check heartbeat status in the dashboard). diff --git a/apps/docs/src/content/docs/installation.md b/apps/docs/src/content/docs/installation.md index 72edfde..8dccd37 100644 --- a/apps/docs/src/content/docs/installation.md +++ b/apps/docs/src/content/docs/installation.md @@ -139,16 +139,15 @@ Dequel stores its configuration in `~/.dequel` (or `$DEQUEL_HOME` if set): ## Building from Source -For local development, run the API and web dashboard directly: +For local development, run the API and web dashboard directly. The API uses +fixed paths for build staging and Caddy routes, so create them once: + +```bash +sudo mkdir -p /app/workspace /caddy/routes +``` ```bash # Terminal 1 — API -export DATABASE_PATH=./data/dequel.db \ - WORKSPACE_ROOT=./workspace \ - CADDY_ROUTES_DIR=./infra/caddy/routes \ - CADDY_BASE_DOMAIN=localhost \ - DOCKER_NETWORK=dequel_net \ - APP_INTERNAL_PORT=3000 bun apps/api/src/index.ts # Terminal 2 — Web diff --git a/apps/docs/src/content/docs/system-config.md b/apps/docs/src/content/docs/system-config.md index e258942..2d3b7af 100644 --- a/apps/docs/src/content/docs/system-config.md +++ b/apps/docs/src/content/docs/system-config.md @@ -1,82 +1,111 @@ --- title: System Configuration category: Core Architecture -description: Configure the Dequel platform via environment variables or the dequel.json config file. +description: How Dequel resolves configuration across fixed system values, environment-only secrets, and editable tunables. slug: system-config --- -Dequel's platform-level settings are configured through environment variables or a `dequel.json` config file. Environment variables take precedence over file values. +Dequel splits platform configuration into three tiers: fixed system values that +are compiled into the API, environment-only secrets that never come from a +config file, and tunables that you can set through environment variables or a +`dequel.json` file. For tunables, environment variables take precedence over +file values. -## Config File Location +## Fixed system values -Dequel looks for `dequel.json` in the following order: +These values are compiled into the API. Environment variables and `dequel.json` +entries with the same names are ignored. -1. `DEQUEL_CONFIG` environment variable pointing to an explicit path -2. `~/.config/dequel/dequel.json` -3. `./dequel.json` (next to the binary) -4. `./data/dequel.json` +| Key | Value | Description | +|-----|-------|-------------| +| `port` | `3001` | API listen port. Caddy, container health checks, and Prometheus target this port. | +| `appInternalPort` | `3000` | Port injected into deployed applications as `PORT`, used for reverse-proxy targets. | +| `workspaceRoot` | `/app/workspace` | Build staging directory. | +| `caddyRoutesDir` | `/caddy/routes` | Directory for local Caddy route files. | +| `caddyDataDir` | `/caddy-data/caddy` | Caddy data mount used to detect real TLS certificates. | +| `gatewayCertDir` | `/app/data` | Directory where the gateway stores its self-signed TLS certificate. | +| `dockerNetwork` | `dequel_net` | Docker network for deployments. | +| `buildkitHost` | `tcp://buildkit:1234` | Buildkit daemon address. | +| `redisUrl` | `redis://redis:6379` | Redis queue address. | -## Reference - -### Security - -| Variable | Default | Description | -|----------|---------|-------------| -| `ENV_ENCRYPTION_KEY` | `dev-env-key-change-me` | Key used to encrypt environment variable values and SMTP passwords at rest | +Remote servers receive route files through SSH or the agent job queue, which +write to `/etc/caddy/routes` on the remote host. -### GitHub Integration +## Environment-only settings -Set these to enable the repo picker in the project creation dialog: +These settings are read from environment variables only. `dequel.json` never +overrides them. | Variable | Default | Description | |----------|---------|-------------| -| `GITHUB_CLIENT_ID` | `""` | GitHub OAuth App client ID | -| `GITHUB_CLIENT_SECRET` | `""` | GitHub OAuth App client secret | -| `GITHUB_APP_NAME` | `Dequel` | GitHub App name displayed during OAuth | -| `GITHUB_WEBHOOK_SECRET` | `""` | Secret for GitHub webhook verification | - -Config file equivalent: +| `DATABASE_URL` | `""` | PostgreSQL connection string. The install script generates a random database password, and Docker Compose passes the assembled URL to the API and gateway. | +| `ENV_ENCRYPTION_KEY` | `dev-env-key-change-me` | Key that encrypts secrets at rest: environment variables, SSH keys, LLM provider keys, SMTP passwords, S3 credentials, and GitHub session tokens. The install script generates a random key and stores it in `.env`. | -```json -{ - "githubClientId": "Iv1...", - "githubClientSecret": "...", - "githubAppName": "Dequel", - "githubWebhookSecret": "..." -} -``` +On boot, the API re-encrypts any stored secrets that were written with the +default key so a fresh install key takes effect without manual migration. -On boot, these values seed the `github_integrations` table. You can also update them from the Settings page in the dashboard. +## Tunables -### SMTP - -SMTP settings are not read from environment variables or the config file. Configure them from the Settings page in the dashboard, where you can also send a test email to verify the setup. The password is encrypted at rest using `ENV_ENCRYPTION_KEY`. - -### Ingress +Set tunables through environment variables or the config file. In Docker +installs, Compose passes a subset (such as `CADDY_BASE_DOMAIN` and +`GRAFANA_URL`) from `.env`; add others to the api service's `environment` list +in `docker-compose.yml` to make them effective. | Variable | Default | Description | |----------|---------|-------------| -| `CADDY_BASE_DOMAIN` | `localhost` | Base domain for deployment subdomains. Deployed apps are reachable at `https://.`. GitHub webhook URLs are derived from this automatically. Set to a real domain (e.g. `example.com`) and configure wildcard DNS for auto-SSL. | -| `CADDY_EMAIL` | `""` | Email address for Let's Encrypt SSL certificate expiration notifications | +| `CADDY_BASE_DOMAIN` | `localhost` | Base domain for deployment subdomains. Deployed apps are reachable at `https://.`. Set a real domain and configure wildcard DNS for auto-SSL. | +| `CONTROL_PLANE_URL` | `""` | Public URL of this control plane, used by remote agents. | +| `AGENT_TUNNEL_URL` | `""` | Tunnel endpoint for agent connections. | +| `QUEUE_CONCURRENCY` | `3` | Number of deployment jobs processed in parallel. | +| `QUEUE_RETRY_MAX` | `5` | Maximum retry attempts for failed jobs. | +| `QUEUE_RETRY_BASE_MS` | `5000` | Base delay between job retries. | +| `ALERT_EVAL_INTERVAL_MS` | `60000` | How often alert rules are evaluated. | +| `FAILURE_SWEEP_INTERVAL_MS` | `60000` | How often failed deployments are swept for notifications. | +| `GRAFANA_URL` | `http://grafana:3000` | Grafana base URL shown in the dashboard. | +| `GRAFANA_USER` | `admin` | Grafana sign-in user. | +| `GRAFANA_PASS` | `admin` | Grafana sign-in password. | +| `WIREGUARD_SERVER_CONTAINER` | `""` | Container name of the WireGuard server. | +| `WIREGUARD_SERVER_PUBLIC_KEY` | `""` | Public key of the WireGuard server. | +| `WIREGUARD_SERVER_ENDPOINT` | `""` | WireGuard server endpoint. | +| `WIREGUARD_SERVER_IP` | `10.200.0.1` | WireGuard server IP address. | +| `WIREGUARD_PEER_CIDR` | `10.200.0.0/24` | CIDR range assigned to WireGuard peers. | +| `FAILOVER_DISABLED` | `""` | Set to a non-empty value to disable the failover monitor. | +| `FAILOVER_MIN_INTERVAL_MS` | `600000` | Minimum interval between failover checks for a deployment. | + +`CADDY_EMAIL` is a separate Compose-level setting: set it in `.env` to receive +Let's Encrypt certificate expiration notices from the Caddy service. + +## Config file location + +The API looks for `dequel.json` when resolving tunables, in the following +order: + +1. `DEQUEL_CONFIG` environment variable pointing to an explicit path +2. `~/.config/dequel/dequel.json` +3. `./dequel.json` (next to the process working directory) +4. `./data/dequel.json` -Config file equivalent: +File keys use the same names as the environment variables, in upper snake +case, for example `QUEUE_CONCURRENCY`. Numeric values must be JSON numbers, +not strings. ```json { - "caddyBaseDomain": "example.com", - "caddyEmail": "admin@example.com" + "CADDY_BASE_DOMAIN": "example.com", + "QUEUE_CONCURRENCY": 5, + "ALERT_EVAL_INTERVAL_MS": 120000 } ``` -## Full Config File Example +## GitHub integration -```json -{ - "githubClientId": "Iv1...", - "githubClientSecret": "...", - "githubAppName": "MyDequel", - "githubWebhookSecret": "...", - "envEncryptionKey": "your-secure-key-here" -} -``` +Configure GitHub OAuth and webhooks from **Settings** in the dashboard. Dequel +stores the client ID, client secret, app name, and webhook secret in the +database. No environment variables or config file keys are read for GitHub. + +## SMTP +SMTP settings are not read from environment variables or the config file. +Configure them from the Settings page in the dashboard, where you can also +send a test email to verify the setup. The password is encrypted at rest with +`ENV_ENCRYPTION_KEY`. diff --git a/docker-compose.yml b/docker-compose.yml index fce45c4..ff1e6dc 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -48,8 +48,8 @@ services: DOCKER_BIN: /usr/bin/docker RAILPACK_VERBOSE: "1" RAILPACK_BUILD_TIMEOUT_MS: "1200000" - GRAFANA_URL: http://grafana:3000/grafana POSTGRES_CONTAINER: ${POSTGRES_CONTAINER:-dequel-postgres-1} + ENV_ENCRYPTION_KEY: ${ENV_ENCRYPTION_KEY:-dev-env-key-change-me} volumes: - ./workspace:/app/workspace - ./infra/caddy/routes:/caddy/routes @@ -172,6 +172,7 @@ services: CADDY_BASE_DOMAIN: ${CADDY_BASE_DOMAIN:-localhost} GATEWAY_CERT_DIR: /app/data CADDY_DATA_DIR: /caddy-data/caddy + ENV_ENCRYPTION_KEY: ${ENV_ENCRYPTION_KEY:-dev-env-key-change-me} ports: - "443:443" volumes: diff --git a/scripts/dequel b/scripts/dequel index 401d86f..1df2d7b 100755 --- a/scripts/dequel +++ b/scripts/dequel @@ -22,6 +22,29 @@ success() { printf " ${GREEN}✓${NC} %s\n" "$*"; } warn() { printf " ${AMBER}⚠${NC} %s\n" "$*"; } fail() { printf " ${RED}✗${NC} %s\n" "$*"; exit 1; } +ensure_encryption_key() { + [ -n "${ENV_ENCRYPTION_KEY:-}" ] && return 0 + local key="" + if [ -f "$DEQUEL_HOME/data/dequel.json" ]; then + key="$(sed -n 's/^.*"ENV_ENCRYPTION_KEY"[[:space:]]*:[[:space:]]*"\([^"]*\)".*$/\1/p' "$DEQUEL_HOME/data/dequel.json" | head -n1)" + fi + if [ -z "$key" ]; then + key="$(openssl rand -hex 32 2>/dev/null || dd if=/dev/urandom bs=32 count=1 status=none 2>/dev/null | od -A n -t x1 | tr -d ' \n')" + fi + [ -n "$key" ] || fail "Cannot generate encryption key" + export ENV_ENCRYPTION_KEY="$key" + local env_file="$DEQUEL_HOME/.env" + if [ ! -f "$env_file" ]; then + printf '# Dequel environment configuration\n' > "$env_file" + chmod 600 "$env_file" + fi + if ! grep -q '^ENV_ENCRYPTION_KEY=..*' "$env_file" 2>/dev/null; then + sed -i '/^ENV_ENCRYPTION_KEY=/d' "$env_file" 2>/dev/null || true + echo "ENV_ENCRYPTION_KEY=$key" >> "$env_file" + fi +} +ensure_encryption_key + cmd() { if [ ! -f "$COMPOSE_FILE" ]; then fail "Dequel not found at $DEQUEL_HOME. Run install.sh first or set \$DEQUEL_HOME." diff --git a/scripts/install.sh b/scripts/install.sh index 0ee4722..646da4a 100755 --- a/scripts/install.sh +++ b/scripts/install.sh @@ -166,8 +166,7 @@ prompt_config() { cat > "$INSTALL_DIR/data/dequel.json" < "$INSTALL_DIR/.env" From 7b6e78c0631e7a4cba991e4ee3efbd783d154081 Mon Sep 17 00:00:00 2001 From: Lftobs Date: Fri, 9 Oct 2026 12:58:38 +0100 Subject: [PATCH 09/17] docs(changelog): include config refactor and database docs in v0.4.0 --- apps/docs/src/content/changelogs/v0.4.0.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/apps/docs/src/content/changelogs/v0.4.0.md b/apps/docs/src/content/changelogs/v0.4.0.md index 8cf4412..9fa1e08 100644 --- a/apps/docs/src/content/changelogs/v0.4.0.md +++ b/apps/docs/src/content/changelogs/v0.4.0.md @@ -24,6 +24,8 @@ date: "2026-10-03" - Update database creation engine selection UI ([faef669](https://github.com/Lftobs/dequel/commit/faef6693959dab303340f0245430c48504d08db2)) - Improve SQL query editor ui ([f38b1a6](https://github.com/Lftobs/dequel/commit/f38b1a68fac5d9af7be44f124e6c879b68bf19ff)) - Switch compose file to prebuilt images for api and web ([3e6f917](https://github.com/Lftobs/dequel/commit/3e6f917403b233dcc43d60068ed18ddaf77174b6)) +- Separate fixed values from tunables ([e70a76c](https://github.com/Lftobs/dequel/commit/e70a76c33ca2a193f8e2a0b3f3b1ee75137e625c)) +- Clarify database exposure and MySQL access paths ([91e928b](https://github.com/Lftobs/dequel/commit/91e928bd73edfabfe33e87d84da4bcea2fe87b8c)) - Make dequel dashboard 100% mobile responsive ([905719f](https://github.com/Lftobs/dequel/commit/905719f336fd79c3818fe45cc023d0df7fdedc95)) - Make all settings sections, tables, tabs and dialogs fully mobile responsive ([1049120](https://github.com/Lftobs/dequel/commit/1049120cf5c7243be5d75c4d9474a4ab326f9bb4)) From 41b07b76dec89f474c5722ade3f699394c6b0c04 Mon Sep 17 00:00:00 2001 From: Lftobs Date: Fri, 9 Oct 2026 13:37:59 +0100 Subject: [PATCH 10/17] Create encryption-migration.ts --- apps/api/src/db/encryption-migration.ts | 326 ++++++++++++++++++++++++ 1 file changed, 326 insertions(+) create mode 100644 apps/api/src/db/encryption-migration.ts diff --git a/apps/api/src/db/encryption-migration.ts b/apps/api/src/db/encryption-migration.ts new file mode 100644 index 0000000..b9ae84c --- /dev/null +++ b/apps/api/src/db/encryption-migration.ts @@ -0,0 +1,326 @@ +import { and, eq, isNotNull } from "drizzle-orm"; +import { config } from "../utils/config"; +import { DEFAULT_ENV_ENCRYPTION_KEY, decryptValue, encryptValue } from "../utils/crypto"; +import { getDb } from "./db-provider"; +import { + backupStorageSettings, + environmentVariables, + githubSessions, + llmProviderKeys, + projects, + servers, + sharedEnvVars, + smtpSettings, + sshKeys, +} from "./schema"; + +type Triple = { encrypted: string; iv: string; tag: string }; +type StoredTriple = { enc: string | null; iv: string | null; tag: string | null }; +type StoreResult = { migrated: number; unknown: number }; +type Store = { name: string; run: (activeKey: string) => Promise }; + +const processRows = async ( + rows: R[], + getTriple: (row: R) => StoredTriple, + write: (row: R, next: Triple) => Promise, + activeKey: string, +): Promise => { + let migrated = 0; + let unknown = 0; + for (const row of rows) { + const { enc, iv, tag } = getTriple(row); + if (!enc || !iv || !tag) continue; + try { + decryptValue(enc, iv, tag, activeKey); + continue; + } catch {} + let plain: string; + try { + plain = decryptValue(enc, iv, tag, DEFAULT_ENV_ENCRYPTION_KEY); + } catch { + unknown += 1; + continue; + } + await write(row, encryptValue(plain, activeKey)); + migrated += 1; + } + return { migrated, unknown }; +}; + +const STORES: Store[] = [ + { + name: "environment_variables", + run: async (activeKey) => { + const db = await getDb(); + const rows = await db + .select() + .from(environmentVariables) + .where( + and( + isNotNull(environmentVariables.valueEncrypted), + isNotNull(environmentVariables.valueIv), + isNotNull(environmentVariables.valueTag), + ), + ); + return processRows( + rows, + (r) => ({ enc: r.valueEncrypted, iv: r.valueIv, tag: r.valueTag }), + (r, next) => + db + .update(environmentVariables) + .set({ valueEncrypted: next.encrypted, valueIv: next.iv, valueTag: next.tag }) + .where(eq(environmentVariables.id, r.id)) + .execute(), + activeKey, + ); + }, + }, + { + name: "shared_env_vars", + run: async (activeKey) => { + const db = await getDb(); + const rows = await db + .select() + .from(sharedEnvVars) + .where( + and( + isNotNull(sharedEnvVars.valueEncrypted), + isNotNull(sharedEnvVars.valueIv), + isNotNull(sharedEnvVars.valueTag), + ), + ); + return processRows( + rows, + (r) => ({ enc: r.valueEncrypted, iv: r.valueIv, tag: r.valueTag }), + (r, next) => + db + .update(sharedEnvVars) + .set({ valueEncrypted: next.encrypted, valueIv: next.iv, valueTag: next.tag }) + .where(eq(sharedEnvVars.id, r.id)) + .execute(), + activeKey, + ); + }, + }, + { + name: "ssh_keys", + run: async (activeKey) => { + const db = await getDb(); + const rows = await db + .select() + .from(sshKeys) + .where( + and( + isNotNull(sshKeys.privateKeyEncrypted), + isNotNull(sshKeys.privateKeyIv), + isNotNull(sshKeys.privateKeyTag), + ), + ); + return processRows( + rows, + (r) => ({ enc: r.privateKeyEncrypted, iv: r.privateKeyIv, tag: r.privateKeyTag }), + (r, next) => + db + .update(sshKeys) + .set({ + privateKeyEncrypted: next.encrypted, + privateKeyIv: next.iv, + privateKeyTag: next.tag, + }) + .where(eq(sshKeys.id, r.id)) + .execute(), + activeKey, + ); + }, + }, + { + name: "servers", + run: async (activeKey) => { + const db = await getDb(); + const rows = await db + .select() + .from(servers) + .where(and(isNotNull(servers.sshKey), isNotNull(servers.sshKeyIv), isNotNull(servers.sshKeyTag))); + return processRows( + rows, + (r) => ({ enc: r.sshKey, iv: r.sshKeyIv, tag: r.sshKeyTag }), + (r, next) => + db + .update(servers) + .set({ sshKey: next.encrypted, sshKeyIv: next.iv, sshKeyTag: next.tag }) + .where(eq(servers.id, r.id)) + .execute(), + activeKey, + ); + }, + }, + { + name: "smtp_settings", + run: async (activeKey) => { + const db = await getDb(); + const rows = await db + .select() + .from(smtpSettings) + .where( + and(isNotNull(smtpSettings.passEncrypted), isNotNull(smtpSettings.passIv), isNotNull(smtpSettings.passTag)), + ); + return processRows( + rows, + (r) => ({ enc: r.passEncrypted, iv: r.passIv, tag: r.passTag }), + (r, next) => + db + .update(smtpSettings) + .set({ passEncrypted: next.encrypted, passIv: next.iv, passTag: next.tag }) + .where(eq(smtpSettings.id, r.id)) + .execute(), + activeKey, + ); + }, + }, + { + name: "backup_storage_settings", + run: async (activeKey) => { + const db = await getDb(); + const rows = await db + .select() + .from(backupStorageSettings) + .where( + and( + isNotNull(backupStorageSettings.s3SecretAccessKeyEncrypted), + isNotNull(backupStorageSettings.s3SecretAccessKeyIv), + isNotNull(backupStorageSettings.s3SecretAccessKeyTag), + ), + ); + return processRows( + rows, + (r) => ({ + enc: r.s3SecretAccessKeyEncrypted, + iv: r.s3SecretAccessKeyIv, + tag: r.s3SecretAccessKeyTag, + }), + (r, next) => + db + .update(backupStorageSettings) + .set({ + s3SecretAccessKeyEncrypted: next.encrypted, + s3SecretAccessKeyIv: next.iv, + s3SecretAccessKeyTag: next.tag, + }) + .where(eq(backupStorageSettings.id, r.id)) + .execute(), + activeKey, + ); + }, + }, + { + name: "llm_provider_keys", + run: async (activeKey) => { + const db = await getDb(); + const rows = await db + .select() + .from(llmProviderKeys) + .where( + and( + isNotNull(llmProviderKeys.keyEncrypted), + isNotNull(llmProviderKeys.keyIv), + isNotNull(llmProviderKeys.keyTag), + ), + ); + return processRows( + rows, + (r) => ({ enc: r.keyEncrypted, iv: r.keyIv, tag: r.keyTag }), + (r, next) => + db + .update(llmProviderKeys) + .set({ keyEncrypted: next.encrypted, keyIv: next.iv, keyTag: next.tag }) + .where(eq(llmProviderKeys.provider, r.provider)) + .execute(), + activeKey, + ); + }, + }, + { + name: "github_sessions", + run: async (activeKey) => { + const db = await getDb(); + const rows = await db + .select() + .from(githubSessions) + .where( + and( + isNotNull(githubSessions.accessTokenEncrypted), + isNotNull(githubSessions.accessTokenIv), + isNotNull(githubSessions.accessTokenTag), + ), + ); + return processRows( + rows, + (r) => ({ enc: r.accessTokenEncrypted, iv: r.accessTokenIv, tag: r.accessTokenTag }), + (r, next) => + db + .update(githubSessions) + .set({ + accessTokenEncrypted: next.encrypted, + accessTokenIv: next.iv, + accessTokenTag: next.tag, + }) + .where(eq(githubSessions.id, r.id)) + .execute(), + activeKey, + ); + }, + }, + { + name: "projects", + run: async (activeKey) => { + const db = await getDb(); + const rows = await db + .select() + .from(projects) + .where( + and( + isNotNull(projects.githubTokenEncrypted), + isNotNull(projects.githubTokenIv), + isNotNull(projects.githubTokenTag), + ), + ); + return processRows( + rows, + (r) => ({ enc: r.githubTokenEncrypted, iv: r.githubTokenIv, tag: r.githubTokenTag }), + (r, next) => + db + .update(projects) + .set({ + githubTokenEncrypted: next.encrypted, + githubTokenIv: next.iv, + githubTokenTag: next.tag, + }) + .where(eq(projects.id, r.id)) + .execute(), + activeKey, + ); + }, + }, +]; + +export const migrateEncryptionKeys = async (activeKey: string = config.envEncryptionKey): Promise => { + if (activeKey === DEFAULT_ENV_ENCRYPTION_KEY) return; + let migratedTotal = 0; + let unknownTotal = 0; + for (const store of STORES) { + try { + const res = await store.run(activeKey); + migratedTotal += res.migrated; + unknownTotal += res.unknown; + if (res.migrated > 0) console.log(`[EncryptionMigration] ${store.name}: re-encrypted ${res.migrated} row(s)`); + if (res.unknown > 0) + console.warn(`[EncryptionMigration] ${store.name}: ${res.unknown} row(s) use an unknown key, left untouched`); + } catch (err) { + console.error(`[EncryptionMigration] ${store.name}: failed —`, err); + } + } + if (migratedTotal > 0) console.log(`[EncryptionMigration] done — ${migratedTotal} row(s) now use the configured key`); + else console.log("[EncryptionMigration] already using the configured key"); + if (unknownTotal > 0) + console.warn(`[EncryptionMigration] ${unknownTotal} row(s) total use an unknown key and were left untouched`); +}; From 7416cd025d7096715be748b3a7fdcc8c6483ddf8 Mon Sep 17 00:00:00 2001 From: Lftobs Date: Fri, 9 Oct 2026 13:41:48 +0100 Subject: [PATCH 11/17] docs(changelog): include encryption migration in v0.4.0 --- apps/docs/src/content/changelogs/v0.4.0.md | 1 + 1 file changed, 1 insertion(+) diff --git a/apps/docs/src/content/changelogs/v0.4.0.md b/apps/docs/src/content/changelogs/v0.4.0.md index 9fa1e08..f6ffd66 100644 --- a/apps/docs/src/content/changelogs/v0.4.0.md +++ b/apps/docs/src/content/changelogs/v0.4.0.md @@ -15,6 +15,7 @@ date: "2026-10-03" - Add inline MongoDB document editing ([4ead155](https://github.com/Lftobs/dequel/commit/4ead15502a7aaa24cd09b2477409dea1c277a494)) - Route TLS database traffic on port 443 ([8111b92](https://github.com/Lftobs/dequel/commit/8111b926be66562f1c4b89757eeaea090df19a8d)) - Forward remote database traffic over SSH ([42fef01](https://github.com/Lftobs/dequel/commit/42fef016ab9140d137835fb5a829da5ca55ea177)) +- Add database encryption migration for existing plaintext secrets ([41b07b7](https://github.com/Lftobs/dequel/commit/41b07b76dec89f474c5722ade3f699394c6b0c04)) ### Improvements From 81ea4b09011f5d6a57a9fe888f446a8c1a65a0df Mon Sep 17 00:00:00 2001 From: Lftobs Date: Sun, 11 Oct 2026 16:21:09 +0100 Subject: [PATCH 12/17] fix(api): remove redundant zombie reaper initialization --- apps/api/src/index.ts | 2 -- apps/api/src/utils/process-exec.ts | 27 --------------------------- 2 files changed, 29 deletions(-) diff --git a/apps/api/src/index.ts b/apps/api/src/index.ts index 199fd9c..8d188c8 100644 --- a/apps/api/src/index.ts +++ b/apps/api/src/index.ts @@ -22,12 +22,10 @@ import { serverManager } from "./servers/manager"; import { cleanupExpiredTokens, initAuth } from "./utils/auth"; import { config } from "./utils/config"; import { startDomainPolling } from "./utils/domain-verifier"; -import { initZombieReaper } from "./utils/process-exec"; import { loadOrCreateJwtSecret } from "./utils/secrets"; import { captureTelemetry } from "./utils/telemetry"; const bootstrap = async () => { - initZombieReaper(); await mkdir(config.workspaceRoot, { recursive: true }); await mkdir(config.caddyRoutesDir, { recursive: true }); diff --git a/apps/api/src/utils/process-exec.ts b/apps/api/src/utils/process-exec.ts index 970ff71..e9a634c 100644 --- a/apps/api/src/utils/process-exec.ts +++ b/apps/api/src/utils/process-exec.ts @@ -63,33 +63,6 @@ export function terminateWithEscalation(child: ChildProcess, escalationGraceMs = }; } -let reaperInitialized = false; - -export function initZombieReaper(): void { - if (reaperInitialized) return; - reaperInitialized = true; - - if (process.platform !== "linux") return; - - try { - const { dlopen, FFIType } = require("bun:ffi"); - const libc = dlopen("libc.so.6", { - waitpid: { - args: [FFIType.i32, FFIType.ptr, FFIType.i32], - returns: FFIType.i32, - }, - }); - - const reap = () => { - try { - while (libc.symbols.waitpid(-1, null, 1) > 0) {} - } catch {} - }; - - process.on("SIGCHLD", reap); - } catch {} -} - export function safeSpawn(cmd: string, args: string[], options: SafeSpawnOptions = {}): Promise { return new Promise((resolve, reject) => { const isPosix = process.platform !== "win32"; From 0dce87eb16ac7402a0b29aaa606eb648c67cf3f0 Mon Sep 17 00:00:00 2001 From: Lftobs Date: Sun, 11 Oct 2026 16:53:07 +0100 Subject: [PATCH 13/17] fix(docs): make landing page and docs nav mobile responsive --- AGENTS.md | 1 + apps/docs/components/CTA.astro | 22 +- apps/docs/components/Comparison.astro | 42 +-- apps/docs/components/Features.astro | 38 +-- apps/docs/components/Hero.astro | 24 +- apps/docs/components/HowItWorks.astro | 36 ++- apps/docs/components/Stats.astro | 12 +- apps/docs/components/TerminalDemo.astro | 12 +- .../components/architecture/ArchCanvas.astro | 4 +- .../components/architecture/archController.ts | 9 + apps/docs/pages/index.astro | 288 +++++++----------- apps/docs/styles/landing-components.css | 42 ++- apps/docs/styles/landing.css | 58 ++++ apps/docs/theme.css | 167 +++++----- 14 files changed, 405 insertions(+), 350 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index 1cc881d..9727100 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -137,6 +137,7 @@ bun run sync-versions # Syncs VERSION → sub-package.json files - No emojis in code or UI unless explicitly requested - Functional components with hooks (React) - Tailwind CSS for styling (both web and docs) +- All new UI created (both web and docs) must be mobile responsive across all viewport sizes (mobile, tablet, and desktop) with no horizontal overflow or broken navigation ## Adding a Doc Page diff --git a/apps/docs/components/CTA.astro b/apps/docs/components/CTA.astro index 1d1fcc6..3323bda 100644 --- a/apps/docs/components/CTA.astro +++ b/apps/docs/components/CTA.astro @@ -1,11 +1,11 @@ --- --- -
-
+
+
-
- +
+ @@ -18,25 +18,25 @@
-

+

Ready to own your deployment platform?

-

+

Spin up Dequel on your server in seconds with zero third-party dependencies.

-
+
curl -fsSL https://dequel.intrep.xyz/install.sh | bash - 1-Line Install +
-
- +
+ Install Dequel Now → - + Read Documentation
diff --git a/apps/docs/components/Comparison.astro b/apps/docs/components/Comparison.astro index e90d149..ef545ed 100644 --- a/apps/docs/components/Comparison.astro +++ b/apps/docs/components/Comparison.astro @@ -1,77 +1,77 @@ --- --- -
-
+
+
PaaS Paradigm Shift
-

+

Stop fighting infrastructure.
Start shipping instantly.

-

+

Dequel replaces fragmented DevOps tools with a unified self-hosted platform.

-
+
-
+
-
+
TRADITIONAL DEVOPS
-

+

More tools. More upkeep.

-
    -
  • +
      +
    • — Maintain scripts, manifests, and ingress rules
      • -
    • +
    • — Renew certificates and route domains by hand
      • -
    • +
    • — Stitch together logs, metrics, and alerts
-
+
High overhead · Fragmented stack
-
+
-
+
WITH DEQUEL
-

+

One command. One platform.

-
    -
  • +
      +
    • ✓ Deploy from Git, ZIP, or Docker Compose
      • -
    • +
    • ✓ Automate builds, SSL, routing, and scaling
      • -
    • +
    • ✓ Keep complete control on your own VPS
-
+
Zero lock-in · 100% owned by you
diff --git a/apps/docs/components/Features.astro b/apps/docs/components/Features.astro index fff052c..b298327 100644 --- a/apps/docs/components/Features.astro +++ b/apps/docs/components/Features.astro @@ -88,20 +88,20 @@ scaling_policy: ]; --- -
-
+
+
Platform Capabilities
-

+

Everything you need to ship.
Powered by your infrastructure.

-

+

Explore Dequel's core PaaS engines built into a single lightweight control plane.

-
+
{tabs.map((tab, idx) => ( ))} @@ -123,24 +123,24 @@ scaling_policy: id={`tab-panel-${tab.id}`} class={`tab-panel ${idx === 0 ? 'block' : 'hidden'} transition-all duration-300`} > -
+
-
+
PaaS Module: {tab.id}
-

+

{tab.title}

-

+

{tab.description}

-
    +
      {tab.highlights.map((item) => (
    • - ✓ + ✓ {item}
      • ))} @@ -155,13 +155,13 @@ scaling_policy:
-
-
-
- - {tab.label} Preview +
+
+
+ + {tab.label} Preview
-
+
Dequel v1.0
- 
{tab.codePreview}
+ 
{tab.codePreview}
Status: Active Zero Config Required diff --git a/apps/docs/components/Hero.astro b/apps/docs/components/Hero.astro index 84ad598..2d66199 100644 --- a/apps/docs/components/Hero.astro +++ b/apps/docs/components/Hero.astro @@ -12,25 +12,25 @@ const { } = Astro.props; --- -
+
-

+

{titleNormal}. {titleItalic}

-

+

{subText}

-
- +
+ Get Started in 1 Command → @@ -38,10 +38,10 @@ const { href="https://github.com/Lftobs/dequel" target="_blank" rel="noopener noreferrer" - class="cf-pill-btn-glass flex items-center gap-2" + class="cf-pill-btn-glass flex items-center justify-center gap-2 w-full sm:w-auto" >
- $ - curl -fsSL https://dequel.intrep.xyz/install.sh | bash + $ + curl -fsSL https://dequel.intrep.xyz/install.sh | bash - +
diff --git a/apps/docs/components/HowItWorks.astro b/apps/docs/components/HowItWorks.astro index 3df7847..bd47872 100644 --- a/apps/docs/components/HowItWorks.astro +++ b/apps/docs/components/HowItWorks.astro @@ -2,31 +2,31 @@ import ArchCanvas from "./architecture/ArchCanvas.astro"; --- -
+
-
+
Engine Architecture
-

+

Your own PaaS, running on your own servers.

-

+

Dequel orchestrates raw Docker containers, Buildkit compilation, Redis job queues, PostgreSQL persistence, and dynamic Caddy routing into one unified platform.

-
+
-
+
- + DEQUEL ENGINE TOPOLOGY - • - dequel_net bridge + +
+ ← Swipe diagram →
-
+
-
+
-
+
STAGE 01: INGESTION • Zero DevOps overhead
-

+

01. Multi-Channel Trigger Ingestion

@@ -108,8 +109,9 @@ import ArchCanvas from "./architecture/ArchCanvas.astro"; .dq-step-pill { display: inline-flex; align-items: center; - gap: 0.5rem; - padding: 0.45rem 0.95rem; + justify-content: center; + gap: 0.45rem; + padding: 0.5rem 0.85rem; border-radius: 9999px; background: #111115; border: 1px solid #222228; @@ -117,6 +119,7 @@ import ArchCanvas from "./architecture/ArchCanvas.astro"; cursor: pointer; transition: all 180ms ease; user-select: none; + white-space: nowrap; } .dq-step-pill:hover { border-color: #3f3f46; @@ -129,6 +132,7 @@ import ArchCanvas from "./architecture/ArchCanvas.astro"; border-radius: 50%; background-color: #52525b; transition: all 180ms ease; + flex-shrink: 0; } .dq-step-pill .step-num { font-family: 'JetBrains Mono', monospace; @@ -137,7 +141,7 @@ import ArchCanvas from "./architecture/ArchCanvas.astro"; } .dq-step-pill .step-label { font-family: 'Space Grotesk', sans-serif; - font-size: 12.5px; + font-size: 12px; font-weight: 600; } .dq-step-pill.active { diff --git a/apps/docs/components/Stats.astro b/apps/docs/components/Stats.astro index 448df86..aafcc11 100644 --- a/apps/docs/components/Stats.astro +++ b/apps/docs/components/Stats.astro @@ -7,19 +7,19 @@ const stats = [ ]; --- -

+
-
+
{stats.map((stat) => ( -
-
+
+
{stat.value}
-
+
{stat.label}
-
+
{stat.sub}
diff --git a/apps/docs/components/TerminalDemo.astro b/apps/docs/components/TerminalDemo.astro index 14085d3..e8e9bf3 100644 --- a/apps/docs/components/TerminalDemo.astro +++ b/apps/docs/components/TerminalDemo.astro @@ -1,11 +1,11 @@ --- --- -
-
-
+
+
+
Developer Experience
-

+

From git push to production
in under 4 seconds.

@@ -25,7 +25,7 @@
-
+
@@ -35,7 +35,7 @@ dequel cli v1.0.0 bash
-
+
$ diff --git a/apps/docs/components/architecture/ArchCanvas.astro b/apps/docs/components/architecture/ArchCanvas.astro index db6c58c..3debce3 100644 --- a/apps/docs/components/architecture/ArchCanvas.astro +++ b/apps/docs/components/architecture/ArchCanvas.astro @@ -1,11 +1,11 @@ --- --- -
+
diff --git a/apps/docs/components/architecture/archController.ts b/apps/docs/components/architecture/archController.ts index 015e0ac..9f602af 100644 --- a/apps/docs/components/architecture/archController.ts +++ b/apps/docs/components/architecture/archController.ts @@ -75,6 +75,15 @@ export function initArchController() { chipV4.setAttribute("stroke", "#ea580c"); } } + + // 6. Auto-pan horizontal scroll container on mobile viewports + const scrollEl = document.getElementById("dq-arch-scroll"); + if (scrollEl && scrollEl.scrollWidth > scrollEl.clientWidth) { + const ratios = [0, 0.25, 0.48, 0.72, 1.0, 0.48]; + const ratio = ratios[stepIndex - 1] ?? 0; + const maxScroll = scrollEl.scrollWidth - scrollEl.clientWidth; + scrollEl.scrollTo({ left: maxScroll * ratio, behavior: "smooth" }); + } } function startTimer() { diff --git a/apps/docs/pages/index.astro b/apps/docs/pages/index.astro index 7f356c6..497a883 100644 --- a/apps/docs/pages/index.astro +++ b/apps/docs/pages/index.astro @@ -43,99 +43,87 @@ import CTA from "../components/CTA.astro"; Dequel — Self-Hosted PaaS Engine & Container Platform - + + +
@@ -146,82 +134,27 @@ import CTA from "../components/CTA.astro";
-