diff --git a/AGENTS.md b/AGENTS.md index 1cc881d..9727100 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -137,6 +137,7 @@ bun run sync-versions # Syncs VERSION → sub-package.json files - No emojis in code or UI unless explicitly requested - Functional components with hooks (React) - Tailwind CSS for styling (both web and docs) +- All new UI created (both web and docs) must be mobile responsive across all viewport sizes (mobile, tablet, and desktop) with no horizontal overflow or broken navigation ## Adding a Doc Page diff --git a/apps/api/src/api/auth-middleware.ts b/apps/api/src/api/auth-middleware.ts new file mode 100644 index 0000000..1b5de77 --- /dev/null +++ b/apps/api/src/api/auth-middleware.ts @@ -0,0 +1,51 @@ +import type { Elysia } from "elysia"; +import { fail } from "./response"; + +const BYPASS_PATHS = new Set([ + "/api/auth/login", + "/api/auth/logout", + "/api/auth/refresh", + "/api/auth/me", + "/api/health", + "/api/github/callback", + "/api/github/webhook", + "/api/agents/register", + "/api/agents/p2p-sync", +]); + +export const authMiddleware = (app: Elysia) => + app.onBeforeHandle(async ({ request, set, path }) => { + if (BYPASS_PATHS.has(path)) return; + + const cookie = request.headers.get("cookie") || ""; + const match = cookie.match(/(?:^|;\s*)dequel_session=([^;]+)/); + if (match) { + const { verifyAccessToken } = await import("../utils/auth"); + const payload = await verifyAccessToken(match[1]); + if (payload) return; + } + + const authHeader = request.headers.get("authorization"); + if (authHeader?.startsWith("Bearer ")) { + const token = authHeader.slice(7); + if (token) { + const { validateApiKey } = await import("../db/repo"); + const { canPerformMethod, parsePermissions, requiredPermissionForMethod } = await import( + "../utils/permissions" + ); + const key = await validateApiKey(token); + if (!key) { + set.status = 401; + return fail("Invalid API key"); + } + if (!canPerformMethod(parsePermissions(key.permissions), request.method)) { + set.status = 403; + return fail(`API key is missing the ${requiredPermissionForMethod(request.method)} scope`); + } + return; + } + } + + set.status = 401; + return fail("Authentication required"); + }); diff --git a/apps/api/src/api/databases/index.ts b/apps/api/src/api/databases/index.ts index fdd2526..b8c5220 100644 --- a/apps/api/src/api/databases/index.ts +++ b/apps/api/src/api/databases/index.ts @@ -6,14 +6,18 @@ import { deleteDatabase, getDatabaseById, getProjectById, + getServerById, listAllDatabases, listDatabases, updateDatabaseSettings, updateDatabaseStatus, } from "../../db/repo"; +import { loadCaddyCert } from "../../tunnels/cert-store"; import type { Database } from "../../types"; import { config } from "../../utils/config"; +import { removeDbGatewayRoute, syncDbGatewayRoute } from "../../utils/db-caddy-route"; import { resolveServerIp } from "../../utils/dns"; +import { probeTcp } from "../../utils/tcp-probe"; import { created, fail, ok } from "../response"; import { executeDatabaseQuery, getDatabaseTables } from "./query"; @@ -60,6 +64,14 @@ const isNonLoopbackIp = (value: string): boolean => { return !value.startsWith("127."); }; +const resolveDbExternalHost = async (serverId: string | null): Promise => { + if (serverId) { + const server = await getServerById(serverId); + if (server && isNonLoopbackIp(server.host)) return server.host; + } + return resolveServerIp(); +}; + const createManagedDatabase = async (body: any, projectId: string | null, set: any) => { if (projectId && !(await getProjectById(projectId))) { set.status = 404; @@ -76,6 +88,7 @@ const createManagedDatabase = async (body: any, projectId: string | null, set: a return fail("A database with this name already exists"); } const dbRecord = await createDbRecord({ ...validation.input, projectId }); + void syncDbGatewayRoute(dbRecord); const { provisionDatabase } = await import("../../databases/manager"); provisionDatabase(dbRecord).catch((err: Error) => console.error("DB provision failed", err)); return created(sanitizeDatabase(dbRecord)); @@ -118,27 +131,50 @@ export const databasesRoutes = new Elysia() ? `${dbRecord.internalHost}.${baseDomain}` : null; if (gatewayHost) { + const reachable = await probeTcp(gatewayHost, 443); + const hasRealCert = config.caddyDataDir + ? (await loadCaddyCert(config.caddyDataDir, gatewayHost)) !== null + : false; return ok({ username: dbRecord.username, password: dbRecord.password, internalConnectionString: dbRecord.connectionString, - externalConnectionString: buildGatewayConnectionString(dbRecord, gatewayHost), + externalConnectionString: buildGatewayConnectionString(dbRecord, gatewayHost, hasRealCert), externalHost: gatewayHost, externalPort: 443, + externalReachable: reachable, + warning: reachable + ? null + : `Gateway endpoint ${gatewayHost}:443 is not reachable from the network — check the gateway service and DNS.`, }); } - const externalHost = dbRecord.publicAccess && dbRecord.externalPort ? await resolveServerIp() : null; + const externalHost = + dbRecord.publicAccess && dbRecord.externalPort ? await resolveDbExternalHost(dbRecord.serverId ?? null) : null; const usableHost = externalHost && isNonLoopbackIp(externalHost) ? externalHost : null; - const externalConnectionString = usableHost - ? buildConnectionString(dbRecord, usableHost, dbRecord.externalPort!) - : null; + if (usableHost && dbRecord.externalPort) { + const reachable = await probeTcp(usableHost, dbRecord.externalPort); + return ok({ + username: dbRecord.username, + password: dbRecord.password, + internalConnectionString: dbRecord.connectionString, + externalConnectionString: buildConnectionString(dbRecord, usableHost, dbRecord.externalPort), + externalHost: usableHost, + externalPort: dbRecord.externalPort, + externalReachable: reachable, + warning: reachable + ? null + : `Public endpoint ${usableHost}:${dbRecord.externalPort} is not reachable — open the port in your cloud security group or firewall to allow external clients.`, + }); + } return ok({ username: dbRecord.username, password: dbRecord.password, internalConnectionString: dbRecord.connectionString, - externalConnectionString, - externalHost: usableHost, - externalPort: usableHost ? dbRecord.externalPort : null, + externalConnectionString: null, + externalHost: null, + externalPort: null, + externalReachable: null, + warning: null, }); }) .post("/databases/:id/start", async ({ params: { id }, set }) => { @@ -235,7 +271,12 @@ export const databasesRoutes = new Elysia() backupSchedule: body?.backupSchedule, backupRetention: body?.backupRetention, }); - return ok(sanitizeDatabase(updated!)); + if (!updated) { + set.status = 404; + return fail("Database not found"); + } + void syncDbGatewayRoute(updated); + return ok(sanitizeDatabase(updated)); }) .delete("/databases/:id", async ({ params: { id }, set }) => { const dbRecord = await findDatabase(id, set); @@ -259,6 +300,7 @@ export const databasesRoutes = new Elysia() return fail("Database resources could not be deleted; cleanup will be retried automatically"); } await deleteDatabase(id); + void removeDbGatewayRoute(current); return ok(null, "Database deleted"); }); }); @@ -272,12 +314,13 @@ const buildConnectionString = (dbRecord: DatabaseRecord, host: string, port: num return `${protocol}://${dbRecord.username}:${dbRecord.password}@${host}:${port}/${dbRecord.databaseName}`; }; -const buildGatewayConnectionString = (dbRecord: DatabaseRecord, host: string) => { +const buildGatewayConnectionString = (dbRecord: DatabaseRecord, host: string, hasRealCert: boolean) => { const auth = `${dbRecord.username}:${dbRecord.password}`; const path = `/${dbRecord.databaseName}`; if (dbRecord.type === "redis") return `rediss://:${dbRecord.password}@${host}:443`; if (dbRecord.type === "mongodb") { - return `mongodb://${auth}@${host}:443${path}?authSource=admin&tls=true&tlsAllowInvalidCertificates=true`; + const tls = hasRealCert ? "tls=true" : "tls=true&tlsAllowInvalidCertificates=true"; + return `mongodb://${auth}@${host}:443${path}?authSource=admin&${tls}`; } return `postgresql://${auth}@${host}:443${path}?sslmode=require`; }; diff --git a/apps/api/src/api/deployments/index.ts b/apps/api/src/api/deployments/index.ts index 1342418..69c9d17 100644 --- a/apps/api/src/api/deployments/index.ts +++ b/apps/api/src/api/deployments/index.ts @@ -16,6 +16,7 @@ import { executorFor } from "../../executors/dispatch"; import { orchestrator } from "../../orchestrator"; import { summarizeDeploymentError } from "../../orchestrator/deployment-errors"; import { logBus } from "../../orchestrator/log-bus"; +import { readRuntimeLogs, runtimeLogFollowerArgs } from "../../orchestrator/runtime-logs"; import { config } from "../../utils/config"; import { terminateWithEscalation } from "../../utils/process-exec"; import { isPrivateGitUrl } from "../../utils/validate"; @@ -142,9 +143,17 @@ export const deploymentsRoutes = new Elysia() set.status = 400; return fail("archive file is required for upload source"); } + const safeName = basename(file.name || "project.zip").replace(/[^a-zA-Z0-9._-]/g, "_"); + if (![".zip", ".tar", ".tar.gz", ".tgz"].some((ext) => safeName.toLowerCase().endsWith(ext))) { + set.status = 400; + return fail("Unsupported archive format. Use .zip, .tar, .tar.gz, or .tgz"); + } + if (file.size === 0) { + set.status = 400; + return fail("Archive file is empty"); + } const uploadsDir = join(config.workspaceRoot, "uploads"); await mkdir(uploadsDir, { recursive: true }); - const safeName = basename(file.name || "project.zip").replace(/[^a-zA-Z0-9._-]/g, "_"); const uploadPath = join(uploadsDir, `${Date.now()}-${safeName}`); const bytes = new Uint8Array(await file.arrayBuffer()); await writeFile(uploadPath, bytes); @@ -330,20 +339,8 @@ export const deploymentsRoutes = new Elysia() set.status = 404; return fail("Deployment not found"); } - const { run } = await import("../../orchestrator/runtime"); - const containerName = deployment.containerName || `deploy-${id}`; try { - const output = await run("docker", ["logs", "--tail", "200", containerName]); - const lines = output - .split("\n") - .filter(Boolean) - .map((line, i) => ({ - sequence: i + 1, - message: line, - timestamp: new Date().toISOString(), - stage: "runtime" as const, - })); - return ok(lines); + return ok(await readRuntimeLogs(deployment)); } catch { return ok([]); } @@ -355,7 +352,6 @@ export const deploymentsRoutes = new Elysia() return fail("Deployment not found"); } const encoder = new TextEncoder(); - const containerName = deployment.containerName || `deploy-${id}`; let closed = false; let childProcess: import("node:child_process").ChildProcess | null = null; const stop = () => { @@ -375,16 +371,27 @@ export const deploymentsRoutes = new Elysia() controller.enqueue(encoder.encode(`event: ${eventName}\ndata: ${JSON.stringify(payload)}\n\n`)); }; const { spawn } = await import("node:child_process"); - const isPosix = process.platform !== "win32"; - const child = spawn("docker", ["logs", "--tail", "100", "--follow", containerName], { - detached: isPosix, + const child = spawn("docker", await runtimeLogFollowerArgs(deployment), { stdio: ["ignore", "pipe", "pipe"], }); childProcess = child; if (closed) { terminateWithEscalation(child, 2000); + try { + controller.close(); + } catch {} return; } + child.on("error", (err) => { + send("log", { + sequence: 0, + message: `docker logs failed: ${err.message}`, + timestamp: new Date().toISOString(), + stage: "runtime", + }); + send("close", { reason: "spawn failed" }); + stop(); + }); let seq = 0; child.stdout.on("data", (chunk: Buffer) => { if (closed) return; diff --git a/apps/api/src/api/github/index.ts b/apps/api/src/api/github/index.ts index b877def..17470e8 100644 --- a/apps/api/src/api/github/index.ts +++ b/apps/api/src/api/github/index.ts @@ -9,7 +9,6 @@ import { setGithubIntegration, } from "../../db/repo"; import { orchestrator } from "../../orchestrator"; -import { config } from "../../utils/config"; import { fail, ok } from "../response"; const validateToken = async (token: string): Promise => { @@ -249,7 +248,11 @@ export const githubRoutes = new Elysia({ prefix: "/github" }) } const webhookUrl = `${publicUrl(request).origin}/api/github/webhook`; const integration = await getGithubIntegration(); - const secret = integration?.webhookSecret || config.githubWebhookSecret; + const secret = integration?.webhookSecret || ""; + if (!secret) { + set.status = 400; + return fail("Set a webhook secret in Settings before registering a webhook"); + } try { const hooks = await fetchGitHub(`/repos/${params.owner}/${params.repo}/hooks`, token); diff --git a/apps/api/src/api/index.ts b/apps/api/src/api/index.ts index 571912f..42708fc 100644 --- a/apps/api/src/api/index.ts +++ b/apps/api/src/api/index.ts @@ -1,9 +1,10 @@ import { Elysia } from "elysia"; -import { fail } from "./response"; +import { fixdiagRoutes } from "../fixdiag/routes"; import { agentRoutes } from "./agents"; import { alertsRoutes } from "./alerts"; import { apiKeysRoutes } from "./api-keys"; import { authRoutes } from "./auth"; +import { authMiddleware } from "./auth-middleware"; import { backupRoutes } from "./backups"; import { databasesRoutes } from "./databases"; import { deploymentsRoutes } from "./deployments"; @@ -14,59 +15,17 @@ import { healthRoutes } from "./health"; import { projectsRoutes } from "./projects"; import { projectStatusRoutes } from "./projects/status"; import { prometheusRoutes } from "./prometheus"; +import { fail } from "./response"; import { routesRoutes } from "./routes"; import { scalingRoutes } from "./scaling"; import { serverInfoRoutes } from "./server-info"; import { serversRoutes } from "./servers"; import { settingsRoutes } from "./settings"; import { llmSettingsRoutes } from "./settings/llm"; -import { fixdiagRoutes } from "../fixdiag/routes"; import { sharedEnvLinksRoutes, sharedEnvVarsRoutes } from "./shared-env-vars"; import { sshKeysRoutes } from "./ssh-keys"; import { volumesRoutes } from "./volumes"; -const BYPASS_PATHS = new Set([ - "/api/auth/login", - "/api/auth/logout", - "/api/auth/refresh", - "/api/auth/me", - "/api/health", - "/api/github/callback", - "/api/github/webhook", - "/api/agents/register", - "/api/agents/p2p-sync", -]); - -const authMiddleware = (app: Elysia) => - app.onBeforeHandle(async ({ request, set, path }) => { - if (BYPASS_PATHS.has(path)) return; - - const cookie = request.headers.get("cookie") || ""; - const match = cookie.match(/(?:^|;\s*)dequel_session=([^;]+)/); - if (match) { - const { verifyAccessToken } = await import("../utils/auth"); - const payload = await verifyAccessToken(match[1]); - if (payload) return; - set.status = 401; - return fail("Invalid session"); - } - - const authHeader = request.headers.get("authorization"); - if (authHeader?.startsWith("Bearer ")) { - const token = authHeader.slice(7); - if (token) { - const { validateApiKey } = await import("../db/repo"); - const key = await validateApiKey(token); - if (key) return; - set.status = 401; - return fail("Invalid API key"); - } - } - - set.status = 401; - return fail("Authentication required"); - }); - const INTERNAL_ERROR = /Failed query:|params:|getaddrinfo|ECONNREFUSED|ETIMEDOUT|EAI_AGAIN|EHOSTUNREACH|timeout exceeded|node:internal|Cannot read propert|is not a function|is not a constructor|Unexpected token/i; diff --git a/apps/api/src/api/projects/index.ts b/apps/api/src/api/projects/index.ts index fe698b2..f1f6c9b 100644 --- a/apps/api/src/api/projects/index.ts +++ b/apps/api/src/api/projects/index.ts @@ -9,14 +9,16 @@ import { listProjects, updateProject, } from "../../db/repo"; +import { destroyComposeStack } from "../../orchestrator/compose"; import { failoverProject } from "../../orchestrator/failover"; -import { reloadCaddy, tryRun } from "../../orchestrator/runtime"; +import { cleanupProjectContainers } from "../../orchestrator/project-cleanup"; +import { reloadCaddy } from "../../orchestrator/runtime"; import { config } from "../../utils/config"; -import { dockerBin } from "../../utils/docker-bin"; +import { dockerRunTry, ensureContainerRemoved, ensureVolumeRemoved } from "../../utils/docker-run"; import { removeFromCaddyRoute } from "../../utils/domain-verifier"; import { buildProjectRequestHostRegex, caddyRequestLogSelector } from "../../utils/loki"; -import { isPort, isPrivateGitUrl, SERVICE_NAME_RE, validateComposeServices } from "../../utils/validate"; import { captureTelemetry } from "../../utils/telemetry"; +import { isPort, isPrivateGitUrl, SERVICE_NAME_RE, validateComposeServices } from "../../utils/validate"; import { created, fail, ok } from "../response"; const validateComposeFields = (body: any): string | null => { @@ -168,21 +170,15 @@ export const projectsRoutes = new Elysia() return fail("Project not found"); } - for (const name of info.deploymentContainerNames) { - await tryRun(dockerBin, ["stop", "-t", "5", name]); - await tryRun(dockerBin, ["rm", "-f", name]); - } - for (const name of info.databaseContainerNames) { - await tryRun(dockerBin, ["stop", "-t", "5", name]); - await tryRun(dockerBin, ["rm", "-f", name]); - } - - for (const name of [...info.databaseVolumeNames, ...info.volumeDockerNames]) { - await tryRun(dockerBin, ["volume", "rm", "-f", name]); - } - - for (const tag of info.deploymentImageTags) { - if (tag) await tryRun(dockerBin, ["rmi", "-f", tag]); + const cleanupFailures = await cleanupProjectContainers(info, { + getServerById, + ensureContainerRemoved, + ensureVolumeRemoved, + destroyComposeStack, + dockerRunTry, + }); + if (cleanupFailures.length > 0) { + console.error(`Project ${id} cleanup incomplete:`, cleanupFailures); } for (const { domain, projectName } of info.domains) { diff --git a/apps/api/src/api/scaling/index.ts b/apps/api/src/api/scaling/index.ts index ef1c7b6..b5370c0 100644 --- a/apps/api/src/api/scaling/index.ts +++ b/apps/api/src/api/scaling/index.ts @@ -1,7 +1,49 @@ import { Elysia } from "elysia"; -import { deleteScalingPolicy, getProjectById, getScalingPolicy, upsertScalingPolicy } from "../../db/repo"; +import { + deleteScalingPolicy, + getProjectById, + getScalingPolicy, + ScalingRangeError, + upsertScalingPolicy, +} from "../../db/repo"; import { fail, ok } from "../response"; +const MAX_REPLICAS = 100; + +const isPercent = (value: unknown): value is number => + typeof value === "number" && Number.isInteger(value) && value >= 0 && value <= 100; + +const isMinReplicas = (value: unknown): value is number => + typeof value === "number" && Number.isInteger(value) && value >= 0 && value <= MAX_REPLICAS; + +const isMaxReplicas = (value: unknown): value is number => + typeof value === "number" && Number.isInteger(value) && value >= 1 && value <= MAX_REPLICAS; + +const isCooldown = (value: unknown): value is number => + typeof value === "number" && Number.isInteger(value) && value >= 1 && value <= 86_400; + +const validatePolicyBody = (body: any): string | null => { + if (body.cpuThresholdPercent !== undefined && !isPercent(body.cpuThresholdPercent)) { + return "cpuThresholdPercent must be an integer between 0 and 100"; + } + if (body.memoryThresholdPercent !== undefined && !isPercent(body.memoryThresholdPercent)) { + return "memoryThresholdPercent must be an integer between 0 and 100"; + } + if (body.minReplicas !== undefined && !isMinReplicas(body.minReplicas)) { + return `minReplicas must be an integer between 0 and ${MAX_REPLICAS}`; + } + if (body.maxReplicas !== undefined && !isMaxReplicas(body.maxReplicas)) { + return `maxReplicas must be an integer between 1 and ${MAX_REPLICAS}`; + } + if (body.cooldownSeconds !== undefined && !isCooldown(body.cooldownSeconds)) { + return "cooldownSeconds must be an integer between 1 and 86400"; + } + if (body.enabled !== undefined && typeof body.enabled !== "boolean") { + return "enabled must be a boolean"; + } + return null; +}; + export const scalingRoutes = new Elysia() .get("/projects/:id/scaling", async ({ params, set }) => { const policy = await getScalingPolicy(params.id); @@ -16,6 +58,11 @@ export const scalingRoutes = new Elysia() set.status = 400; return fail("body is required"); } + const bodyError = validatePolicyBody(body); + if (bodyError) { + set.status = 400; + return fail(bodyError); + } const project = await getProjectById(params.id); if (!project) { set.status = 404; @@ -25,12 +72,20 @@ export const scalingRoutes = new Elysia() set.status = 400; return fail("Cannot enable autoscaling on a project without CPU resource limits configured."); } - return ok( - await upsertScalingPolicy({ - projectId: params.id, - ...body, - }), - ); + try { + return ok( + await upsertScalingPolicy({ + projectId: params.id, + ...body, + }), + ); + } catch (err) { + if (err instanceof ScalingRangeError) { + set.status = 400; + return fail(err.message); + } + throw err; + } }) .delete("/projects/:id/scaling", async ({ params }) => { await deleteScalingPolicy(params.id); diff --git a/apps/api/src/api/ssh-keys/index.ts b/apps/api/src/api/ssh-keys/index.ts index bd0cec4..48fac03 100644 --- a/apps/api/src/api/ssh-keys/index.ts +++ b/apps/api/src/api/ssh-keys/index.ts @@ -1,23 +1,95 @@ +import { createPrivateKey } from "node:crypto"; import { Elysia } from "elysia"; -import { createSshKey, deleteSshKey, getSshKeyById, listSshKeys } from "../../db/repo/ssh-keys"; +import { + computeSshKeyFingerprint, + createSshKey, + deleteSshKey, + getSshKeyByFingerprint, + getSshKeyById, + listSshKeys, +} from "../../db/repo/ssh-keys"; import { fail, ok } from "../response"; +const PRIVATE_KEY_PATTERN = + /^-----BEGIN ((?:[A-Z0-9]+ )*?)PRIVATE KEY-----\r?\n([\s\S]*?)-----END \1PRIVATE KEY-----\s*$/; +const BASE64_PATTERN = /^(?:[A-Za-z0-9+/]{4})*(?:[A-Za-z0-9+/]{2}==|[A-Za-z0-9+/]{3}=)?$/; + +const validatePrivateKey = (value: string): string | null => { + const trimmed = value.trim(); + const match = PRIVATE_KEY_PATTERN.exec(trimmed); + if (!match) return "privateKey must be a PEM or OpenSSH private key"; + const keyType = (match[1] ?? "").trim(); + if (keyType === "OPENSSH") { + const body = match[2] + .split(/\r?\n/) + .filter((line) => !line.includes(":")) + .join("") + .replace(/\s/g, ""); + if (body.length < 16 || !BASE64_PATTERN.test(body)) { + return "privateKey body is not valid base64"; + } + return null; + } + try { + createPrivateKey(trimmed); + return null; + } catch (err) { + if (/passphrase/i.test((err as Error).message)) { + return "privateKey is passphrase-protected; provide an unencrypted key"; + } + return "privateKey could not be parsed as a valid private key"; + } +}; + +const isUniqueViolation = (err: unknown): boolean => { + const e = err as { code?: string; cause?: { code?: string } }; + return e?.code === "23505" || e?.cause?.code === "23505"; +}; + +const duplicateKey = async (fingerprint: string, set: any) => { + const existing = await getSshKeyByFingerprint(fingerprint); + if (!existing) return null; + set.status = 409; + return fail(`SSH key already exists in the pool as "${existing.name}"`); +}; + export const sshKeysRoutes = new Elysia() .get("/ssh-keys", async () => { return ok(await listSshKeys()); }) .post("/ssh-keys", async ({ body, set }: any) => { - if (!body?.name || !body.privateKey) { + if ( + typeof body?.name !== "string" || + !body.name.trim() || + typeof body.privateKey !== "string" || + !body.privateKey + ) { set.status = 400; return fail("name and privateKey are required"); } - const created = await createSshKey({ - name: body.name, - privateKey: body.privateKey, - tags: body.tags, - }); - set.status = 201; - return ok(created); + const keyError = validatePrivateKey(body.privateKey); + if (keyError) { + set.status = 400; + return fail(keyError); + } + const fingerprint = computeSshKeyFingerprint(body.privateKey); + const existing = await duplicateKey(fingerprint, set); + if (existing) return existing; + try { + const created = await createSshKey({ + name: body.name, + privateKey: body.privateKey, + tags: body.tags, + }); + set.status = 201; + return ok(created); + } catch (err) { + if (isUniqueViolation(err)) { + const raced = await duplicateKey(fingerprint, set); + if (raced) return raced; + } + throw err; + } }) .get("/ssh-keys/:id", async ({ params, set }: any) => { const key = await getSshKeyById(params.id); diff --git a/apps/api/src/db/encryption-migration.ts b/apps/api/src/db/encryption-migration.ts new file mode 100644 index 0000000..b9ae84c --- /dev/null +++ b/apps/api/src/db/encryption-migration.ts @@ -0,0 +1,326 @@ +import { and, eq, isNotNull } from "drizzle-orm"; +import { config } from "../utils/config"; +import { DEFAULT_ENV_ENCRYPTION_KEY, decryptValue, encryptValue } from "../utils/crypto"; +import { getDb } from "./db-provider"; +import { + backupStorageSettings, + environmentVariables, + githubSessions, + llmProviderKeys, + projects, + servers, + sharedEnvVars, + smtpSettings, + sshKeys, +} from "./schema"; + +type Triple = { encrypted: string; iv: string; tag: string }; +type StoredTriple = { enc: string | null; iv: string | null; tag: string | null }; +type StoreResult = { migrated: number; unknown: number }; +type Store = { name: string; run: (activeKey: string) => Promise }; + +const processRows = async ( + rows: R[], + getTriple: (row: R) => StoredTriple, + write: (row: R, next: Triple) => Promise, + activeKey: string, +): Promise => { + let migrated = 0; + let unknown = 0; + for (const row of rows) { + const { enc, iv, tag } = getTriple(row); + if (!enc || !iv || !tag) continue; + try { + decryptValue(enc, iv, tag, activeKey); + continue; + } catch {} + let plain: string; + try { + plain = decryptValue(enc, iv, tag, DEFAULT_ENV_ENCRYPTION_KEY); + } catch { + unknown += 1; + continue; + } + await write(row, encryptValue(plain, activeKey)); + migrated += 1; + } + return { migrated, unknown }; +}; + +const STORES: Store[] = [ + { + name: "environment_variables", + run: async (activeKey) => { + const db = await getDb(); + const rows = await db + .select() + .from(environmentVariables) + .where( + and( + isNotNull(environmentVariables.valueEncrypted), + isNotNull(environmentVariables.valueIv), + isNotNull(environmentVariables.valueTag), + ), + ); + return processRows( + rows, + (r) => ({ enc: r.valueEncrypted, iv: r.valueIv, tag: r.valueTag }), + (r, next) => + db + .update(environmentVariables) + .set({ valueEncrypted: next.encrypted, valueIv: next.iv, valueTag: next.tag }) + .where(eq(environmentVariables.id, r.id)) + .execute(), + activeKey, + ); + }, + }, + { + name: "shared_env_vars", + run: async (activeKey) => { + const db = await getDb(); + const rows = await db + .select() + .from(sharedEnvVars) + .where( + and( + isNotNull(sharedEnvVars.valueEncrypted), + isNotNull(sharedEnvVars.valueIv), + isNotNull(sharedEnvVars.valueTag), + ), + ); + return processRows( + rows, + (r) => ({ enc: r.valueEncrypted, iv: r.valueIv, tag: r.valueTag }), + (r, next) => + db + .update(sharedEnvVars) + .set({ valueEncrypted: next.encrypted, valueIv: next.iv, valueTag: next.tag }) + .where(eq(sharedEnvVars.id, r.id)) + .execute(), + activeKey, + ); + }, + }, + { + name: "ssh_keys", + run: async (activeKey) => { + const db = await getDb(); + const rows = await db + .select() + .from(sshKeys) + .where( + and( + isNotNull(sshKeys.privateKeyEncrypted), + isNotNull(sshKeys.privateKeyIv), + isNotNull(sshKeys.privateKeyTag), + ), + ); + return processRows( + rows, + (r) => ({ enc: r.privateKeyEncrypted, iv: r.privateKeyIv, tag: r.privateKeyTag }), + (r, next) => + db + .update(sshKeys) + .set({ + privateKeyEncrypted: next.encrypted, + privateKeyIv: next.iv, + privateKeyTag: next.tag, + }) + .where(eq(sshKeys.id, r.id)) + .execute(), + activeKey, + ); + }, + }, + { + name: "servers", + run: async (activeKey) => { + const db = await getDb(); + const rows = await db + .select() + .from(servers) + .where(and(isNotNull(servers.sshKey), isNotNull(servers.sshKeyIv), isNotNull(servers.sshKeyTag))); + return processRows( + rows, + (r) => ({ enc: r.sshKey, iv: r.sshKeyIv, tag: r.sshKeyTag }), + (r, next) => + db + .update(servers) + .set({ sshKey: next.encrypted, sshKeyIv: next.iv, sshKeyTag: next.tag }) + .where(eq(servers.id, r.id)) + .execute(), + activeKey, + ); + }, + }, + { + name: "smtp_settings", + run: async (activeKey) => { + const db = await getDb(); + const rows = await db + .select() + .from(smtpSettings) + .where( + and(isNotNull(smtpSettings.passEncrypted), isNotNull(smtpSettings.passIv), isNotNull(smtpSettings.passTag)), + ); + return processRows( + rows, + (r) => ({ enc: r.passEncrypted, iv: r.passIv, tag: r.passTag }), + (r, next) => + db + .update(smtpSettings) + .set({ passEncrypted: next.encrypted, passIv: next.iv, passTag: next.tag }) + .where(eq(smtpSettings.id, r.id)) + .execute(), + activeKey, + ); + }, + }, + { + name: "backup_storage_settings", + run: async (activeKey) => { + const db = await getDb(); + const rows = await db + .select() + .from(backupStorageSettings) + .where( + and( + isNotNull(backupStorageSettings.s3SecretAccessKeyEncrypted), + isNotNull(backupStorageSettings.s3SecretAccessKeyIv), + isNotNull(backupStorageSettings.s3SecretAccessKeyTag), + ), + ); + return processRows( + rows, + (r) => ({ + enc: r.s3SecretAccessKeyEncrypted, + iv: r.s3SecretAccessKeyIv, + tag: r.s3SecretAccessKeyTag, + }), + (r, next) => + db + .update(backupStorageSettings) + .set({ + s3SecretAccessKeyEncrypted: next.encrypted, + s3SecretAccessKeyIv: next.iv, + s3SecretAccessKeyTag: next.tag, + }) + .where(eq(backupStorageSettings.id, r.id)) + .execute(), + activeKey, + ); + }, + }, + { + name: "llm_provider_keys", + run: async (activeKey) => { + const db = await getDb(); + const rows = await db + .select() + .from(llmProviderKeys) + .where( + and( + isNotNull(llmProviderKeys.keyEncrypted), + isNotNull(llmProviderKeys.keyIv), + isNotNull(llmProviderKeys.keyTag), + ), + ); + return processRows( + rows, + (r) => ({ enc: r.keyEncrypted, iv: r.keyIv, tag: r.keyTag }), + (r, next) => + db + .update(llmProviderKeys) + .set({ keyEncrypted: next.encrypted, keyIv: next.iv, keyTag: next.tag }) + .where(eq(llmProviderKeys.provider, r.provider)) + .execute(), + activeKey, + ); + }, + }, + { + name: "github_sessions", + run: async (activeKey) => { + const db = await getDb(); + const rows = await db + .select() + .from(githubSessions) + .where( + and( + isNotNull(githubSessions.accessTokenEncrypted), + isNotNull(githubSessions.accessTokenIv), + isNotNull(githubSessions.accessTokenTag), + ), + ); + return processRows( + rows, + (r) => ({ enc: r.accessTokenEncrypted, iv: r.accessTokenIv, tag: r.accessTokenTag }), + (r, next) => + db + .update(githubSessions) + .set({ + accessTokenEncrypted: next.encrypted, + accessTokenIv: next.iv, + accessTokenTag: next.tag, + }) + .where(eq(githubSessions.id, r.id)) + .execute(), + activeKey, + ); + }, + }, + { + name: "projects", + run: async (activeKey) => { + const db = await getDb(); + const rows = await db + .select() + .from(projects) + .where( + and( + isNotNull(projects.githubTokenEncrypted), + isNotNull(projects.githubTokenIv), + isNotNull(projects.githubTokenTag), + ), + ); + return processRows( + rows, + (r) => ({ enc: r.githubTokenEncrypted, iv: r.githubTokenIv, tag: r.githubTokenTag }), + (r, next) => + db + .update(projects) + .set({ + githubTokenEncrypted: next.encrypted, + githubTokenIv: next.iv, + githubTokenTag: next.tag, + }) + .where(eq(projects.id, r.id)) + .execute(), + activeKey, + ); + }, + }, +]; + +export const migrateEncryptionKeys = async (activeKey: string = config.envEncryptionKey): Promise => { + if (activeKey === DEFAULT_ENV_ENCRYPTION_KEY) return; + let migratedTotal = 0; + let unknownTotal = 0; + for (const store of STORES) { + try { + const res = await store.run(activeKey); + migratedTotal += res.migrated; + unknownTotal += res.unknown; + if (res.migrated > 0) console.log(`[EncryptionMigration] ${store.name}: re-encrypted ${res.migrated} row(s)`); + if (res.unknown > 0) + console.warn(`[EncryptionMigration] ${store.name}: ${res.unknown} row(s) use an unknown key, left untouched`); + } catch (err) { + console.error(`[EncryptionMigration] ${store.name}: failed —`, err); + } + } + if (migratedTotal > 0) console.log(`[EncryptionMigration] done — ${migratedTotal} row(s) now use the configured key`); + else console.log("[EncryptionMigration] already using the configured key"); + if (unknownTotal > 0) + console.warn(`[EncryptionMigration] ${unknownTotal} row(s) total use an unknown key and were left untouched`); +}; diff --git a/apps/api/src/db/migrate.ts b/apps/api/src/db/migrate.ts index 8707c6d..12d63ee 100644 --- a/apps/api/src/db/migrate.ts +++ b/apps/api/src/db/migrate.ts @@ -1,7 +1,5 @@ import { migrate as drizzleMigrate } from "drizzle-orm/node-postgres/migrator"; -import { config } from "../utils/config"; import { getDb } from "./client"; -import { getGithubIntegration, setGithubIntegration } from "./repo/github"; export const migrate = async () => { const db = await getDb(); @@ -13,41 +11,10 @@ export const migrate = async () => { const msg = err?.message ?? String(err); const innerCode = err?.cause?.code; if (err?.code === "42P07" || innerCode === "42P07" || msg.includes("already exists")) { - console.log("[Migrate] Some tables already exist, continuing to seed"); + console.log("[Migrate] Some tables already exist, continuing"); } else { console.error("[Migrate] Migration failed:", msg); throw err; } } - - await seedFromConfig(); -}; - -const seedFromConfig = async () => { - if (config.githubClientId && config.githubClientSecret) { - const existing = await getGithubIntegration(); - if (existing) { - if ( - existing.clientId !== config.githubClientId || - existing.clientSecret !== config.githubClientSecret || - (config.githubWebhookSecret && existing.webhookSecret !== config.githubWebhookSecret) - ) { - await setGithubIntegration({ - clientId: config.githubClientId, - clientSecret: config.githubClientSecret, - appName: config.githubAppName, - webhookSecret: config.githubWebhookSecret || undefined, - }); - console.log("[Config] Synced GitHub integration from config file"); - } - } else { - await setGithubIntegration({ - clientId: config.githubClientId, - clientSecret: config.githubClientSecret, - appName: config.githubAppName, - webhookSecret: config.githubWebhookSecret || undefined, - }); - console.log("[Config] Seeded GitHub integration from config file"); - } - } }; diff --git a/apps/api/src/db/repo/deployments.ts b/apps/api/src/db/repo/deployments.ts index c35c26e..304ba82 100644 --- a/apps/api/src/db/repo/deployments.ts +++ b/apps/api/src/db/repo/deployments.ts @@ -1,5 +1,5 @@ import { randomUUID } from "node:crypto"; -import { desc, eq } from "drizzle-orm"; +import { and, desc, eq } from "drizzle-orm"; import type { CreateDeploymentInput, Deployment, DeploymentLog, DeploymentStatus, LogEvent } from "../../types"; import { getDb } from "../db-provider"; import { deploymentLogs, deployments } from "../schema"; @@ -68,6 +68,18 @@ export const listDeployments = async (projectId?: string, offset = 0, limit = 50 return rows.map(mapDeployment); }; +export const getLatestRunningDeployment = async (projectId: string): Promise => { + const db = await getDb(); + const [row] = await db + .select() + .from(deployments) + .where(and(eq(deployments.projectId, projectId), eq(deployments.status, "running"))) + .orderBy(desc(deployments.createdAt)) + .limit(1) + .execute(); + return row ? mapDeployment(row) : null; +}; + export const countDeployments = async (projectId?: string): Promise => { const db = await getDb(); const rows = projectId diff --git a/apps/api/src/db/repo/index.ts b/apps/api/src/db/repo/index.ts index d99287a..7add6c0 100644 --- a/apps/api/src/db/repo/index.ts +++ b/apps/api/src/db/repo/index.ts @@ -45,6 +45,7 @@ export { createDeployment, deleteDeploymentAndLogs, getDeploymentById, + getLatestRunningDeployment, getLogs, listDeployments, updateDeploymentCommitSha, @@ -116,7 +117,12 @@ export { updateRouteStatus, upsertRoute, } from "./routes"; -export { deleteScalingPolicy, getScalingPolicy, upsertScalingPolicy } from "./scaling"; +export { + ScalingRangeError, + deleteScalingPolicy, + getScalingPolicy, + upsertScalingPolicy, +} from "./scaling"; export type { ServerConnection } from "./servers"; export { createServer, diff --git a/apps/api/src/db/repo/projects.ts b/apps/api/src/db/repo/projects.ts index 34d53c1..66a2592 100644 --- a/apps/api/src/db/repo/projects.ts +++ b/apps/api/src/db/repo/projects.ts @@ -16,10 +16,20 @@ import { import { formatTimestamp, getRowsAffected, now } from "./helpers"; export interface ProjectCleanupInfo { - deploymentContainerNames: string[]; - deploymentImageTags: string[]; - databaseContainerNames: string[]; - databaseVolumeNames: string[]; + isCompose: boolean; + projectServerId: string | null; + deploymentTargets: { + id: string; + containerName: string | null; + imageTag: string | null; + serverId: string | null; + }[]; + databaseTargets: { + containerName: string | null; + proxyContainerName: string | null; + serverId: string | null; + }[]; + databaseVolumes: { volumeName: string; serverId: string | null }[]; volumeDockerNames: string[]; domains: { domain: string; projectName: string }[]; slug: string; @@ -163,12 +173,21 @@ export const deleteProjectCascade = async (id: string): Promise d.containerName).map((d) => d.containerName!); - const deploymentImageTags = depRows.filter((d) => d.imageTag).map((d) => d.imageTag!); + const deploymentTargets = depRows.map((d) => ({ + id: d.id, + containerName: d.containerName, + imageTag: d.imageTag, + serverId: d.serverId, + })); for (const dep of depRows) { await db.delete(deploymentLogs).where(eq(deploymentLogs.deploymentId, dep.id)).execute(); @@ -185,12 +204,23 @@ export const deleteProjectCascade = async (id: string): Promise d.containerName).map((d) => d.containerName!); - const databaseVolumeNames = dbRows.filter((d) => d.volumeName).map((d) => d.volumeName!); + const databaseTargets = dbRows.map((d) => ({ + containerName: d.containerName, + proxyContainerName: d.proxyContainerName, + serverId: d.serverId, + })); + const databaseVolumes = dbRows + .filter((d) => d.volumeName) + .map((d) => ({ volumeName: d.volumeName as string, serverId: d.serverId })); await db.update(databases).set({ projectId: null, updatedAt: now() }).where(eq(databases.projectId, id)).execute(); const domainRows = await db @@ -206,10 +236,11 @@ export const deleteProjectCascade = async (id: string): Promise => { const db = await getDb(); - const existing = await db - .select() - .from(scalingPolicies) - .where(eq(scalingPolicies.projectId, input.projectId)) - .execute(); const timestamp = now(); - if (existing[0]) { - const updates: Record = { updatedAt: timestamp }; - if (input.minReplicas !== undefined) updates.minReplicas = input.minReplicas; - if (input.maxReplicas !== undefined) updates.maxReplicas = input.maxReplicas; - if (input.cpuThresholdPercent !== undefined) updates.cpuThresholdPercent = input.cpuThresholdPercent; - if (input.memoryThresholdPercent !== undefined) updates.memoryThresholdPercent = input.memoryThresholdPercent; - if (input.cooldownSeconds !== undefined) updates.cooldownSeconds = input.cooldownSeconds; - if (input.enabled !== undefined) updates.enabled = input.enabled; - await db.update(scalingPolicies).set(updates).where(eq(scalingPolicies.projectId, input.projectId)).execute(); - const [updated] = await db + return db.transaction(async (tx) => { + const existing = await tx .select() .from(scalingPolicies) .where(eq(scalingPolicies.projectId, input.projectId)) + .for("update") .execute(); - return mapScalingPolicy(updated); - } - const id = randomUUID(); - await db - .insert(scalingPolicies) - .values({ - id, - projectId: input.projectId, - minReplicas: input.minReplicas ?? 1, - maxReplicas: input.maxReplicas ?? 5, - cpuThresholdPercent: input.cpuThresholdPercent ?? 70, - memoryThresholdPercent: input.memoryThresholdPercent ?? 85, - cooldownSeconds: input.cooldownSeconds ?? 120, - enabled: input.enabled !== undefined ? input.enabled : true, - createdAt: timestamp, - updatedAt: timestamp, - }) - .execute(); - const [row] = await db.select().from(scalingPolicies).where(eq(scalingPolicies.id, id)).execute(); - return mapScalingPolicy(row); + const current = existing[0]; + const minReplicas = input.minReplicas ?? current?.minReplicas ?? 1; + const maxReplicas = input.maxReplicas ?? current?.maxReplicas ?? 5; + if (maxReplicas < minReplicas) throw new ScalingRangeError(); + if (current) { + const updates: Record = { updatedAt: timestamp }; + if (input.minReplicas !== undefined) updates.minReplicas = input.minReplicas; + if (input.maxReplicas !== undefined) updates.maxReplicas = input.maxReplicas; + if (input.cpuThresholdPercent !== undefined) updates.cpuThresholdPercent = input.cpuThresholdPercent; + if (input.memoryThresholdPercent !== undefined) updates.memoryThresholdPercent = input.memoryThresholdPercent; + if (input.cooldownSeconds !== undefined) updates.cooldownSeconds = input.cooldownSeconds; + if (input.enabled !== undefined) updates.enabled = input.enabled; + await tx.update(scalingPolicies).set(updates).where(eq(scalingPolicies.projectId, input.projectId)).execute(); + const [updated] = await tx + .select() + .from(scalingPolicies) + .where(eq(scalingPolicies.projectId, input.projectId)) + .execute(); + return mapScalingPolicy(updated); + } + const id = randomUUID(); + await tx + .insert(scalingPolicies) + .values({ + id, + projectId: input.projectId, + minReplicas, + maxReplicas, + cpuThresholdPercent: input.cpuThresholdPercent ?? 70, + memoryThresholdPercent: input.memoryThresholdPercent ?? 85, + cooldownSeconds: input.cooldownSeconds ?? 120, + enabled: input.enabled !== undefined ? input.enabled : true, + createdAt: timestamp, + updatedAt: timestamp, + }) + .execute(); + const [row] = await tx.select().from(scalingPolicies).where(eq(scalingPolicies.id, id)).execute(); + return mapScalingPolicy(row); + }); }; export const getScalingPolicy = async (projectId: string): Promise => { diff --git a/apps/api/src/db/repo/ssh-keys.ts b/apps/api/src/db/repo/ssh-keys.ts index fe788cd..490382f 100644 --- a/apps/api/src/db/repo/ssh-keys.ts +++ b/apps/api/src/db/repo/ssh-keys.ts @@ -7,8 +7,8 @@ import { getDb } from "../db-provider"; import { sshKeys } from "../schema"; import { getRowsAffected, now } from "./helpers"; -const computeFingerprint = (privateKey: string): string => { - return createHash("sha256").update(privateKey).digest("hex").slice(0, 32); +export const computeSshKeyFingerprint = (privateKey: string): string => { + return createHash("sha256").update(privateKey.trim()).digest("hex").slice(0, 32); }; const mapSshKey = (row: typeof sshKeys.$inferSelect): SshKey => ({ @@ -24,7 +24,7 @@ const mapSshKey = (row: typeof sshKeys.$inferSelect): SshKey => ({ export const createSshKey = async (input: CreateSshKeyInput): Promise => { const id = randomUUID(); const timestamp = now(); - const fingerprint = computeFingerprint(input.privateKey); + const fingerprint = computeSshKeyFingerprint(input.privateKey); const encrypted = encryptValue(input.privateKey, config.envEncryptionKey); const db = await getDb(); await db @@ -57,6 +57,12 @@ export const getSshKeyById = async (id: string): Promise => { return row ? mapSshKey(row) : null; }; +export const getSshKeyByFingerprint = async (fingerprint: string): Promise => { + const db = await getDb(); + const [row] = await db.select().from(sshKeys).where(eq(sshKeys.fingerprint, fingerprint)).execute(); + return row ? mapSshKey(row) : null; +}; + export const getSshKeyPrivateKey = async (id: string): Promise => { const db = await getDb(); const [row] = await db diff --git a/apps/api/src/executors/__tests__/dispatch.test.ts b/apps/api/src/executors/__tests__/dispatch.test.ts index ab8d3bb..3ed6ebb 100644 --- a/apps/api/src/executors/__tests__/dispatch.test.ts +++ b/apps/api/src/executors/__tests__/dispatch.test.ts @@ -2,6 +2,8 @@ import { describe, expect, it } from "bun:test"; import { executorFor } from "../dispatch"; import { buildRemoteDeployScript, parseRemoteBuildResult } from "../ssh-build-script"; +const generator = "export const generateDynamicRailpackJson = async () => {};\n"; + const input = { deploymentId: "deployment-1", workspaceRoot: "/var/lib/dequel/workspace", @@ -11,6 +13,7 @@ const input = { imageTag: "example-api-deploym:latest", clearCache: false, environmentVariables: [{ key: "NODE_ENV", value: "production" }], + railpackGenerator: generator, }; describe("remote SSH build script", () => { @@ -47,6 +50,45 @@ describe("remote SSH build script", () => { const script = buildRemoteDeployScript(input); expect(script).toContain("curl -fsSL https://railpack.com/install.sh"); }); + + it("ships the railpack generator into the workspace after the clone", () => { + const script = buildRemoteDeployScript(input); + expect(script).toContain("cat > \"$PROJECT_DIR/.dequel-railpack-gen.ts\" <<'DEQUEL_RAILPACK_GEN_EOF'"); + expect(script).toContain(generator); + expect(script).toContain("await generateDynamicRailpackJson("); + expect(script.indexOf("git clone")).toBeLessThan(script.indexOf("DEQUEL_RAILPACK_GEN_EOF")); + expect(script).toContain('bun "$PROJECT_DIR/.dequel-railpack-gen.ts" "$PROJECT_DIR"'); + expect(script).toContain("DEQUEL_RAILPACK_GEN_EOF\nbun "); + }); + + it("installs bun before generating the config", () => { + const script = buildRemoteDeployScript(input); + expect(script).toContain("bun-linux-${BUN_ARCH}.zip"); + expect(script).toContain("python3 -m zipfile -e"); + expect(script).toContain("bun.sh/install"); + expect(script).toContain('export PATH="$HOME/.bun/bin:$PATH"'); + expect(script.indexOf("bun-linux-${BUN_ARCH}.zip")).toBeLessThan(script.indexOf(".dequel-railpack-gen.ts")); + }); + + it("passes source dir and project type to the generator", () => { + const script = buildRemoteDeployScript({ + ...input, + sourceDir: "client", + projectType: "static", + buildCommand: "npm run build", + startCommand: "npm start", + installCommand: null, + outputDir: null, + }); + expect(script).toContain( + `bun "$PROJECT_DIR/.dequel-railpack-gen.ts" "$PROJECT_DIR" 'client' 'static' 'npm run build' 'npm start' '' ''`, + ); + }); + + it("escapes single quotes in the source dir", () => { + const script = buildRemoteDeployScript({ ...input, sourceDir: "it's" }); + expect(script).toContain(`"$PROJECT_DIR" 'it'\\''s'`); + }); }); describe("remote build result parsing", () => { diff --git a/apps/api/src/executors/ssh-build-script.ts b/apps/api/src/executors/ssh-build-script.ts index 6b94e4c..72dee50 100644 --- a/apps/api/src/executors/ssh-build-script.ts +++ b/apps/api/src/executors/ssh-build-script.ts @@ -7,10 +7,32 @@ export interface SshBuildScriptInput { imageTag: string; clearCache?: boolean; environmentVariables: { key: string; value: string }[]; + sourceDir?: string | null; + projectType?: string | null; + buildCommand?: string | null; + installCommand?: string | null; + outputDir?: string | null; + startCommand?: string | null; + railpackGenerator: string; } const sh = (value: string) => `'${value.replace(/'/g, `'\\''`)}'`; +const GENERATOR_EOF = "DEQUEL_RAILPACK_GEN_EOF"; + +const GENERATOR_CLI = `const [workspace, sourceDir, projectType, buildCommand, startCommand, installCommand, outputDir] = + process.argv.slice(2); +await generateDynamicRailpackJson( + workspace, + sourceDir || null, + projectType || null, + buildCommand || null, + startCommand || null, + async (line) => console.log(line), + installCommand || null, + outputDir || null, +);`; + export const buildRemoteDeployScript = (input: SshBuildScriptInput): string => { const cacheKey = input.imageTag .split(":")[0] @@ -28,6 +50,16 @@ export const buildRemoteDeployScript = (input: SshBuildScriptInput): string => { const remoteWorkspace = input.workspaceRoot.startsWith("/app") ? "$HOME/.dequel/workspace" : input.workspaceRoot; + const generatorArgs = [ + '"$PROJECT_DIR"', + sh(input.sourceDir ?? ""), + sh(input.projectType ?? ""), + sh(input.buildCommand ?? ""), + sh(input.startCommand ?? ""), + sh(input.installCommand ?? ""), + sh(input.outputDir ?? ""), + ].join(" "); + return [ "set -euo pipefail", "", @@ -55,6 +87,45 @@ export const buildRemoteDeployScript = (input: SshBuildScriptInput): string => { `git clone --depth 1 ${sh(input.gitUrl)} .`, checkoutSha, "", + 'if command -v bun >/dev/null 2>&1 || [ -x "$HOME/.bun/bin/bun" ]; then', + ' echo "[build] bun already installed"', + "else", + ' echo "[build] Installing bun (required to generate railpack.json)"', + ' ARCH="$(uname -m)"', + ' case "$ARCH" in', + " x86_64) BUN_ARCH=x64 ;;", + " aarch64|arm64) BUN_ARCH=aarch64 ;;", + ' *) echo "[build] unsupported architecture for bun: $ARCH" >&2; exit 1 ;;', + " esac", + " if command -v python3 >/dev/null 2>&1; then", + ' mkdir -p "$HOME/.bun/bin"', + ' BUN_TMP="$(mktemp -d)"', + ' curl -fsSL -o "$BUN_TMP/bun.zip" "https://github.com/oven-sh/bun/releases/latest/download/bun-linux-${BUN_ARCH}.zip"', + ' python3 -m zipfile -e "$BUN_TMP/bun.zip" "$BUN_TMP"', + ' mv "$BUN_TMP/bun-linux-${BUN_ARCH}/bun" "$HOME/.bun/bin/bun"', + ' chmod +x "$HOME/.bun/bin/bun"', + ' rm -rf "$BUN_TMP"', + " elif command -v unzip >/dev/null 2>&1; then", + ' curl -fsSL https://bun.sh/install | BUN_INSTALL="$HOME/.bun" bash', + " else", + ' echo "[build] bun install requires python3 or unzip on this machine" >&2', + " exit 1", + " fi", + "fi", + 'export PATH="$HOME/.bun/bin:$PATH"', + "if ! command -v bun >/dev/null 2>&1; then", + ' echo "[build] bun is required to generate railpack.json but is not installed" >&2', + " exit 1", + "fi", + "", + 'echo "[build] Generating dynamic railpack.json for caching and monorepo resolution"', + `cat > "$PROJECT_DIR/.dequel-railpack-gen.ts" <<'${GENERATOR_EOF}'`, + input.railpackGenerator, + GENERATOR_CLI, + GENERATOR_EOF, + `bun "$PROJECT_DIR/.dequel-railpack-gen.ts" ${generatorArgs}`, + 'rm -f "$PROJECT_DIR/.dequel-railpack-gen.ts"', + "", 'SHA="$(git rev-parse HEAD)"', `echo "[build] Building image ${input.imageTag} with Railpack"`, `railpack build --name ${sh(input.imageTag)} --progress plain --cache-key ${sh(effectiveCacheKey)} \\`, diff --git a/apps/api/src/executors/ssh.ts b/apps/api/src/executors/ssh.ts index 2c2eff5..fa9771e 100644 --- a/apps/api/src/executors/ssh.ts +++ b/apps/api/src/executors/ssh.ts @@ -1,8 +1,10 @@ -import { summarizeDeploymentError } from "../orchestrator/deployment-errors"; +import { readFileSync } from "node:fs"; +import { join } from "node:path"; +import { remoteScriptFailure, summarizeDeploymentError } from "../orchestrator/deployment-errors"; import type { Deployment, Project, Server } from "../types"; import { config } from "../utils/config"; import { CANCELLED_FAILURE_REASON } from "../utils/failure-outcome"; -import { removeRemoteCaddyRoute, runRemoteScript, syncRemoteCaddyRoute } from "../utils/ssh"; +import { RemoteScriptAbortedError, removeRemoteCaddyRoute, runRemoteScript, syncRemoteCaddyRoute } from "../utils/ssh"; import { emitLog } from "./logging"; import { buildRemoteDeployScript, parseRemoteBuildResult } from "./ssh-build-script"; import { @@ -15,6 +17,8 @@ import type { DeploymentExecutor, ExecutorCancelInput, ExecutorDeployInput, Exec let repoModule: typeof import("../db/repo") | null = null; let runtimeModule: typeof import("../orchestrator/runtime") | null = null; +const railpackGenerator = readFileSync(join(import.meta.dir, "../orchestrator/railpack-config-utils.ts"), "utf8"); + const getRepo = async () => (repoModule ??= await import("../db/repo")); const getRuntime = async () => (runtimeModule ??= await import("../orchestrator/runtime")); @@ -108,7 +112,7 @@ const deployComposeRemote = async (deployment: Deployment, project: Project, ser await emitLog(deployment.id, "build", line); }, }); - if (result.code !== 0) throw new Error(result.stderr || result.stdout || "Remote compose build failed"); + if (result.code !== 0) throw new Error(remoteScriptFailure(result, "Remote compose build failed")); const composeResult = parseRemoteComposeResult(result.stdout); if (!composeResult) throw new Error("Remote compose completed without a result marker"); @@ -171,10 +175,10 @@ const deployComposeRemote = async (deployment: Deployment, project: Project, ser const domain = rawBaseDomain === "localhost" ? `${fallbackDomain}:80` : fallbackDomain; snippet = `${domain} {\n log {\n output stdout\n format json\n }\n reverse_proxy ${primary.container}:${primary.port} {\n header_up Host {upstream_hostport}\n }\n}\n`; } - const { DB_SERVICE_NAMES } = await import("../utils/compose-ingress"); + const { isDbServiceName } = await import("../utils/compose-ingress"); for (const svc of webServices) { if (svc.name === primaryServiceName) continue; - if (DB_SERVICE_NAMES.has(svc.name)) continue; + if (isDbServiceName(svc.name)) continue; const customMatch = customMappings.find((c) => c.serviceName === svc.name); const domains: string[] = []; if (customMatch?.subdomain?.trim()) { @@ -262,6 +266,34 @@ const markFailed = async (deploymentId: string, error: unknown) => { await recordDeploymentFailure({ deploymentId, reason: message, source: "ssh" }); }; +const remoteBuilds = new Map(); + +export const parseRemoteBuildPgid = (line: string): string | undefined => { + const match = line.match(/DEQUEL_PGID:\s*(\d+)/); + if (!match) return undefined; + const pgid = Number(match[1]); + if (!Number.isSafeInteger(pgid) || pgid <= 1) return undefined; + return match[1]; +}; + +export const killRemoteBuildGroup = async ( + server: Parameters[0], + pgid: string, + run: typeof runRemoteScript = runRemoteScript, +): Promise => { + try { + const result = await run( + server, + `kill -TERM -- -${pgid} 2>/dev/null || true +sleep 3 +kill -KILL -- -${pgid} 2>/dev/null || true`, + ); + return result.code === 0; + } catch { + return false; + } +}; + export const sshExecutor: DeploymentExecutor = { mode: "ssh", @@ -288,7 +320,8 @@ export const sshExecutor: DeploymentExecutor = { const imageTag = `${slugify(project.name)}-${deployment.id.slice(0, 8)}:latest`; const envVars = await listEnvironmentVariablesForDeploy(project.id, deployment.environment ?? undefined); - const script = buildRemoteDeployScript({ + const pgidEcho = `echo "DEQUEL_PGID:$(ps -o pgid= -p $$ | tr -d ' ')"`; + const script = `${pgidEcho}\n${buildRemoteDeployScript({ deploymentId: deployment.id, workspaceRoot: config.workspaceRoot, gitUrl: deployment.sourceRef, @@ -297,15 +330,30 @@ export const sshExecutor: DeploymentExecutor = { imageTag, clearCache: deployment.clearCache ?? false, environmentVariables: envVars, - }); - + sourceDir: project.sourceDir, + projectType: project.projectType, + buildCommand: project.buildCommand, + installCommand: project.installCommand, + outputDir: project.outputDir, + startCommand: project.startCommand, + railpackGenerator, + })}`; + + const buildEntry = { controller: new AbortController() }; + remoteBuilds.set(deployment.id, buildEntry); try { const result = await runRemoteScript(server, script, { + signal: buildEntry.controller.signal, onLog: async (line) => { + const pgid = parseRemoteBuildPgid(line); + if (pgid) { + if (buildEntry.pgid === undefined) buildEntry.pgid = pgid; + return; + } await emitLog(deployment.id, "build", line); }, }); - if (result.code !== 0) throw new Error(result.stderr || result.stdout || "Remote build failed"); + if (result.code !== 0) throw new Error(remoteScriptFailure(result, "Remote build failed")); const buildResult = parseRemoteBuildResult(result.stdout); if (!buildResult) throw new Error("Remote build completed without a result marker"); @@ -325,7 +373,11 @@ export const sshExecutor: DeploymentExecutor = { await emitLog(current.id, "system", `Marked inactive (superseded by ${deployment.id.slice(0, 8)})`); } } catch (error) { - await markFailed(deployment.id, error); + if (!(error instanceof RemoteScriptAbortedError)) { + await markFailed(deployment.id, error); + } + } finally { + remoteBuilds.delete(deployment.id); } }, @@ -401,7 +453,7 @@ export const sshExecutor: DeploymentExecutor = { await deleteDeploymentAndLogs(deployment.id); }, - async cancel({ deployment }: ExecutorCancelInput) { + async cancel({ deployment, server }: ExecutorCancelInput) { const { recordDeploymentCancellation } = await getRepo(); if (deployment.status !== "pending" && deployment.status !== "building") return; await recordDeploymentCancellation({ @@ -409,6 +461,27 @@ export const sshExecutor: DeploymentExecutor = { reason: CANCELLED_FAILURE_REASON, source: "ssh", }); - await emitLog(deployment.id, "system", "Deployment cancelled by user (remote build may continue on the server)"); + const entry = remoteBuilds.get(deployment.id); + let remoteStopped = false; + if (entry) { + entry.controller.abort(); + if (entry.pgid) { + let timer: ReturnType | undefined; + const timeout = new Promise((resolve) => { + timer = setTimeout(() => resolve(false), 10_000); + }); + remoteStopped = await Promise.race([killRemoteBuildGroup(server, entry.pgid), timeout]); + clearTimeout(timer); + } + } + await emitLog( + deployment.id, + "system", + remoteStopped + ? "Deployment cancelled by user — remote build processes stopped" + : entry?.pgid + ? "Deployment cancelled by user — remote build termination could not be confirmed" + : "Deployment cancelled by user (remote build may continue on the server)", + ); }, }; diff --git a/apps/api/src/fixdiag/provider-models.ts b/apps/api/src/fixdiag/provider-models.ts index 308efa0..b66b674 100644 --- a/apps/api/src/fixdiag/provider-models.ts +++ b/apps/api/src/fixdiag/provider-models.ts @@ -6,22 +6,9 @@ export interface FetchModelsOptions { } export const DEFAULT_PROVIDER_MODELS: Record = { - openai: ["gpt-4o", "gpt-4o-mini", "o1", "o3-mini", "gpt-4-turbo", "gpt-6.1-sol", "gpt-6-astra", "gpt-5.6-sol"], - anthropic: [ - "claude-3-5-sonnet-latest", - "claude-3-5-haiku-latest", - "claude-3-opus-latest", - "claude-sonnet-5", - "claude-opus-5", - ], - gemini: [ - "gemini-2.5-flash", - "gemini-2.5-pro", - "gemini-1.5-pro", - "gemini-1.5-flash", - "gemini-3.8-flash", - "gemini-3.7-flash", - ], + openai: ["gpt-4o", "gpt-4o-mini", "o1", "o3-mini", "gpt-4-turbo"], + anthropic: ["claude-3-5-sonnet-latest", "claude-3-5-haiku-latest", "claude-3-opus-latest"], + gemini: ["gemini-2.5-flash", "gemini-2.5-pro", "gemini-1.5-pro", "gemini-1.5-flash"], groq: [ "llama-3.3-70b-versatile", "llama-3.1-8b-instant", @@ -29,7 +16,7 @@ export const DEFAULT_PROVIDER_MODELS: Record = { "moonshotai/kimi-k2-instruct", "openai/gpt-oss-120b", ], - ollama: ["llama3:latest", "qwen2.5-coder:latest", "mistral:latest", "deepseek-r1:latest", "qwen3.5:0.8b"], + ollama: ["llama3:latest", "qwen2.5-coder:latest", "mistral:latest", "deepseek-r1:latest"], custom: [], }; diff --git a/apps/api/src/index.ts b/apps/api/src/index.ts index c296cee..86c8305 100644 --- a/apps/api/src/index.ts +++ b/apps/api/src/index.ts @@ -6,10 +6,11 @@ import { apiRoutes } from "./api"; import { startBackupScheduler } from "./backup/scheduler"; import { startDatabaseMonitoring } from "./databases/manager"; import { getDb } from "./db/db-provider"; +import { migrateEncryptionKeys } from "./db/encryption-migration"; import { migrate } from "./db/migrate"; import { ensureLocalServer } from "./db/repo"; import { markInterruptedDiagRuns } from "./db/repo/diag-runs"; -import { deployments } from "./db/schema"; +import { deployments, projects } from "./db/schema"; import { alertEvaluator } from "./monitoring/evaluator"; import { startFailureNotifier } from "./monitoring/failure-notifier"; import { orchestrator } from "./orchestrator"; @@ -21,14 +22,10 @@ import { serverManager } from "./servers/manager"; import { cleanupExpiredTokens, initAuth } from "./utils/auth"; import { config } from "./utils/config"; import { startDomainPolling } from "./utils/domain-verifier"; -import { initZombieReaper } from "./utils/process-exec"; import { loadOrCreateJwtSecret } from "./utils/secrets"; - -import { projects } from "./db/schema"; import { captureTelemetry } from "./utils/telemetry"; const bootstrap = async () => { - initZombieReaper(); await mkdir(config.workspaceRoot, { recursive: true }); await mkdir(config.caddyRoutesDir, { recursive: true }); @@ -36,6 +33,10 @@ const bootstrap = async () => { initAuth(jwtSecret); await migrate(); + await migrateEncryptionKeys().catch((err) => { + console.error("[EncryptionMigration] failed:", err); + if (process.env.NODE_ENV === "production") throw err; + }); await ensureLocalServer(); const interrupted = await markInterruptedDiagRuns().catch(() => 0); if (interrupted > 0) console.log(`[Fixdiag] Marked ${interrupted} interrupted diagnosis run(s) as failed`); @@ -99,7 +100,12 @@ dequel_uptime_seconds ${uptimeSec} }; const app = new Elysia() - .use(cors()) + .use( + cors({ + origin: () => true, + credentials: false, + }), + ) .onBeforeHandle(() => { metrics.requestsTotal++; }) diff --git a/apps/api/src/orchestrator/__tests__/deployment-errors.test.ts b/apps/api/src/orchestrator/__tests__/deployment-errors.test.ts index fdbf786..4b0a5bd 100644 --- a/apps/api/src/orchestrator/__tests__/deployment-errors.test.ts +++ b/apps/api/src/orchestrator/__tests__/deployment-errors.test.ts @@ -1,5 +1,5 @@ import { describe, expect, test } from "bun:test"; -import { summarizeDeploymentError } from "../deployment-errors"; +import { remoteScriptFailure, summarizeDeploymentError } from "../deployment-errors"; describe("summarizeDeploymentError", () => { test("extracts an actionable runtime requirement from noisy build output", () => { @@ -33,3 +33,24 @@ unrecognized image format`); ); }); }); + +describe("remoteScriptFailure", () => { + test("prefers stdout so git progress on stderr cannot become the failure reason", () => { + const message = remoteScriptFailure( + { stdout: "✖ Railpack could not determine how to build the app.", stderr: "Cloning into '.'..." }, + "Remote build failed", + ); + expect(message).toContain("Railpack could not determine"); + expect(message).toContain("Cloning into"); + }); + + test("keeps stderr when stdout is empty", () => { + expect(remoteScriptFailure({ stdout: "", stderr: "Permission denied (publickey)." }, "fallback")).toBe( + "Permission denied (publickey).", + ); + }); + + test("falls back when both streams are empty", () => { + expect(remoteScriptFailure({ stdout: "", stderr: "" }, "Remote build failed")).toBe("Remote build failed"); + }); +}); diff --git a/apps/api/src/orchestrator/__tests__/failover-runner.ts b/apps/api/src/orchestrator/__tests__/failover-runner.ts index b500c01..7a691bd 100644 --- a/apps/api/src/orchestrator/__tests__/failover-runner.ts +++ b/apps/api/src/orchestrator/__tests__/failover-runner.ts @@ -1,4 +1,5 @@ import { mock } from "bun:test"; +import net from "node:net"; const fileUrl = (relPath: string) => new URL(relPath, import.meta.url).toString(); @@ -7,6 +8,22 @@ let servers: any[] = []; let projects: any[] = []; let deploymentsByProject: Record = {}; +const dockerCalls: any[][] = []; + +mock.module(fileUrl("../../utils/docker-run"), () => ({ + getDockerTargetArgs: mock(() => []), + dockerRun: mock(() => Promise.resolve("")), + dockerRunTry: mock((...args: any[]) => { + dockerCalls.push(args); + return Promise.resolve(""); + }), + dockerExecStream: mock(() => null), + dockerExec: mock(() => Promise.resolve({ code: 0, stdout: "", stderr: "" })), + dockerExecWithStdin: mock(() => Promise.resolve({ code: 0, stdout: "", stderr: "" })), + ensureContainerRemoved: mock(() => Promise.resolve()), + ensureVolumeRemoved: mock(() => Promise.resolve()), +})); + mock.module(fileUrl("../../db/repo"), () => ({ getPlatformSettings: mock(() => Promise.resolve(platformSettings)), getProjectById: mock((id: string) => Promise.resolve(projects.find((p) => p.id === id) ?? null)), @@ -15,6 +32,10 @@ mock.module(fileUrl("../../db/repo"), () => ({ listDeployments: mock((projectId?: string) => Promise.resolve(projectId ? (deploymentsByProject[projectId] ?? []) : []), ), + getLatestRunningDeployment: mock((projectId: string) => { + const list = deploymentsByProject[projectId] ?? []; + return Promise.resolve(list.find((d: any) => d.status === "running") ?? list[0] ?? null); + }), createDeployment: mock((input: any) => Promise.resolve({ id: "dep-new", ...input })), updateProject: mock(() => Promise.resolve()), updateDeploymentStatus: mock(() => Promise.resolve()), @@ -47,6 +68,10 @@ mock.module(fileUrl("../../executors/dispatch"), () => ({ })), })); +const probeListener = net.createServer(); +await new Promise((resolve) => probeListener.listen(0, "127.0.0.1", resolve)); +const probePort = (probeListener.address() as net.AddressInfo).port; + const { failoverProject } = await import("../failover"); const results: any = {}; @@ -62,7 +87,10 @@ try { await failoverProject("p1"); results.test1 = { ok: false, error: "should have thrown" }; } catch (e: any) { - results.test1 = { ok: e.message?.includes("No other healthy server available") }; + results.test1 = { + ok: e.message?.includes("No other healthy server available") && e.message?.includes("p1"), + error: e.message, + }; } // Test 2: rejects non-ssh project servers @@ -115,12 +143,20 @@ try { platformSettings = { ingressServerId: "ing" }; servers = [ { id: "ing", name: "Ingress", mode: "ssh" }, - { id: "a", name: "Current", mode: "ssh", status: "connected" }, - { id: "b", name: "Target", mode: "ssh", status: "connected" }, + { id: "a", name: "Current", mode: "ssh", status: "connected", host: "127.0.0.1", port: probePort }, + { id: "b", name: "Target", mode: "ssh", status: "connected", host: "127.0.0.1", port: probePort }, ]; projects = [{ id: "p1", name: "proj-1", serverId: "a" }]; deploymentsByProject = { - p1: [{ id: "dep-1", sourceType: "git", sourceRef: "https://github.com/x/y.git", branch: "main" }], + p1: [ + { + id: "dep-1", + sourceType: "git", + sourceRef: "https://github.com/x/y.git", + branch: "main", + containerName: "app-old", + }, + ], }; const deployment = await failoverProject("p1"); @@ -133,4 +169,85 @@ try { results.test5 = { ok: false, error: e.message }; } +// Test 6: aborts when the chosen target is not actually reachable +let createDeploymentCallsBefore6 = 0; +try { + platformSettings = { ingressServerId: "ing" }; + servers = [ + { id: "ing", name: "Ingress", mode: "ssh" }, + { id: "a", name: "Current", mode: "ssh", status: "connected", host: "127.0.0.1", port: probePort }, + { id: "b", name: "DeadTarget", mode: "ssh", status: "connected", host: "127.0.0.1", port: 1 }, + ]; + projects = [{ id: "p1", name: "proj-1", serverId: "a" }]; + deploymentsByProject = { + p1: [ + { + id: "dep-1", + sourceType: "git", + sourceRef: "https://github.com/x/y.git", + branch: "main", + containerName: "app-old", + }, + ], + }; + const repo6 = await import("../../db/repo"); + createDeploymentCallsBefore6 = (repo6.createDeployment as any).mock.calls.length; + await failoverProject("p1"); + results.test6 = { ok: false, error: "should have thrown" }; +} catch (e: any) { + const repo = await import("../../db/repo"); + const created = (repo.createDeployment as any).mock.calls.length - createDeploymentCallsBefore6; + results.test6 = { + ok: e.message?.includes("not reachable") && created === 0, + error: e.message, + }; +} + +// Test 7: stops the previous deployment's containers on the old server +try { + const primaryRm = dockerCalls.some( + (args) => args[1]?.[0] === "rm" && args[1]?.includes("app-old") && args[2]?.id === "a", + ); + const replicaList = dockerCalls.some( + (args) => + args[1]?.[0] === "ps" && args[1]?.some((a: string) => a.includes("deploy-dep-1-replica-")) && args[2]?.id === "a", + ); + results.test7 = { ok: primaryRm && replicaList }; +} catch (e: any) { + results.test7 = { ok: false, error: e.message }; +} + +// Test 8: refuses a second automatic failover within the minimum interval +try { + await failoverProject("p1", { trigger: "auto" }); + results.test8 = { ok: false, error: "should have thrown" }; +} catch (e: any) { + results.test8 = { ok: e.message?.includes("before another failover"), error: e.message }; +} + +// Test 9: no-deployment error carries project context on a single line +try { + platformSettings = { ingressServerId: "ing" }; + servers = [ + { id: "ing", name: "Ingress", mode: "ssh" }, + { id: "a", name: "Current", mode: "ssh", status: "connected", host: "127.0.0.1", port: probePort }, + ]; + projects = [{ id: "p1", serverId: "a" }]; + deploymentsByProject = { p1: [] }; + await failoverProject("p1"); + results.test9 = { ok: false, error: "should have thrown" }; +} catch (e: any) { + const message = e.message ?? ""; + results.test9 = { + ok: message.includes("no running deployment to fail over") && message.includes("p1") && !message.includes("\n"), + error: message, + }; +} + +probeListener.close(); console.log(JSON.stringify(results)); +const failed = Object.entries(results).filter(([, r]) => (r as any)?.ok !== true); +if (failed.length > 0) { + console.error(`FAIL: ${failed.map(([k]) => k).join(", ")}`); + process.exit(1); +} diff --git a/apps/api/src/orchestrator/__tests__/failover-stale-routes-runner.ts b/apps/api/src/orchestrator/__tests__/failover-stale-routes-runner.ts index 75c81cd..1b13555 100644 --- a/apps/api/src/orchestrator/__tests__/failover-stale-routes-runner.ts +++ b/apps/api/src/orchestrator/__tests__/failover-stale-routes-runner.ts @@ -17,6 +17,7 @@ mock.module(fileUrl("../../db/repo"), () => ({ getServerById: mock((id: string) => Promise.resolve(servers.find((s) => s.id === id) ?? null)), listProjects: mock(() => Promise.resolve(projects)), listDeployments: mock(() => Promise.resolve([])), + getLatestRunningDeployment: mock(() => Promise.resolve(null)), createDeployment: mock(() => Promise.resolve({ id: "dep-new" })), updateProject: mock(() => Promise.resolve()), updateDeploymentStatus: mock(() => Promise.resolve()), diff --git a/apps/api/src/orchestrator/__tests__/failover.test.ts b/apps/api/src/orchestrator/__tests__/failover.test.ts index 5e442b1..890e78e 100644 --- a/apps/api/src/orchestrator/__tests__/failover.test.ts +++ b/apps/api/src/orchestrator/__tests__/failover.test.ts @@ -45,4 +45,28 @@ describe("failoverProject", () => { const r = parse(stdout); expect(r.test5.ok).toBe(true); }); + + it("aborts when the chosen target server is not reachable", async () => { + const { stdout } = await run(); + const r = parse(stdout); + expect(r.test6.ok).toBe(true); + }); + + it("removes the previous deployment's containers on the old server", async () => { + const { stdout } = await run(); + const r = parse(stdout); + expect(r.test7.ok).toBe(true); + }); + + it("refuses a second automatic failover within the minimum interval", async () => { + const { stdout } = await run(); + const r = parse(stdout); + expect(r.test8.ok).toBe(true); + }); + + it("includes project context in the no-deployments error", async () => { + const { stdout } = await run(); + const r = parse(stdout); + expect(r.test9.ok).toBe(true); + }); }); diff --git a/apps/api/src/orchestrator/__tests__/runtime-logs-runner.ts b/apps/api/src/orchestrator/__tests__/runtime-logs-runner.ts new file mode 100644 index 0000000..2159e72 --- /dev/null +++ b/apps/api/src/orchestrator/__tests__/runtime-logs-runner.ts @@ -0,0 +1,67 @@ +import { mock } from "bun:test"; + +const fileUrl = (relPath: string) => new URL(relPath, import.meta.url).toString(); + +const servers: Record = { + "srv-worker": { id: "srv-worker", mode: "ssh", host: "1.2.3.4", port: 22 }, +}; + +const dockerCalls: any[][] = []; +const targetArgsCalls: any[] = []; + +mock.module(fileUrl("../../db/repo"), () => ({ + getServerById: mock((id: string) => Promise.resolve(servers[id] ?? null)), +})); + +mock.module(fileUrl("../../utils/docker-run"), () => ({ + dockerRun: mock((...args: any[]) => { + dockerCalls.push(args); + return Promise.resolve("line one\nline two"); + }), + getDockerTargetArgs: mock((server: any) => { + targetArgsCalls.push(server); + return server?.mode === "ssh" ? ["-H", `ssh://root@${server.host}:${server.port}`] : []; + }), +})); + +try { + const { readRuntimeLogs, runtimeLogFollowerArgs } = await import("../runtime-logs"); + + const results: any = {}; + + const lines = await readRuntimeLogs({ id: "dep-1", containerName: "app-old", serverId: "srv-worker" }); + results.readOnRemote = { + count: lines.length, + first: { sequence: lines[0].sequence, message: lines[0].message, stage: lines[0].stage }, + serverPassed: dockerCalls[0][2]?.id === "srv-worker", + command: dockerCalls[0].slice(0, 2), + }; + + dockerCalls.length = 0; + const localLines = await readRuntimeLogs({ id: "dep-2", containerName: null, serverId: null }); + results.readLocal = { + count: localLines.length, + command: dockerCalls[0].slice(0, 2), + serverNull: dockerCalls[0][2] === null, + }; + + targetArgsCalls.length = 0; + const followerArgs = await runtimeLogFollowerArgs({ id: "dep-3", containerName: "app-1", serverId: "srv-worker" }); + results.follower = { + args: followerArgs, + serverPassed: targetArgsCalls[0]?.id === "srv-worker", + }; + + targetArgsCalls.length = 0; + const localArgs = await runtimeLogFollowerArgs({ id: "dep-4", containerName: null, serverId: null }); + results.followerLocal = { + args: localArgs, + serverNull: targetArgsCalls[0] === null, + }; + + console.log(JSON.stringify(results)); +} catch (err: any) { + console.error("Runner error:", err?.message ?? String(err)); + console.error(err?.stack ?? err); + process.exit(1); +} diff --git a/apps/api/src/orchestrator/__tests__/runtime-logs.test.ts b/apps/api/src/orchestrator/__tests__/runtime-logs.test.ts new file mode 100644 index 0000000..a2cdfa4 --- /dev/null +++ b/apps/api/src/orchestrator/__tests__/runtime-logs.test.ts @@ -0,0 +1,46 @@ +import { describe, expect, it } from "bun:test"; +import { execFile } from "node:child_process"; +import { promisify } from "node:util"; + +const exec = promisify(execFile); + +const run = () => + exec("bun", ["run", "src/orchestrator/__tests__/runtime-logs-runner.ts"], { + cwd: `${import.meta.dir}/../../..`, + timeout: 30000, + }); + +const parse = (stdout: string) => { + const lines = stdout.trim().split("\n"); + return JSON.parse(lines[lines.length - 1]); +}; + +describe("runtime logs target the deployment's server", () => { + it("reads logs through the remote server and maps lines", async () => { + const { stdout } = await run(); + const results = parse(stdout); + + expect(results.readOnRemote.count).toBe(2); + expect(results.readOnRemote.first).toEqual({ sequence: 1, message: "line one", stage: "runtime" }); + expect(results.readOnRemote.serverPassed).toBe(true); + expect(results.readOnRemote.command).toEqual(["docker", ["logs", "--tail", "200", "app-old"]]); + + expect(results.readLocal.count).toBe(2); + expect(results.readLocal.command).toEqual(["docker", ["logs", "--tail", "200", "deploy-dep-2"]]); + expect(results.readLocal.serverNull).toBe(true); + + expect(results.follower.serverPassed).toBe(true); + expect(results.follower.args).toEqual([ + "-H", + "ssh://root@1.2.3.4:22", + "logs", + "--tail", + "100", + "--follow", + "app-1", + ]); + + expect(results.followerLocal.serverNull).toBe(true); + expect(results.followerLocal.args).toEqual(["logs", "--tail", "100", "--follow", "deploy-dep-4"]); + }, 30_000); +}); diff --git a/apps/api/src/orchestrator/compose.ts b/apps/api/src/orchestrator/compose.ts index c90285e..de13f74 100644 --- a/apps/api/src/orchestrator/compose.ts +++ b/apps/api/src/orchestrator/compose.ts @@ -1,8 +1,11 @@ import { spawn } from "node:child_process"; import { existsSync, readFileSync, realpathSync } from "node:fs"; import { dirname, join, resolve, sep } from "node:path"; +import type { Server } from "../types"; import { dockerBin } from "../utils/docker-bin"; -import { killProcessGroup, safeSpawn } from "../utils/process-exec"; +import { dockerRun } from "../utils/docker-run"; +import { killChildTree } from "../utils/proc-group"; +import { safeSpawn } from "../utils/process-exec"; import { CancelledError } from "./railpack"; export interface ComposeTarget { @@ -205,24 +208,14 @@ const spawnComposeCommand = ( cwd, env: envVars || {}, stdio: ["ignore", "pipe", "pipe"], - detached: isPosix, + detached: true, }); let stdout = ""; let stderr = ""; let settled = false; - let escalationTimer: ReturnType | null = null; - - const terminate = () => { - killProcessGroup(child, "SIGTERM"); - escalationTimer = setTimeout(() => { - killProcessGroup(child, "SIGKILL"); - }, 5000); - if (typeof escalationTimer.unref === "function") escalationTimer.unref(); - }; const cleanup = () => { - if (escalationTimer) clearTimeout(escalationTimer); if (signal) { signal.removeEventListener("abort", onAbort); } @@ -240,7 +233,8 @@ const spawnComposeCommand = ( }; const onAbort = () => { - terminate(); + killChildTree(child, "SIGTERM"); + setTimeout(() => killChildTree(child, "SIGKILL"), 5000); finish(undefined, new CancelledError()); }; @@ -339,14 +333,14 @@ export const deployWithCompose = async ( await onLog("Docker Compose stack started."); }; -export const destroyComposeStack = async (projectName: string): Promise => { +export const destroyComposeStack = async (projectName: string, server?: Server | null): Promise => { const filter = `label=com.docker.compose.project=${projectName}`; - const ps = await runDocker(["ps", "-aq", "--filter", filter]); - const ids = ps.stdout.split(/\s+/).filter(Boolean); + const ps = await dockerRun(dockerBin, ["ps", "-aq", "--filter", filter], server).catch(() => ""); + const ids = ps.split(/\s+/).filter((s) => /^[0-9a-f]{12,}$/.test(s)); for (const id of ids) { - await runDocker(["rm", "-f", id]); + await dockerRun(dockerBin, ["rm", "-f", id], server).catch(() => {}); } - await runDocker(["network", "rm", `${projectName}_default`]).catch(() => {}); + await dockerRun(dockerBin, ["network", "rm", `${projectName}_default`], server).catch(() => {}); }; export const getComposeContainerNames = async (projectName: string): Promise> => { diff --git a/apps/api/src/orchestrator/deployment-errors.ts b/apps/api/src/orchestrator/deployment-errors.ts index 8f9bbf1..770a3d2 100644 --- a/apps/api/src/orchestrator/deployment-errors.ts +++ b/apps/api/src/orchestrator/deployment-errors.ts @@ -29,3 +29,6 @@ export const summarizeDeploymentError = (error: unknown): string => { return "Deployment failed. Check the deployment logs for details."; }; + +export const remoteScriptFailure = (result: { stdout: string; stderr: string }, fallback: string): string => + [result.stdout, result.stderr].filter(Boolean).join("\n") || fallback; diff --git a/apps/api/src/orchestrator/failover.ts b/apps/api/src/orchestrator/failover.ts index 5b37cbb..d00f61e 100644 --- a/apps/api/src/orchestrator/failover.ts +++ b/apps/api/src/orchestrator/failover.ts @@ -2,25 +2,33 @@ import net from "node:net"; import { appendLog, createDeployment, + getLatestRunningDeployment, getProjectById, getServerById, - listDeployments, listProjects, listRoutes, updateDeploymentStatus, updateProject, updateRouteStatus, } from "../db/repo"; +import type { Server } from "../types"; +import { config } from "../utils/config"; +import { dockerBin } from "../utils/docker-bin"; +import { dockerRunTry } from "../utils/docker-run"; import { getIngressServer, removeIngressRouteFile } from "../utils/ingress"; import { pickBestServer } from "../utils/server-default"; +import { getContainerName } from "./runtime"; const CHECK_INTERVAL_MS = 30_000; const GRACE_MS = 180_000; const CONNECT_TIMEOUT_MS = 5_000; +const OLD_CONTAINER_CLEANUP_TIMEOUT_MS = 30_000; const unreachableSince = new Map(); const failingOver = new Set(); const previouslyUnreachableServers = new Set(); +const lastFailoverAt = new Map(); +const rateLimitWarned = new Set(); export const isServerReachable = (host: string, port: number = 22): Promise => new Promise((resolve) => { @@ -34,29 +42,97 @@ export const isServerReachable = (host: string, port: number = 22): Promise finish(false)); }); -export const failoverProject = async (projectId: string) => { +const isFailoverDisabled = () => /^(1|true|yes)$/i.test(config.failoverDisabled); + +const formatError = (error: unknown): string => { + const message = error instanceof Error ? error.message : String(error); + return message.trim() || "unknown error"; +}; + +const bestEffort = (work: Promise, label: string): Promise => + Promise.race([ + work, + new Promise((_resolve, reject) => { + const timer = setTimeout(() => reject(new Error("timed out")), OLD_CONTAINER_CLEANUP_TIMEOUT_MS); + timer.unref?.(); + }), + ]) + .then(() => undefined) + .catch((error) => { + console.error(`[Failover] ${label} failed: ${formatError(error)}`); + }); + +const removeOldServerContainers = ( + server: Server, + deployment: { id: string; containerName?: string | null }, + project: { name?: string | null; id: string }, +): Promise => { + const primaryName = + deployment.containerName || getContainerName(deployment.id, project.name ?? undefined, project.id); + const remove = async () => { + const listed = await dockerRunTry( + dockerBin, + ["ps", "-a", "--format", "{{.Names}}", "--filter", `name=deploy-${deployment.id}-replica-`], + server, + ); + const replicas = (listed ?? "") + .split("\n") + .map((line) => line.trim()) + .filter(Boolean); + const names = [primaryName, ...replicas]; + let removed = 0; + for (const name of names) { + await dockerRunTry(dockerBin, ["stop", "-t", "10", name], server); + const result = await dockerRunTry(dockerBin, ["rm", "-f", name], server); + if (result !== undefined) removed++; + } + console.log( + `[Failover] old-container cleanup on ${server.name}: removed ${removed}/${names.length} (${names.join(", ")})`, + ); + }; + return bestEffort(remove(), `old-container cleanup on ${server.name}`); +}; + +export const failoverProject = async ( + projectId: string, + opts: { trigger?: "manual" | "auto" } = {}, +): Promise>> => { if (failingOver.has(projectId)) throw new Error("Failover already in progress for this project"); const project = await getProjectById(projectId); if (!project) throw new Error("Project not found"); if (!project.serverId) throw new Error("Project has no server assigned"); + if ((opts.trigger ?? "manual") === "auto") { + const last = lastFailoverAt.get(projectId); + const since = last ? Date.now() - last : Number.POSITIVE_INFINITY; + if (since < config.failoverMinIntervalMs) { + throw new Error( + `Project failed over ${Math.round(since / 1000)}s ago — waiting ${config.failoverMinIntervalMs / 1000}s before another failover`, + ); + } + } const ingressServer = await getIngressServer(); if (!ingressServer) throw new Error("No ingress server configured"); const currentServer = await getServerById(project.serverId); if (currentServer?.mode !== "ssh") throw new Error("Failover only supports SSH project servers"); - const deployments = await listDeployments(projectId, 0, 1); - const latest = deployments[0]; - if (!latest) throw new Error("Project has no deployments to fail over"); + const latest = await getLatestRunningDeployment(projectId); + if (!latest) { + throw new Error(`Project ${project.name ?? projectId} has no running deployment to fail over from`); + } if (latest.sourceType !== "git") throw new Error("Failover requires a Git deployment"); failingOver.add(projectId); try { const targetId = await pickBestServer(null, project.serverId, ["ssh"]); if (!targetId || targetId === project.serverId || targetId === "local") { - throw new Error("No other healthy server available for failover"); + throw new Error(`No other healthy server available for failover of project ${project.name ?? projectId}`); } const targetServer = await getServerById(targetId); if (!targetServer) throw new Error("Target server not found"); + const targetReachable = await isServerReachable(targetServer.host, targetServer.port || 22); + if (!targetReachable) { + throw new Error(`Target server ${targetServer.name} is not reachable — failover aborted`); + } const deployment = await createDeployment({ projectId, @@ -75,18 +151,24 @@ export const failoverProject = async (projectId: string) => { void executorFor("ssh") .deploy({ deployment, project, server: targetServer }) .catch((error) => { - console.error(`[Failover] Deployment ${deployment.id} failed:`, error); + console.error(`[Failover] Deployment ${deployment.id} failed: ${formatError(error)}`); }); } else { const { queueRemoteDeployment } = await import("../agents/deployments"); await queueRemoteDeployment(deployment, project); } + await removeOldServerContainers(currentServer, latest, project); await updateProject(projectId, { serverId: targetId }); + lastFailoverAt.set(projectId, Date.now()); + rateLimitWarned.delete(projectId); unreachableSince.delete(projectId); await updateDeploymentStatus(latest.id, "inactive", { failureReason: `Superseded by failover deployment to ${targetServer.name}`, }).catch(() => {}); + console.log( + `[Failover] project ${project.name ?? projectId} moved from ${currentServer.name} to ${targetServer.name} (deployment ${deployment.id})`, + ); return deployment; } finally { failingOver.delete(projectId); @@ -148,8 +230,16 @@ export const failoverMonitorTick = async () => { console.log( `[Failover] Server ${server.name} unreachable for ${Math.round((Date.now() - firstSeen) / 1000)}s — failing over project ${project.name}`, ); - failoverProject(project.id).catch((error) => { - console.error(`[Failover] Auto-failover for project ${project.id} failed:`, error); + failoverProject(project.id, { trigger: "auto" }).catch((error) => { + const message = formatError(error); + if (message.includes("before another failover")) { + if (!rateLimitWarned.has(project.id)) { + rateLimitWarned.add(project.id); + console.log(`[Failover] skipping project ${project.name}: ${message}`); + } + return; + } + console.error(`[Failover] Auto-failover for project ${project.id} failed: ${message}`); unreachableSince.delete(project.id); }); }); @@ -159,25 +249,31 @@ export const failoverMonitorTick = async () => { for (const serverId of recoveredServers) { console.log(`[Failover] Server ${serverId} recovered — cleaning up stale routes`); await cleanupStaleRoutes(ingressServer, serverId).catch((error) => { - console.error(`[Failover] Stale route cleanup for server ${serverId} failed:`, error); + console.error(`[Failover] Stale route cleanup for server ${serverId} failed: ${formatError(error)}`); }); } previouslyUnreachableServers.clear(); for (const id of currentUnreachableServers) previouslyUnreachableServers.add(id); } catch (error) { - console.error("[Failover] Monitor tick failed:", error); + console.error(`[Failover] Monitor tick failed: ${formatError(error)}`); } }; export const startFailoverMonitor = () => { + if (isFailoverDisabled()) { + console.log("[Failover] monitor disabled via FAILOVER_DISABLED"); + return null; + } const handle = setInterval(failoverMonitorTick, CHECK_INTERVAL_MS); failoverMonitorTick(); return handle; }; export const failoverState = () => ({ + disabled: isFailoverDisabled(), unreachable: [...unreachableSince.entries()].map(([projectId, since]) => ({ projectId, unreachableForMs: Date.now() - since, })), + minIntervalMs: config.failoverMinIntervalMs, }); diff --git a/apps/api/src/orchestrator/project-cleanup.ts b/apps/api/src/orchestrator/project-cleanup.ts new file mode 100644 index 0000000..3a70f46 --- /dev/null +++ b/apps/api/src/orchestrator/project-cleanup.ts @@ -0,0 +1,110 @@ +import type { ProjectCleanupInfo } from "../db/repo"; +import type { Server } from "../types"; + +const CONTAINER_ID = /^[0-9a-f]{12,}$/; + +export interface ProjectCleanupRunner { + getServerById: (id: string) => Promise; + ensureContainerRemoved: (name: string, server?: Server | null) => Promise; + ensureVolumeRemoved: (name: string, server?: Server | null) => Promise; + destroyComposeStack: (projectName: string, server?: Server | null) => Promise; + dockerRunTry: (cmd: string, args: string[], server?: Server | null) => Promise; +} + +export type ProjectCleanupTarget = Pick< + ProjectCleanupInfo, + "isCompose" | "projectServerId" | "deploymentTargets" | "databaseTargets" | "databaseVolumes" | "volumeDockerNames" +>; + +export const cleanupProjectContainers = async ( + info: ProjectCleanupTarget, + runner: ProjectCleanupRunner, +): Promise => { + const failures: string[] = []; + const serverCache = new Map(); + const serverFor = async (serverId: string | null): Promise => { + if (!serverId) return null; + if (!serverCache.has(serverId)) { + serverCache.set(serverId, await runner.getServerById(serverId).catch(() => null)); + } + return serverCache.get(serverId) ?? null; + }; + const requireServer = async (label: string, serverId: string | null): Promise => { + if (!serverId) return null; + const server = await serverFor(serverId); + if (!server) { + failures.push(`${label}: owning server ${serverId} could not be resolved`); + return undefined; + } + return server; + }; + const attempt = async (label: string, fn: () => Promise) => { + try { + await fn(); + } catch (err) { + failures.push(`${label}: ${(err as Error)?.message ?? err}`); + } + }; + + for (const dep of info.deploymentTargets) { + const label = `deployment ${dep.id}`; + const server = await requireServer(label, dep.serverId); + if (server === undefined) continue; + if (info.isCompose) { + await attempt(`${label} compose stack`, () => + runner.destroyComposeStack(dep.containerName || `deploy-${dep.id}`, server), + ); + } else { + if (dep.containerName) { + await attempt(`${label} container ${dep.containerName}`, () => + runner.ensureContainerRemoved(dep.containerName as string, server), + ); + } + const replicas = await runner + .dockerRunTry("docker", ["ps", "-aq", "--filter", `name=deploy-${dep.id}-replica-`], server) + .catch(() => undefined); + for (const cid of (replicas ?? "").split(/\s+/).filter((s) => CONTAINER_ID.test(s))) { + await attempt(`${label} replica ${cid}`, () => runner.dockerRunTry("docker", ["rm", "-f", cid], server)); + } + } + if (dep.imageTag) { + await attempt(`${label} image ${dep.imageTag}`, () => + runner.dockerRunTry("docker", ["rmi", "-f", dep.imageTag], server), + ); + } + } + + for (const dbTarget of info.databaseTargets) { + const label = `database ${dbTarget.containerName ?? dbTarget.proxyContainerName ?? "container"}`; + const server = await requireServer(label, dbTarget.serverId); + if (server === undefined) continue; + if (dbTarget.containerName) { + await attempt(`${label} container`, () => + runner.ensureContainerRemoved(dbTarget.containerName as string, server), + ); + } + if (dbTarget.proxyContainerName) { + await attempt(`${label} proxy`, () => + runner.ensureContainerRemoved(dbTarget.proxyContainerName as string, server), + ); + } + } + + for (const vol of info.databaseVolumes) { + const label = `database volume ${vol.volumeName}`; + const server = await requireServer(label, vol.serverId); + if (server === undefined) continue; + await attempt(label, () => runner.ensureVolumeRemoved(vol.volumeName, server)); + } + + if (info.volumeDockerNames.length > 0) { + const server = await requireServer("project volume", info.projectServerId); + if (server !== undefined) { + for (const name of info.volumeDockerNames) { + await attempt(`project volume ${name}`, () => runner.ensureVolumeRemoved(name, server)); + } + } + } + + return failures; +}; diff --git a/apps/api/src/orchestrator/runtime-logs.ts b/apps/api/src/orchestrator/runtime-logs.ts new file mode 100644 index 0000000..76f31e9 --- /dev/null +++ b/apps/api/src/orchestrator/runtime-logs.ts @@ -0,0 +1,41 @@ +import { getServerById } from "../db/repo"; +import type { Server } from "../types"; +import { dockerRun, getDockerTargetArgs } from "../utils/docker-run"; + +export interface RuntimeLogLine { + sequence: number; + message: string; + timestamp: string; + stage: "runtime"; +} + +interface DeploymentTarget { + id: string; + containerName?: string | null; + serverId?: string | null; +} + +const resolveServer = async (deployment: DeploymentTarget): Promise => + deployment.serverId ? await getServerById(deployment.serverId) : null; + +const containerNameFor = (deployment: DeploymentTarget): string => + deployment.containerName || `deploy-${deployment.id}`; + +export const readRuntimeLogs = async (deployment: DeploymentTarget): Promise => { + const server = await resolveServer(deployment); + const output = await dockerRun("docker", ["logs", "--tail", "200", containerNameFor(deployment)], server); + return output + .split("\n") + .filter(Boolean) + .map((line, i) => ({ + sequence: i + 1, + message: line, + timestamp: new Date().toISOString(), + stage: "runtime" as const, + })); +}; + +export const runtimeLogFollowerArgs = async (deployment: DeploymentTarget): Promise => { + const server = await resolveServer(deployment); + return [...getDockerTargetArgs(server), "logs", "--tail", "100", "--follow", containerNameFor(deployment)]; +}; diff --git a/apps/api/src/scaling/__tests__/engine.test.ts b/apps/api/src/scaling/__tests__/engine.test.ts index 6ddb3c8..4273c98 100644 --- a/apps/api/src/scaling/__tests__/engine.test.ts +++ b/apps/api/src/scaling/__tests__/engine.test.ts @@ -311,6 +311,52 @@ describe("evaluate", () => { expect(mockWriteFilePath).toBeNull(); }); + it("scales down at idle even when cpuThresholdPercent is 0", async () => { + mockGetScalingPolicyResult = () => ({ + ...TEST_POLICY, + cpuThresholdPercent: 0, + minReplicas: 1, + cooldownSeconds: 0, + }); + mockRedisData = { lowCpuSince: String(Date.now() - 310_000), lastScaleDown: "0" }; + mockReadFileContent = "proj-1.localhost:80 {\n reverse_proxy deploy-dep-1:3000 deploy-dep-1-replica-2:3000\n}\n"; + mockTryRunImpl = async () => ""; + mockRunImpl = async (_cmd: string, args: string[]) => { + if (args.includes("stats")) return '{"CPUPerc":"0%","MemUsage":"64MiB / 512MiB"}'; + if (args[0] === "stop") return ""; + if (args[0] === "rm") return ""; + if (args[0] === "ps") return "caddy-abc123\n"; + if (args.includes("reload")) return ""; + return ""; + }; + + await (scalingEngine as any).evaluate(TEST_DEPLOYMENT); + expect(mockWriteFilePath).toBeTruthy(); + expect(mockWriteContent).toContain("deploy-dep-1"); + expect(mockWriteContent).not.toContain("replica-2"); + }); + + it("scales up on any observed CPU when cpuThresholdPercent is 0", async () => { + mockGetScalingPolicyResult = () => ({ ...TEST_POLICY, cpuThresholdPercent: 0, cooldownSeconds: 0 }); + mockRedisData = { highCpuSince: String(Date.now() - 10_000), lastScaleUp: "0" }; + mockReadFileContent = "proj-1.localhost:80 {\n reverse_proxy deploy-dep-1:3000\n}\n"; + + mockRunImpl = async (_cmd: string, args: string[]) => { + if (args.some((a) => a.includes("stats"))) return '{"CPUPerc":"1%","MemUsage":"128MiB / 512MiB"}'; + if (args.some((a) => a.includes("Image"))) return "my-app:latest"; + if (args.some((a) => a.includes("Env"))) return '["PORT=3000"]'; + if (args.some((a) => a.includes("Mounts"))) return "[]"; + if (args[0] === "run") return "new-container-id"; + if (args[0] === "ps") return "caddy-abc123\n"; + if (args.some((a) => a.includes("reload"))) return ""; + return ""; + }; + + await (scalingEngine as any).evaluate(TEST_DEPLOYMENT); + expect(mockWriteFilePath).toBeTruthy(); + expect(mockWriteContent).toContain("deploy-dep-1-replica-2"); + }); + it("skips evaluation when the agent server is offline", async () => { mockGetScalingPolicyResult = () => ({ ...TEST_POLICY, cooldownSeconds: 0 }); mockRedisData = { highCpuSince: String(Date.now() - 10_000), lastScaleUp: "0" }; diff --git a/apps/api/src/servers/prepare.ts b/apps/api/src/servers/prepare.ts index cf5c3da..e9dd28d 100644 --- a/apps/api/src/servers/prepare.ts +++ b/apps/api/src/servers/prepare.ts @@ -50,6 +50,125 @@ if [ -n "$SUDO" ]; then fi log "docker" "Docker ready" +find_container() { + $SUDO docker ps -a --format '{{.Names}}' 2>/dev/null | grep -m1 -x -F -- "$1" || true +} + +find_running_container() { + $SUDO docker ps --format '{{.Names}}' 2>/dev/null | grep -m1 -x -F -- "$1" || true +} + +remove_stopped() { + local label="$1" name="$2" + local all running + all="$($SUDO docker ps -a --format '{{.Names}}' 2>/dev/null || true)" + running="$($SUDO docker ps --format '{{.Names}}' 2>/dev/null || true)" + if grep -qxF "$name" <<< "$all" && ! grep -qxF "$name" <<< "$running"; then + if $SUDO docker rm -f "$name" >/dev/null 2>&1; then + log "$label" "Removed stopped container $name" + fi + fi +} + +equivalent_caddy() { + local name image ports mounts + while IFS='|' read -r name image ports; do + if [ -z "$name" ]; then + continue + fi + case "$name" in + *dequel-caddy*) + echo "$name" + return 0 + ;; + esac + case "$image" in + *caddy*) ;; + *) continue ;; + esac + case "$ports" in + *":80->"*|*":443->"*) ;; + *) continue ;; + esac + mounts="$(docker inspect -f '{{range .Mounts}}{{.Destination}} {{end}}' "$name" 2>/dev/null || true)" + case "$mounts" in + */etc/caddy/routes*) + echo "$name" + return 0 + ;; + esac + done <<< "$(docker ps --format '{{.Names}}|{{.Image}}|{{.Ports}}' 2>/dev/null || true)" + return 0 +} + +equivalent_buildkit() { + local name image + while IFS='|' read -r name image; do + if [ -z "$name" ]; then + continue + fi + case "$name $image" in + *buildkit*) + echo "$name" + return 0 + ;; + esac + done <<< "$(docker ps --format '{{.Names}}|{{.Image}}' 2>/dev/null || true)" + return 0 +} + +equivalent_running() { + case "$1" in + caddy) + equivalent_caddy + ;; + buildkit) + equivalent_buildkit + ;; + esac + return 0 +} + +ensure_container() { + local label="$1" name="$2" + shift 2 + local running existing equivalent err + running="$(find_running_container "$name")" + if [ -n "$running" ]; then + log "$label" "Container already running as $running" + remove_stopped "$label" "$name" + return 0 + fi + existing="$(find_container "$name")" + if [ -n "$existing" ]; then + log "$label" "Starting existing container $existing..." + if $SUDO docker start "$existing" >/dev/null 2>&1; then + log "$label" "Started $existing" + return 0 + fi + remove_stopped "$label" "$name" + log "$label" "Existing container $existing did not start" + fi + log "$label" "Starting Dequel $label container..." + err="" + if err="$($SUDO docker run "$@" 2>&1 >/dev/null)"; then + log "$label" "Started $label container" + return 0 + fi + equivalent="$(equivalent_running "$label")" + if [ -n "$equivalent" ]; then + log "$label" "Launch failed but equivalent container $equivalent is already running - reusing it" + remove_stopped "$label" "$name" + return 0 + fi + remove_stopped "$label" "$name" + if [ -z "$err" ]; then + err="unknown docker error" + fi + echo "[prepare:$label] Failed to start $label container: $err" >&2 + exit 1 +} + install_caddy() { $SUDO mkdir -p /etc/caddy/routes if [ ! -f /etc/caddy/Caddyfile ]; then @@ -60,12 +179,7 @@ install_caddy() { fi $SUDO docker network create dequel_net 2>/dev/null || true - if ! docker ps -a --format '{{.Names}}' 2>/dev/null | grep -q '^dequel-caddy$'; then - log "caddy" "Starting Dequel Caddy container..." - $SUDO docker run -d --restart unless-stopped --name dequel-caddy --network dequel_net -p 80:80 -p 443:443 -v /etc/caddy/Caddyfile:/etc/caddy/Caddyfile -v /etc/caddy/routes:/etc/caddy/routes -v caddy_data:/data caddy:alpine 2>/dev/null || true - else - $SUDO docker start dequel-caddy 2>/dev/null || true - fi + ensure_container caddy dequel-caddy -d --restart unless-stopped --name dequel-caddy --network dequel_net -p 80:80 -p 443:443 -v /etc/caddy/Caddyfile:/etc/caddy/Caddyfile -v /etc/caddy/routes:/etc/caddy/routes -v caddy_data:/data caddy:alpine } install_caddy @@ -83,10 +197,7 @@ else log "railpack" "Railpack install failed - builds will attempt auto-install" fi -if ! docker ps -a --format '{{.Names}}' 2>/dev/null | grep -q '^buildkit$'; then - log "buildkit" "Starting BuildKit container..." - $SUDO docker run -d --restart unless-stopped --name buildkit --privileged moby/buildkit:latest 2>/dev/null || true -fi +ensure_container buildkit buildkit -d --restart unless-stopped --name buildkit --privileged moby/buildkit:latest log "done" "Server preparation complete" `; diff --git a/apps/api/src/tunnels/cert-store.ts b/apps/api/src/tunnels/cert-store.ts new file mode 100644 index 0000000..ee74705 --- /dev/null +++ b/apps/api/src/tunnels/cert-store.ts @@ -0,0 +1,56 @@ +import { X509Certificate } from "node:crypto"; +import { readdir, readFile, stat } from "node:fs/promises"; +import { join } from "node:path"; + +export interface LoadedCert { + cert: string; + key: string; +} + +interface CacheEntry { + mtimeMs: number; + notAfterMs: number; + value: LoadedCert | null; +} + +const cache = new Map(); + +const findCrtPath = async (dataDir: string, host: string): Promise => { + try { + const issuers = await readdir(join(dataDir, "certificates")); + for (const issuer of issuers) { + const crtPath = join(dataDir, "certificates", issuer, host, `${host}.crt`); + const st = await stat(crtPath).catch(() => null); + if (st?.isFile()) return crtPath; + } + } catch { + return null; + } + return null; +}; + +export const loadCaddyCert = async (dataDir: string, host: string, now = Date.now()): Promise => { + const crtPath = await findCrtPath(dataDir, host); + if (!crtPath) { + cache.delete(host); + return null; + } + const st = await stat(crtPath); + const cached = cache.get(host); + if (cached && cached.mtimeMs === st.mtimeMs && (cached.value === null || cached.notAfterMs > now + 60_000)) { + return cached.value; + } + try { + const keyPath = crtPath.replace(/\.crt$/, ".key"); + const [cert, key] = await Promise.all([readFile(crtPath, "utf8"), readFile(keyPath, "utf8")]); + const x509 = new X509Certificate(cert); + const notAfterMs = Date.parse(x509.validTo); + const valid = x509.checkHost(host) && notAfterMs > now + 60_000; + const value = valid ? { cert, key } : null; + cache.set(host, { mtimeMs: st.mtimeMs, notAfterMs, value }); + return value; + } catch { + cache.delete(host); + return null; + } +}; diff --git a/apps/api/src/tunnels/gateway.ts b/apps/api/src/tunnels/gateway.ts index f25ee70..bf19e1c 100644 --- a/apps/api/src/tunnels/gateway.ts +++ b/apps/api/src/tunnels/gateway.ts @@ -5,6 +5,7 @@ import { Pool } from "pg"; import { config } from "../utils/config"; import { decryptValue } from "../utils/crypto"; import { type GatewayCert, loadOrCreateGatewayCert } from "./cert"; +import { loadCaddyCert } from "./cert-store"; import { parseClientHello, parsePostgresPreamble, readClientHello } from "./peek"; import { type GatewayRoute, type RouteRow, resolveGatewayRoute } from "./routes"; import { forwardViaSsh, type SshEngineTarget } from "./ssh-tunnel"; @@ -15,6 +16,7 @@ export interface GatewayOptions { caddyHost: string; caddyPort: number; cert: GatewayCert; + caddyDataDir?: string; lookup: (internalHost: string) => Promise; } @@ -45,12 +47,15 @@ const forwardToCaddy = (client: Socket, acc: Buffer, caddyHost: string, caddyPor link([client, upstream]); }; -const terminateForEngine = ( +const terminateForEngine = async ( client: Socket, acc: Buffer, - cert: GatewayCert, route: Extract, + sni: string | null, + options: GatewayOptions, ) => { + const caddyCert = options.caddyDataDir && sni ? await loadCaddyCert(options.caddyDataDir, sni) : null; + const cert = caddyCert ?? options.cert; const tlsOptions: ServerOptions = { key: cert.key, cert: cert.cert }; const terminator = createTlsServer(tlsOptions, (plain) => { if (route.ssh) { @@ -102,7 +107,7 @@ export const startGateway = async (options: GatewayOptions): Promise { const snippet = ingressSite("app.example.com", "203.0.113.10"); expect(snippet).toContain("reverse_proxy 203.0.113.10:80"); expect(snippet).not.toContain("header_up Host"); + expect(snippet).toContain("log {"); + expect(snippet).toContain("output stdout"); }); }); diff --git a/apps/api/src/utils/compose-ingress.ts b/apps/api/src/utils/compose-ingress.ts index 001f169..e4be476 100644 --- a/apps/api/src/utils/compose-ingress.ts +++ b/apps/api/src/utils/compose-ingress.ts @@ -8,6 +8,14 @@ export interface ComposeIngressHostname { export const DB_SERVICE_NAMES = new Set(["db", "postgres", "mysql", "redis", "mongo", "database"]); +const DB_NAME_PATTERN = + /(^|[-_])(db|dbs|postgres|postgresql|mysql|mariadb|mongo|mongodb|redis|valkey|database|elasticsearch|opensearch)($|[-_0-9])/i; + +export const isDbServiceName = (name: string): boolean => { + const lower = name.toLowerCase(); + return DB_SERVICE_NAMES.has(lower) || lower.endsWith("db") || DB_NAME_PATTERN.test(lower); +}; + export const computeComposeIngressHostnames = ( webServices: { name: string; port: number }[], primaryServiceName: string, @@ -29,7 +37,7 @@ export const computeComposeIngressHostnames = ( if (svc.name === primaryServiceName) continue; const customMatch = customMappings.find((c) => c.serviceName === svc.name); - if (!customMatch && DB_SERVICE_NAMES.has(svc.name)) continue; + if (isDbServiceName(svc.name)) continue; const subdomainPrefix = customMatch?.subdomain?.trim() || svc.name; diff --git a/apps/api/src/utils/config-loader.ts b/apps/api/src/utils/config-loader.ts index 02a81a6..d9463f6 100644 --- a/apps/api/src/utils/config-loader.ts +++ b/apps/api/src/utils/config-loader.ts @@ -5,24 +5,24 @@ import { resolve } from "node:path"; const XDG_CONFIG_HOME = process.env.XDG_CONFIG_HOME || `${homedir()}/.config`; export interface FileConfig { - port?: number; - databaseUrl?: string; - workspaceRoot?: string; - caddyRoutesDir?: string; - caddyBaseDomain?: string; - dockerNetwork?: string; - appInternalPort?: number; - buildkitHost?: string; - envEncryptionKey?: string; - redisUrl?: string; - queueConcurrency?: number; - queueRetryMax?: number; - queueRetryBaseMs?: number; - alertEvalIntervalMs?: number; - githubClientId?: string; - githubClientSecret?: string; - githubAppName?: string; - githubWebhookSecret?: string; + CADDY_BASE_DOMAIN?: string; + CONTROL_PLANE_URL?: string; + AGENT_TUNNEL_URL?: string; + QUEUE_CONCURRENCY?: number; + QUEUE_RETRY_MAX?: number; + QUEUE_RETRY_BASE_MS?: number; + ALERT_EVAL_INTERVAL_MS?: number; + FAILURE_SWEEP_INTERVAL_MS?: number; + GRAFANA_URL?: string; + GRAFANA_USER?: string; + GRAFANA_PASS?: string; + WIREGUARD_SERVER_CONTAINER?: string; + WIREGUARD_SERVER_PUBLIC_KEY?: string; + WIREGUARD_SERVER_ENDPOINT?: string; + WIREGUARD_SERVER_IP?: string; + WIREGUARD_PEER_CIDR?: string; + FAILOVER_DISABLED?: string; + FAILOVER_MIN_INTERVAL_MS?: number; } const searchPaths = (): string[] => { diff --git a/apps/api/src/utils/config.ts b/apps/api/src/utils/config.ts index b54ddc0..5d94a19 100644 --- a/apps/api/src/utils/config.ts +++ b/apps/api/src/utils/config.ts @@ -1,4 +1,5 @@ import { loadFileConfig } from "./config-loader"; +import { DEFAULT_ENV_ENCRYPTION_KEY } from "./crypto"; const fileConfig = loadFileConfig(); @@ -12,35 +13,53 @@ const withFile = (key: string, envDefault: string, transform?: (v: string) => return transform ? transform(envDefault) : (envDefault as unknown as T); }; +const withEnv = (key: string, envDefault: string, transform?: (v: string) => T): T => { + const envVal = process.env[key]; + if (envVal !== undefined) { + return transform ? transform(envVal) : (envVal as unknown as T); + } + return transform ? transform(envDefault) : (envDefault as unknown as T); +}; + +const isProductionEnv = process.env.NODE_ENV === "production"; +const envEncryptionKey = withEnv("ENV_ENCRYPTION_KEY", DEFAULT_ENV_ENCRYPTION_KEY); +if (isProductionEnv && envEncryptionKey === DEFAULT_ENV_ENCRYPTION_KEY) { + throw new Error( + "ENV_ENCRYPTION_KEY must be set to a unique value in production; refusing to fall back to the default development key", + ); +} +const databaseUrl = withEnv("DATABASE_URL", ""); +if (isProductionEnv && !databaseUrl) { + throw new Error("DATABASE_URL must be set in production"); +} + const SYSTEM = { dockerNetwork: "dequel_net", buildkitHost: "tcp://buildkit:1234", redisUrl: "redis://redis:6379", dequelSourceRepoUrl: "https://github.com/Lftobs/dequel.git", + workspaceRoot: "/app/workspace", + caddyRoutesDir: "/caddy/routes", + caddyDataDir: "/caddy-data/caddy", + port: 3001, + appInternalPort: 3000, + gatewayCertDir: "/app/data", + dequelSlackWebhookUrl: "", + dequelSlackChannel: "", } as const; export const config = { ...SYSTEM, - databaseUrl: withFile("DATABASE_URL", ""), - workspaceRoot: withFile("WORKSPACE_ROOT", "/app/workspace"), - caddyRoutesDir: withFile("CADDY_ROUTES_DIR", "/caddy/routes"), - port: withFile("PORT", "17474", Number), + databaseUrl, caddyBaseDomain: withFile("CADDY_BASE_DOMAIN", "localhost"), controlPlaneUrl: withFile("CONTROL_PLANE_URL", ""), agentTunnelUrl: withFile("AGENT_TUNNEL_URL", ""), - appInternalPort: withFile("APP_INTERNAL_PORT", "17476", Number), - envEncryptionKey: withFile("ENV_ENCRYPTION_KEY", "dev-env-key-change-me"), - dequelSlackWebhookUrl: withFile("DEQUEL_SLACK_WEBHOOK_URL", ""), - dequelSlackChannel: withFile("DEQUEL_SLACK_CHANNEL", ""), + envEncryptionKey, queueConcurrency: withFile("QUEUE_CONCURRENCY", "3", Number), queueRetryMax: withFile("QUEUE_RETRY_MAX", "5", Number), queueRetryBaseMs: withFile("QUEUE_RETRY_BASE_MS", "5000", Number), alertEvalIntervalMs: withFile("ALERT_EVAL_INTERVAL_MS", "60000", Number), failureSweepIntervalMs: withFile("FAILURE_SWEEP_INTERVAL_MS", "60000", Number), - githubClientId: withFile("GITHUB_CLIENT_ID", ""), - githubClientSecret: withFile("GITHUB_CLIENT_SECRET", ""), - githubAppName: withFile("GITHUB_APP_NAME", "Dequel"), - githubWebhookSecret: withFile("GITHUB_WEBHOOK_SECRET", ""), grafanaUrl: withFile("GRAFANA_URL", "http://grafana:3000"), grafanaUser: withFile("GRAFANA_USER", "admin"), grafanaPass: withFile("GRAFANA_PASS", "admin"), @@ -49,5 +68,6 @@ export const config = { wireguardServerEndpoint: withFile("WIREGUARD_SERVER_ENDPOINT", ""), wireguardServerIp: withFile("WIREGUARD_SERVER_IP", "10.200.0.1"), wireguardPeerCidr: withFile("WIREGUARD_PEER_CIDR", "10.200.0.0/24"), - gatewayCertDir: withFile("GATEWAY_CERT_DIR", "/app/data"), + failoverDisabled: withFile("FAILOVER_DISABLED", ""), + failoverMinIntervalMs: withFile("FAILOVER_MIN_INTERVAL_MS", "600000", Number), }; diff --git a/apps/api/src/utils/crypto.ts b/apps/api/src/utils/crypto.ts index af77208..0cafe4b 100644 --- a/apps/api/src/utils/crypto.ts +++ b/apps/api/src/utils/crypto.ts @@ -3,6 +3,8 @@ import { createCipheriv, createDecipheriv, randomBytes, scryptSync } from "node: const ALGO = "aes-256-gcm"; const IV_LEN = 12; +export const DEFAULT_ENV_ENCRYPTION_KEY = "dev-env-key-change-me"; + export const deriveKey = (secret: string) => scryptSync(secret, "dequel-env", 32); export const encryptValue = (value: string, secret: string) => { diff --git a/apps/api/src/utils/db-caddy-route.ts b/apps/api/src/utils/db-caddy-route.ts new file mode 100644 index 0000000..93fde03 --- /dev/null +++ b/apps/api/src/utils/db-caddy-route.ts @@ -0,0 +1,68 @@ +import { existsSync, mkdirSync, readFileSync, unlinkSync, writeFileSync } from "node:fs"; +import { dirname, join } from "node:path"; +import { reloadCaddy } from "../orchestrator/runtime"; +import { config } from "./config"; + +export interface DbRouteRecord { + internalHost: string; + type: string; + publicAccess: boolean; +} + +export interface DbRouteOpts { + routesDir?: string; + reloadFn?: () => Promise; + baseDomain?: string; +} + +const gatewayBaseDomain = (opts?: DbRouteOpts): string => opts?.baseDomain ?? config.caddyBaseDomain; + +export const dbGatewayEligible = (db: DbRouteRecord, opts?: DbRouteOpts): boolean => { + const base = gatewayBaseDomain(opts); + return db.publicAccess && db.type !== "mysql" && Boolean(base) && base !== "localhost"; +}; + +export const dbGatewayHost = (db: DbRouteRecord, opts?: DbRouteOpts): string => + `${db.internalHost}.${gatewayBaseDomain(opts)}`; + +const routePath = (routesDir: string, internalHost: string): string => + join(routesDir, `database-${internalHost}.caddy`); + +const writeAndReload = async (filePath: string, content: string, reloadFn: () => Promise) => { + mkdirSync(dirname(filePath), { recursive: true }); + if (existsSync(filePath) && readFileSync(filePath, "utf8") === content) return; + writeFileSync(filePath, content, "utf8"); + await reloadFn(); +}; + +export const syncDbGatewayRoute = async (db: DbRouteRecord, opts?: DbRouteOpts): Promise => { + const routesDir = opts?.routesDir ?? config.caddyRoutesDir; + const reloadFn = opts?.reloadFn ?? reloadCaddy; + const filePath = routePath(routesDir, db.internalHost); + try { + if (!dbGatewayEligible(db, opts)) { + if (existsSync(filePath)) { + unlinkSync(filePath); + await reloadFn(); + } + return; + } + await writeAndReload(filePath, `${dbGatewayHost(db, opts)} {\n\trespond 404\n}\n`, reloadFn); + } catch (e) { + console.warn(`Could not sync Caddy route for database ${db.internalHost}:`, e instanceof Error ? e.message : e); + } +}; + +export const removeDbGatewayRoute = async (db: DbRouteRecord, opts?: DbRouteOpts): Promise => { + const routesDir = opts?.routesDir ?? config.caddyRoutesDir; + const reloadFn = opts?.reloadFn ?? reloadCaddy; + const filePath = routePath(routesDir, db.internalHost); + try { + if (existsSync(filePath)) { + unlinkSync(filePath); + await reloadFn(); + } + } catch (e) { + console.warn(`Could not remove Caddy route for database ${db.internalHost}:`, e instanceof Error ? e.message : e); + } +}; diff --git a/apps/api/src/utils/ingress.ts b/apps/api/src/utils/ingress.ts index cdb3936..22e9b0a 100644 --- a/apps/api/src/utils/ingress.ts +++ b/apps/api/src/utils/ingress.ts @@ -1,7 +1,7 @@ import { rm, writeFile } from "node:fs/promises"; import { join } from "node:path"; import { createAgentJob, getPlatformSettings, getServerById, upsertRoute } from "../db/repo"; -import { caddyReverseProxy, caddySite } from "./caddy-site"; +import { CADDY_ACCESS_LOG_BLOCK, caddyReverseProxy, caddySite } from "./caddy-site"; import { config } from "./config"; import { removeRemoteCaddyRoute, syncRemoteCaddyRoute } from "./ssh"; @@ -47,7 +47,7 @@ export const projectServerSite = ( }; export const ingressSite = (hostname: string, upstreamHost: string): string => - `${hostname} {\n reverse_proxy ${upstreamHost}:80\n}\n`; + `${hostname} {\n${CADDY_ACCESS_LOG_BLOCK}\n reverse_proxy ${upstreamHost}:80\n}\n`; export const syncIngressRoute = async ( ingressServer: { id: string; mode: string }, diff --git a/apps/api/src/utils/permissions.ts b/apps/api/src/utils/permissions.ts index 4dcb249..8ffd6d4 100644 --- a/apps/api/src/utils/permissions.ts +++ b/apps/api/src/utils/permissions.ts @@ -23,3 +23,16 @@ export const hasPermission = (permissions: Permission[], requiredAction: string, return false; }); }; + +const READ_METHODS = new Set(["GET", "HEAD"]); + +const isReadMethod = (method: string): boolean => READ_METHODS.has(method.toUpperCase()); + +export const requiredPermissionForMethod = (method: string): string => + isReadMethod(method) ? "deploy:read" : "deploy:write"; + +export const canPerformMethod = (permissions: Permission[], method: string): boolean => { + const canWrite = hasPermission(permissions, "deploy:write"); + if (isReadMethod(method)) return canWrite || hasPermission(permissions, "deploy:read"); + return canWrite; +}; diff --git a/apps/api/src/utils/proc-group.ts b/apps/api/src/utils/proc-group.ts new file mode 100644 index 0000000..f9f2c90 --- /dev/null +++ b/apps/api/src/utils/proc-group.ts @@ -0,0 +1,12 @@ +import type { ChildProcess } from "node:child_process"; + +export const killChildTree = (child: ChildProcess, signal: NodeJS.Signals = "SIGTERM"): void => { + if (child.pid == null) return; + try { + process.kill(-child.pid, signal); + } catch { + try { + child.kill(signal); + } catch {} + } +}; diff --git a/apps/api/src/utils/process-exec.ts b/apps/api/src/utils/process-exec.ts index 970ff71..e9a634c 100644 --- a/apps/api/src/utils/process-exec.ts +++ b/apps/api/src/utils/process-exec.ts @@ -63,33 +63,6 @@ export function terminateWithEscalation(child: ChildProcess, escalationGraceMs = }; } -let reaperInitialized = false; - -export function initZombieReaper(): void { - if (reaperInitialized) return; - reaperInitialized = true; - - if (process.platform !== "linux") return; - - try { - const { dlopen, FFIType } = require("bun:ffi"); - const libc = dlopen("libc.so.6", { - waitpid: { - args: [FFIType.i32, FFIType.ptr, FFIType.i32], - returns: FFIType.i32, - }, - }); - - const reap = () => { - try { - while (libc.symbols.waitpid(-1, null, 1) > 0) {} - } catch {} - }; - - process.on("SIGCHLD", reap); - } catch {} -} - export function safeSpawn(cmd: string, args: string[], options: SafeSpawnOptions = {}): Promise { return new Promise((resolve, reject) => { const isPosix = process.platform !== "win32"; diff --git a/apps/api/src/utils/server-default.ts b/apps/api/src/utils/server-default.ts index 610aca3..9ab23ba 100644 --- a/apps/api/src/utils/server-default.ts +++ b/apps/api/src/utils/server-default.ts @@ -1,6 +1,8 @@ -import { ensureLocalServer, getServerById, listProjects, listServers } from "../db/repo"; +import type { listServers } from "../db/repo"; import { isServerPreparing } from "../servers/prepare"; +const repo = () => import("../db/repo"); + const isServerHealthy = (s: Awaited>[number]): boolean => { if (isServerPreparing(s.id)) return false; if (s.mode === "agent") { @@ -18,11 +20,13 @@ export const pickBestServer = async ( modeFilter?: string[], ): Promise => { if (preferredServerId && preferredServerId !== excludeServerId) { + const { getServerById } = await repo(); const preferred = await getServerById(preferredServerId); if (preferred && (preferred.mode === "local" || isServerHealthy(preferred))) { if (!modeFilter || modeFilter.includes(preferred.mode)) return preferredServerId; } } + const { ensureLocalServer, listProjects, listServers } = await repo(); const servers = await listServers(); const projects = await listProjects(); const projectCount = new Map(); diff --git a/apps/api/src/utils/ssh.ts b/apps/api/src/utils/ssh.ts index c8c3f67..913d09e 100644 --- a/apps/api/src/utils/ssh.ts +++ b/apps/api/src/utils/ssh.ts @@ -13,6 +13,13 @@ export interface SshExecutionOptions { const SSH_KEYS_DIR = join(tmpdir(), "dequel_ssh_keys"); +export class RemoteScriptAbortedError extends Error { + constructor() { + super("Remote script aborted"); + this.name = "RemoteScriptAbortedError"; + } +} + export const ensureSshKey = (server: { host: string; port?: number; @@ -281,6 +288,10 @@ export const runRemoteScript = ( options: { onLog?: (line: string) => Promise | void; signal?: AbortSignal } = {}, ): Promise<{ code: number; stdout: string; stderr: string }> => { return new Promise((resolve, reject) => { + if (options.signal?.aborted) { + reject(new RemoteScriptAbortedError()); + return; + } const keyPath = ensureSshKey(server); const keyArgs = keyPath ? ["-i", keyPath, "-o", "IdentitiesOnly=yes"] : []; const user = server.sshUser || "root"; @@ -319,27 +330,40 @@ export const runRemoteScript = ( terminationHandle = terminateWithEscalation(child, 3000); }; + const makeLineEmitter = () => { + let pending = ""; + return { + write(text: string) { + pending += text; + const parts = pending.split("\n"); + pending = parts.pop() ?? ""; + for (const line of parts) { + if (line) options.onLog?.(line); + } + }, + flush() { + if (pending) options.onLog?.(pending); + pending = ""; + }, + }; + }; + + const emitStdout = options.onLog ? makeLineEmitter() : null; + const emitStderr = options.onLog ? makeLineEmitter() : null; + child.stdout?.on("data", (chunk) => { const text = String(chunk); stdout += text; - if (options.onLog) { - text - .split("\n") - .filter(Boolean) - .forEach((line) => options.onLog!(line)); - } + emitStdout?.write(text); }); child.stderr?.on("data", (chunk) => { const text = String(chunk); stderr += text; - if (options.onLog) { - text - .split("\n") - .filter(Boolean) - .forEach((line) => options.onLog!(line)); - } + emitStderr?.write(text); }); child.on("close", (code) => { + emitStdout?.flush(); + emitStderr?.flush(); if (terminationHandle) terminationHandle.cancel(); resolve({ code: code ?? 1, stdout: stdout.trim(), stderr: stderr.trim() }); }); @@ -347,16 +371,16 @@ export const runRemoteScript = ( terminate(); reject(err); }); + child.stdin?.on("error", () => {}); if (options.signal) { - if (options.signal.aborted) { - terminate(); - reject(new Error("Remote build script aborted")); - } else { - options.signal.addEventListener("abort", () => { - terminate(); - reject(new Error("Remote build script aborted")); - }); - } + options.signal.addEventListener( + "abort", + () => { + child.kill("SIGTERM"); + reject(new RemoteScriptAbortedError()); + }, + { once: true }, + ); } child.stdin.write(script); child.stdin.end(); diff --git a/apps/api/src/utils/tcp-probe.ts b/apps/api/src/utils/tcp-probe.ts new file mode 100644 index 0000000..189285e --- /dev/null +++ b/apps/api/src/utils/tcp-probe.ts @@ -0,0 +1,16 @@ +import { connect } from "node:net"; + +export const probeTcp = (host: string, port: number, timeoutMs = 2000): Promise => + new Promise((resolve) => { + const socket = connect({ host, port }); + let settled = false; + const finish = (ok: boolean) => { + if (settled) return; + settled = true; + socket.destroy(); + resolve(ok); + }; + socket.setTimeout(timeoutMs, () => finish(false)); + socket.once("connect", () => finish(true)); + socket.once("error", () => finish(false)); + }); diff --git a/apps/docs/blume.config.ts b/apps/docs/blume.config.ts index 1ef703d..c70d4ea 100644 --- a/apps/docs/blume.config.ts +++ b/apps/docs/blume.config.ts @@ -33,22 +33,45 @@ export default defineConfig({ "/docs", "/docs/installation", "/docs/quickstart", - "/docs/configuration", { - label: "Core Architecture", - items: ["/docs/deployments", "/docs/env-vars", "/docs/scaling", "/docs/system-config"], + label: "Projects", + items: [ + "/docs/deployments", + "/docs/configuration", + "/docs/env-vars", + "/docs/shared-env-vars", + "/docs/volumes", + "/docs/domains", + "/docs/scaling", + ], }, { - label: "Deployment", - items: ["/docs/ai-diagnosis", "/docs/agent-caddy-routes"], + label: "Data", + items: ["/docs/databases", "/docs/backups"], }, { - label: "Cluster Storage & Data", - items: ["/docs/databases", "/docs/volumes"], + label: "Multi-Server", + items: ["/docs/servers", "/docs/agent-caddy-routes"], }, { - label: "Networking & Security", - items: ["/docs/auth", "/docs/domains", "/docs/ssl"], + label: "AI & Diagnosis", + items: ["/docs/ai-diagnosis"], + }, + { + label: "Observability", + items: ["/docs/observability", "/docs/alerts"], + }, + { + label: "Integrations", + items: ["/docs/github", "/docs/settings"], + }, + { + label: "Security", + items: ["/docs/auth", "/docs/keys", "/docs/ssl"], + }, + { + label: "Platform", + items: ["/docs/system-config", "/docs/cli"], }, { label: "Release", diff --git a/apps/docs/components/CTA.astro b/apps/docs/components/CTA.astro index 1d1fcc6..3323bda 100644 --- a/apps/docs/components/CTA.astro +++ b/apps/docs/components/CTA.astro @@ -1,11 +1,11 @@ --- --- -
-
+
+
-
- +
+ @@ -18,25 +18,25 @@
-

+

Ready to own your deployment platform?

-

+

Spin up Dequel on your server in seconds with zero third-party dependencies.

-
+
curl -fsSL https://dequel.intrep.xyz/install.sh | bash - 1-Line Install +
-
- +
+ Install Dequel Now → - + Read Documentation
diff --git a/apps/docs/components/Comparison.astro b/apps/docs/components/Comparison.astro index e90d149..ef545ed 100644 --- a/apps/docs/components/Comparison.astro +++ b/apps/docs/components/Comparison.astro @@ -1,77 +1,77 @@ --- --- -
-
+
+
PaaS Paradigm Shift
-

+

Stop fighting infrastructure.
Start shipping instantly.

-

+

Dequel replaces fragmented DevOps tools with a unified self-hosted platform.

-
+
-
+
-
+
TRADITIONAL DEVOPS
-

+

More tools. More upkeep.

-
    -
  • +
      +
    • — Maintain scripts, manifests, and ingress rules
      • -
    • +
    • — Renew certificates and route domains by hand
      • -
    • +
    • — Stitch together logs, metrics, and alerts
-
+
High overhead · Fragmented stack
-
+
-
+
WITH DEQUEL
-

+

One command. One platform.

-
    -
  • +
      +
    • ✓ Deploy from Git, ZIP, or Docker Compose
      • -
    • +
    • ✓ Automate builds, SSL, routing, and scaling
      • -
    • +
    • ✓ Keep complete control on your own VPS
-
+
Zero lock-in · 100% owned by you
diff --git a/apps/docs/components/Features.astro b/apps/docs/components/Features.astro index fff052c..b298327 100644 --- a/apps/docs/components/Features.astro +++ b/apps/docs/components/Features.astro @@ -88,20 +88,20 @@ scaling_policy: ]; --- -
-
+
+
Platform Capabilities
-

+

Everything you need to ship.
Powered by your infrastructure.

-

+

Explore Dequel's core PaaS engines built into a single lightweight control plane.

-
+
{tabs.map((tab, idx) => ( ))} @@ -123,24 +123,24 @@ scaling_policy: id={`tab-panel-${tab.id}`} class={`tab-panel ${idx === 0 ? 'block' : 'hidden'} transition-all duration-300`} > -
+
-
+
PaaS Module: {tab.id}
-

+

{tab.title}

-

+

{tab.description}

-
    +
      {tab.highlights.map((item) => (
    • - ✓ + ✓ {item}
      • ))} @@ -155,13 +155,13 @@ scaling_policy:
-
-
-
- - {tab.label} Preview +
+
+
+ + {tab.label} Preview
-
+
Dequel v1.0
- 
{tab.codePreview}
+ 
{tab.codePreview}
Status: Active Zero Config Required diff --git a/apps/docs/components/Hero.astro b/apps/docs/components/Hero.astro index 84ad598..2d66199 100644 --- a/apps/docs/components/Hero.astro +++ b/apps/docs/components/Hero.astro @@ -12,25 +12,25 @@ const { } = Astro.props; --- -
+
-

+

{titleNormal}. {titleItalic}

-

+

{subText}

-
- +
+ Get Started in 1 Command → @@ -38,10 +38,10 @@ const { href="https://github.com/Lftobs/dequel" target="_blank" rel="noopener noreferrer" - class="cf-pill-btn-glass flex items-center gap-2" + class="cf-pill-btn-glass flex items-center justify-center gap-2 w-full sm:w-auto" >
- $ - curl -fsSL https://dequel.intrep.xyz/install.sh | bash + $ + curl -fsSL https://dequel.intrep.xyz/install.sh | bash - +
diff --git a/apps/docs/components/HowItWorks.astro b/apps/docs/components/HowItWorks.astro index 3df7847..bd47872 100644 --- a/apps/docs/components/HowItWorks.astro +++ b/apps/docs/components/HowItWorks.astro @@ -2,31 +2,31 @@ import ArchCanvas from "./architecture/ArchCanvas.astro"; --- -
+
-
+
Engine Architecture
-

+

Your own PaaS, running on your own servers.

-

+

Dequel orchestrates raw Docker containers, Buildkit compilation, Redis job queues, PostgreSQL persistence, and dynamic Caddy routing into one unified platform.

-
+
-
+
- + DEQUEL ENGINE TOPOLOGY - • - dequel_net bridge + +
+ ← Swipe diagram →
-
+
-
+
-
+
STAGE 01: INGESTION • Zero DevOps overhead
-

+

01. Multi-Channel Trigger Ingestion

@@ -108,8 +109,9 @@ import ArchCanvas from "./architecture/ArchCanvas.astro"; .dq-step-pill { display: inline-flex; align-items: center; - gap: 0.5rem; - padding: 0.45rem 0.95rem; + justify-content: center; + gap: 0.45rem; + padding: 0.5rem 0.85rem; border-radius: 9999px; background: #111115; border: 1px solid #222228; @@ -117,6 +119,7 @@ import ArchCanvas from "./architecture/ArchCanvas.astro"; cursor: pointer; transition: all 180ms ease; user-select: none; + white-space: nowrap; } .dq-step-pill:hover { border-color: #3f3f46; @@ -129,6 +132,7 @@ import ArchCanvas from "./architecture/ArchCanvas.astro"; border-radius: 50%; background-color: #52525b; transition: all 180ms ease; + flex-shrink: 0; } .dq-step-pill .step-num { font-family: 'JetBrains Mono', monospace; @@ -137,7 +141,7 @@ import ArchCanvas from "./architecture/ArchCanvas.astro"; } .dq-step-pill .step-label { font-family: 'Space Grotesk', sans-serif; - font-size: 12.5px; + font-size: 12px; font-weight: 600; } .dq-step-pill.active { diff --git a/apps/docs/components/Stats.astro b/apps/docs/components/Stats.astro index 448df86..aafcc11 100644 --- a/apps/docs/components/Stats.astro +++ b/apps/docs/components/Stats.astro @@ -7,19 +7,19 @@ const stats = [ ]; --- -

+
-
+
{stats.map((stat) => ( -
-
+
+
{stat.value}
-
+
{stat.label}
-
+
{stat.sub}
diff --git a/apps/docs/components/TerminalDemo.astro b/apps/docs/components/TerminalDemo.astro index 14085d3..e8e9bf3 100644 --- a/apps/docs/components/TerminalDemo.astro +++ b/apps/docs/components/TerminalDemo.astro @@ -1,11 +1,11 @@ --- --- -
-
-
+
+
+
Developer Experience
-

+

From git push to production
in under 4 seconds.

@@ -25,7 +25,7 @@
-
+
@@ -35,7 +35,7 @@ dequel cli v1.0.0 bash
-
+
$ diff --git a/apps/docs/components/architecture/ArchCanvas.astro b/apps/docs/components/architecture/ArchCanvas.astro index db6c58c..3debce3 100644 --- a/apps/docs/components/architecture/ArchCanvas.astro +++ b/apps/docs/components/architecture/ArchCanvas.astro @@ -1,11 +1,11 @@ --- --- -
+
diff --git a/apps/docs/components/architecture/archController.ts b/apps/docs/components/architecture/archController.ts index 015e0ac..9f602af 100644 --- a/apps/docs/components/architecture/archController.ts +++ b/apps/docs/components/architecture/archController.ts @@ -75,6 +75,15 @@ export function initArchController() { chipV4.setAttribute("stroke", "#ea580c"); } } + + // 6. Auto-pan horizontal scroll container on mobile viewports + const scrollEl = document.getElementById("dq-arch-scroll"); + if (scrollEl && scrollEl.scrollWidth > scrollEl.clientWidth) { + const ratios = [0, 0.25, 0.48, 0.72, 1.0, 0.48]; + const ratio = ratios[stepIndex - 1] ?? 0; + const maxScroll = scrollEl.scrollWidth - scrollEl.clientWidth; + scrollEl.scrollTo({ left: maxScroll * ratio, behavior: "smooth" }); + } } function startTimer() { diff --git a/apps/docs/docs/agent-caddy-routes.md b/apps/docs/docs/agent-caddy-routes.md index 2b215c0..42c2331 100644 --- a/apps/docs/docs/agent-caddy-routes.md +++ b/apps/docs/docs/agent-caddy-routes.md @@ -5,11 +5,15 @@ description: How the agent container manages Caddy route files for deployed appl slug: agent-caddy-routes --- -The `CADDY_ROUTES_DIR` environment variable controls where the API writes Caddy route files. On agent servers, the default is `/etc/caddy/routes`. Each deployed application gets its own `.caddy` file in this directory. +The API writes Caddy route files for local deployments to the fixed +`/caddy/routes` directory. Each deployed application gets its own `.caddy` file +in this directory. ## Route Directory -The route directory path is resolved from the `CADDY_ROUTES_DIR` environment variable. In local mode (API and Caddy on the same host), this defaults to `/caddy/routes`. On remote agent servers, the agent writes to `/etc/caddy/routes`. +The local route directory is fixed at `/caddy/routes` (the API creates it on +startup). On remote servers, routes are written to `/etc/caddy/routes` — over +SSH for SSH-mode servers, or by the agent for agent-mode servers. The directory must exist and be writable by the process managing Caddy. The API creates it automatically on startup. @@ -39,7 +43,7 @@ my-app.localhost:80 { output stdout format json } - reverse_proxy my-app-abc12345:17476 { + reverse_proxy my-app-abc12345:3000 { header_up Host {upstream_hostport} } } @@ -47,7 +51,7 @@ my-app.localhost:80 { - **Hostname:** `.` (e.g., `my-app.localhost:80` or `my-app.example.com`) - **Container name:** `-` (e.g., `my-app-abc12345`) -- **Port:** The application's internal port (default `17476`, or `PORT` env var value) +- **Port:** The application's internal port (the platform default is `3000`, or the project's configured port) For custom domains, multiple hostnames are comma-separated on the first line: @@ -57,7 +61,7 @@ my-app.localhost:80, blog.example.com:80 { output stdout format json } - reverse_proxy my-app-abc12345:17476 { + reverse_proxy my-app-abc12345:3000 { header_up Host {upstream_hostport} } } @@ -70,11 +74,11 @@ my-app.localhost:80, blog.example.com:80 { When a deployment completes successfully: 1. The API builds a Caddy snippet using `buildCaddySnippet()` from `apps/api/src/utils/domain-verifier.ts`. -2. The snippet is written to `/.caddy`. +2. The snippet is written to `/caddy/routes/.caddy`. 3. Caddy is reloaded via `caddy reload --config /etc/caddy/Caddyfile`. 4. A route record is persisted to the database with status `active`. -For remote agent servers, the control plane sends a `reload_routes` job to the agent. The agent writes the file and reloads Caddy on the remote host. +For remote agent servers, the control plane sends a `reload_routes` job to the agent. The agent writes the file and reloads Caddy on the remote host. For SSH-mode servers, the control plane writes the file and reloads Caddy over SSH instead. ### Rollback @@ -107,7 +111,7 @@ Reload is triggered after every route file change (deploy, rollback, delete, cus ### Route files not appearing -1. Check that `CADDY_ROUTES_DIR` is set correctly and the directory exists. +1. Check that the routes directory exists (`/caddy/routes` locally, `/etc/caddy/routes` on remote servers). 2. Verify the API process has write permissions to the directory. 3. For agent servers, ensure the agent is running and reachable (check heartbeat status in the dashboard). diff --git a/apps/docs/docs/alerts.md b/apps/docs/docs/alerts.md new file mode 100644 index 0000000..94db969 --- /dev/null +++ b/apps/docs/docs/alerts.md @@ -0,0 +1,39 @@ +--- +title: Alerts +category: Observability +description: CPU, memory, and downtime alert rules with email, Slack, and webhook channels. +slug: alerts +--- + +The project's **Alerts** tab defines rules evaluated continuously in +the background. When a rule fires, Dequel notifies the channel you +chose — and failure emails link straight back to the logs. + +## Rule types + +- **CPU** — fires when average CPU % exceeds your threshold (default + 80%) for the window. +- **Memory** — same for memory % (default 85%). +- **Downtime** — fires immediately when the deployment stops responding. + +Create a rule with a metric, threshold (1–100), and channel. Toggle +rules on/off without deleting them. + +## Channels + +- **Email** — delivered through your [SMTP settings](/docs/settings). + Failure notifications include a link to the relevant logs. +- **Slack** — posts to an incoming webhook URL (Slack webhook format). +- **Webhook** — generic HTTPS POST to any URL for PagerDuty, Opsgenie, + or custom receivers. + +## Evaluation cadence + +Rules evaluate on the platform interval (`ALERT_EVAL_INTERVAL_MS`, +default 60s; see [System Configuration](/docs/system-config)). +Thresholds combine with duration so brief spikes do not page you. + +## Next steps + +- Read [Settings](/docs/settings) to configure the SMTP relay. +- Read [Observability](/docs/observability) to watch the underlying metrics. diff --git a/apps/docs/docs/auth.md b/apps/docs/docs/auth.md index 704e7fa..d520a5a 100644 --- a/apps/docs/docs/auth.md +++ b/apps/docs/docs/auth.md @@ -1,54 +1,64 @@ --- title: Authentication & Access Control -category: Networking & Security -description: Secure your Dequel dashboard with PAM-based authentication, session management, and API keys. +category: Security +description: Linux-user login, short-lived sessions, and scoped API keys. slug: auth --- -Dequel authenticates users against the **Linux system user database** via PAM (Pluggable Authentication Modules). Only users who are members of the `dequel` group can sign in. +Dequel has two credential types: interactive login for the dashboard +(Linux users) and API keys for programmatic access (CI, scripts). -## User Setup +## Dashboard login -Create a Linux user and add them to the `dequel` group: +Dequel authenticates dashboard users against the host's Linux accounts +through PAM. Only members of the `dequel` group can sign in. Create a +user after installing: ```bash +sudo groupadd -f dequel sudo useradd -m -s /bin/bash sudo passwd sudo usermod -aG dequel ``` -The user signs into the Dequel dashboard with their Linux username and password. +## Sessions -## Session Management +- **Access token** — a JWT in the `dequel_session` cookie, valid 15 + minutes (`HttpOnly`, `SameSite=Lax`, `Secure` behind HTTPS). +- **Refresh token** — a rotating opaque token in the `dequel_refresh` + cookie, valid 7 days. Logging out invalidates it immediately. +- The dashboard silently refreshes the session and redirects to + `/login` when both tokens are gone. -- **Access tokens**: Short-lived (15 min) HMAC-SHA256 JWTs stored in an `httpOnly` cookie. -- **Refresh tokens**: Long-lived (7 days) random tokens stored in the database. Automatically rotated on refresh. -- **Token blacklisting**: Logging out or refreshing invalidates the old refresh token immediately. -- **JWT secret**: Auto-generated on first API startup, stored at `data/.jwt_secret` (0600 permissions). Deleting this file invalidates all sessions. +## API keys -## API Keys +For CI pipelines, scripts, and external tools, create keys on the +**Keys → API Tokens** page (see [Keys](/docs/keys)): -For programmatic access (CI/CD, CLI tools), Dequel uses pre-shared API keys: - -1. Go to **Settings → API Keys** in the dashboard. -2. Create a new key — the full token is shown once. -3. Use it as a Bearer token in the `Authorization` header: +1. Generate a token with a label and scope (CI/CD, monitoring, or full + admin). +2. Copy the `dql_...` token — it is shown once and stored hashed. +3. Send it as a Bearer token: ```bash -curl -H "Authorization: Bearer dqk_" https://dequel.example.com/api/projects +curl -H "Authorization: Bearer dql_" https://dequel.example.com/api/projects ``` -API keys are scoped to the entire platform with the same permissions as the user who created them. +Keys carry read/write scopes enforced per HTTP method. Revoke a key any +time from the same page; deleting it breaks its integrations +immediately. -## Endpoints +## API endpoints | Method | Path | Auth | Description | |--------|------|------|-------------| | `POST` | `/api/auth/login` | None | Sign in with username/password | -| `POST` | `/api/auth/logout` | Session | Sign out, blacklist refresh token | +| `POST` | `/api/auth/logout` | Session | Sign out, invalidate refresh token | | `POST` | `/api/auth/refresh` | Session | Rotate refresh token | -| `GET` | `/api/auth/me` | None | Check current session status | +| `GET` | `/api/auth/me` | None | Current session status | | `GET` | `/api/health` | None | Health check | -The web dashboard redirects to `/login` when no valid session is detected. +## Next steps +- Read [Keys](/docs/keys) for API tokens and the SSH key pool. +- Read [GitHub](/docs/github) for the OAuth repo connection. diff --git a/apps/docs/docs/backups.md b/apps/docs/docs/backups.md new file mode 100644 index 0000000..c31fd09 --- /dev/null +++ b/apps/docs/docs/backups.md @@ -0,0 +1,51 @@ +--- +title: Backups +category: Data +description: Scheduled and on-demand backups with one-click restore. +slug: backups +--- + +Dequel backs up managed databases on a schedule you set per database, +plus the platform's own internal database. Every snapshot can be +restored to its container from the dashboard, the API, or the CLI. + +## Storage + +Configure where backups land under **Settings → Backups**: + +- **Local** — a path on the control plane (default `/data/backups`). +- **S3** — bucket, endpoint, region, and credentials (works with AWS S3 + and S3-compatible stores like R2). + +## Database backups + +Each database carries a backup schedule: enable it during provisioning +or later in the database **Settings** tab, with a cron preset (every 6 +hours, every 12 hours, daily) and a retention window (1–30 snapshots). +The database **Backups** tab lists snapshots with engine, size, and +date, and offers **Take Backup Now** (while running), **Restore**, and +**Delete**. + +Restoring replaces the container's data with the snapshot. Triggering +and restoring are also API calls (`POST /backups/:id/trigger`, +`POST /backups/:id/restore`). + +## System backups + +The platform database itself is snapshotted on its own cron (hourly to +weekly presets) with its own retention, configured in the same settings +section. **Backup System Database Now** takes an immediate snapshot. + +From a shell, the CLI covers the same flows (see [CLI](/docs/cli)): + +```bash +dequel backup # back up the internal database now +dequel backup # back up a managed database now +dequel backup list # list snapshots +dequel backup restore # restore a snapshot +``` + +## Next steps + +- Read [Managed Databases](/docs/databases) for engine operations. +- Read [CLI](/docs/cli) for shell-driven backup commands. diff --git a/apps/docs/docs/changelogs/v0.4.0.md b/apps/docs/docs/changelogs/v0.4.0.md index 209e6ba..505ae4c 100644 --- a/apps/docs/docs/changelogs/v0.4.0.md +++ b/apps/docs/docs/changelogs/v0.4.0.md @@ -19,6 +19,8 @@ description: Release notes and updates for Dequel version 0.4.0. - Track anonymous usage events ([8abdf74](https://github.com/Lftobs/dequel/commit/8abdf7420877621e10ed277ccb63a70df97a3523)) - Add inline MongoDB document editing ([4ead155](https://github.com/Lftobs/dequel/commit/4ead15502a7aaa24cd09b2477409dea1c277a494)) - Route TLS database traffic on port 443 ([8111b92](https://github.com/Lftobs/dequel/commit/8111b926be66562f1c4b89757eeaea090df19a8d)) +- Forward remote database traffic over SSH ([42fef01](https://github.com/Lftobs/dequel/commit/42fef016ab9140d137835fb5a829da5ca55ea177)) +- Add database encryption migration for existing plaintext secrets ([41b07b7](https://github.com/Lftobs/dequel/commit/41b07b76dec89f474c5722ade3f699394c6b0c04)) ### Improvements @@ -28,6 +30,8 @@ description: Release notes and updates for Dequel version 0.4.0. - Update database creation engine selection UI ([faef669](https://github.com/Lftobs/dequel/commit/faef6693959dab303340f0245430c48504d08db2)) - Improve SQL query editor ui ([f38b1a6](https://github.com/Lftobs/dequel/commit/f38b1a68fac5d9af7be44f124e6c879b68bf19ff)) - Switch compose file to prebuilt images for api and web ([3e6f917](https://github.com/Lftobs/dequel/commit/3e6f917403b233dcc43d60068ed18ddaf77174b6)) +- Separate fixed values from tunables ([e70a76c](https://github.com/Lftobs/dequel/commit/e70a76c33ca2a193f8e2a0b3f3b1ee75137e625c)) +- Clarify database exposure and MySQL access paths ([91e928b](https://github.com/Lftobs/dequel/commit/91e928bd73edfabfe33e87d84da4bcea2fe87b8c)) - Make dequel dashboard 100% mobile responsive ([905719f](https://github.com/Lftobs/dequel/commit/905719f336fd79c3818fe45cc023d0df7fdedc95)) - Make all settings sections, tables, tabs and dialogs fully mobile responsive ([1049120](https://github.com/Lftobs/dequel/commit/1049120cf5c7243be5d75c4d9474a4ab326f9bb4)) @@ -44,3 +48,17 @@ description: Release notes and updates for Dequel version 0.4.0. - Clean stale routes from their owning server ([fd2d5e3](https://github.com/Lftobs/dequel/commit/fd2d5e30a6c93c36398404dc360277b30f75b87a)) - Stop previous containers and replicas on redeploy ([ee35155](https://github.com/Lftobs/dequel/commit/ee351553cff825c642195c107169123f34d484ec)) - Use the real container name as the route primary ([aa64121](https://github.com/Lftobs/dequel/commit/aa6412131cde9c4542d543a1609d12ac2afa5aad)) +- Resolve zombie process leaks and subprocess lifecycle management ([6284ed6](https://github.com/Lftobs/dequel/commit/6284ed6299d00fa815e912973d2ea14a1045c986)) +- Address review feedback on stream decoding, process race conditions, and duplicate types ([35ecc92](https://github.com/Lftobs/dequel/commit/35ecc924595e23ce0fc0d614c0e4efffc7b9394c)) +- Pass the project server to rollback cleanup ([aa61f92](https://github.com/Lftobs/dequel/commit/aa61f92afafa9c34b90cb757e57d13292b139777)) +- Count only replicas Docker actually removed ([511d0dc](https://github.com/Lftobs/dequel/commit/511d0dc64eee125f630b37e552049f8ead831ae6)) +- Fail closed when a database allows no CIDR ([3304b53](https://github.com/Lftobs/dequel/commit/3304b530cc7c7a26c8d4178938410cd5b8009111)) +- Let the server finish closing before exiting ([6940ea0](https://github.com/Lftobs/dequel/commit/6940ea04f8a65e5aec9abfe461f44cc48c948d8f)) +- Don't report provisioning before credentials are revealed ([8798c37](https://github.com/Lftobs/dequel/commit/8798c377be26ff2bd56c7623656c0da468108b7d)) +- Live-probe targets, rate-limit auto moves, clean old containers ([ee6bc0e](https://github.com/Lftobs/dequel/commit/ee6bc0ec6dd989598a40844c017607e9238255ea)) +- Run the railpack config generator on remote hosts ([850bf5b](https://github.com/Lftobs/dequel/commit/850bf5b66aacd7e267d9a4de52e8c3fd407d62d0)) +- Access-log ingress routes and target runtime logs at the deployment's server ([34a8638](https://github.com/Lftobs/dequel/commit/34a863854e580fd48f58d77b778e29d38da2b425)) +- Bug-sheet fixes, test-suite repair, and Let's Encrypt certs for database hosts ([31cbea0](https://github.com/Lftobs/dequel/commit/31cbea0fcf6091ea4b1a991e23a0a7cec34812d4)) +- Use real certificates and server hosts for connections ([b7581b7](https://github.com/Lftobs/dequel/commit/b7581b7786dfa290217c914a24d94a69deec2e5b)) +- Remove redundant zombie reaper initialization ([81ea4b0](https://github.com/Lftobs/dequel/commit/81ea4b09011f5d6a57a9fe888f446a8c1a65a0df)) +- Harden deployment and configuration workflows ([6ebfde7](https://github.com/Lftobs/dequel/commit/6ebfde7b15c5ae016232e68b5034a765b04afee8)) diff --git a/apps/docs/docs/cli.md b/apps/docs/docs/cli.md new file mode 100644 index 0000000..e08ffd9 --- /dev/null +++ b/apps/docs/docs/cli.md @@ -0,0 +1,58 @@ +--- +title: CLI Reference +category: Platform +description: Manage Dequel from the shell with the dequel command. +slug: cli +--- + +The installer places the `dequel` command on your `PATH`. It manages +the compose stack, updates, and backups. Configuration lives in +`~/.dequel` (override with `DEQUEL_HOME`). + +## Lifecycle + +| Command | Description | +|---------|-------------| +| `dequel start` | Start all services | +| `dequel stop` | Stop all services | +| `dequel restart` | Restart all services | +| `dequel status` | Show service status | +| `dequel logs [service]` | Follow service logs | +| `dequel uninstall` | Remove everything (prompts first) | +| `dequel --version` | Show version | +| `dequel --help` | Show help | + +## Updates + +```bash +dequel update # latest stable (rolling restart) +dequel update --pre # latest pre-release +dequel update --rc # latest release candidate +dequel update vX.Y.Z # a specific version +``` + +## Backups + +```bash +dequel backup # back up the internal database now +dequel backup # back up a managed database now +dequel backup list # list snapshots +dequel backup restore # restore a snapshot +``` + +Snapshots land in the storage configured under **Settings → Backups** +(see [Backups](/docs/backups)). + +## API access from scripts + +The CLI manages the platform, not your apps. For automation against +your projects, use [API keys](/docs/keys) with `curl`: + +```bash +curl -H "Authorization: Bearer dql_" https://dequel.example.com/api/projects +``` + +## Next steps + +- Read [Installation](/docs/installation) for setup. +- Read [Backups](/docs/backups) for the dashboard backup flows. diff --git a/apps/docs/docs/configuration.md b/apps/docs/docs/configuration.md index 26fbc0a..7e8fc55 100644 --- a/apps/docs/docs/configuration.md +++ b/apps/docs/docs/configuration.md @@ -1,46 +1,62 @@ --- -title: Configuration -category: Getting Started -description: Configure compute bounds, custom ports, and service specs. +title: Project Configuration +category: Projects +description: Build settings, ports, and resource limits for your project. slug: configuration --- -Dequel provides fine-grained control over container resources and network routing to ensure optimal application performance and stability. +Each project carries its build and runtime configuration. You set most of +it in the creation wizard and can change it later on the project's +**Settings** tab (`PATCH /projects/:id`). -## Resource Limits +## Build settings -Configure boundary limits under the Settings layout to avoid cluster resource starvation or application OOM (Out Of Memory) crashes: +**Build strategy** is either `railpack` (automatic single-service build) +or `compose` (multi-container from your repo-root `docker-compose.yml`). +Railpack auto-detects the framework and proposes install, build, and +start commands plus the output directory and port — every field stays +editable for frameworks it guesses wrong. -- **CPU Limit (cores):** The maximum amount of CPU shares a container replica can consume. Specifying `1.0` permits a full single core. -- **Memory Limit (MB):** The boundary RAM limit allocated. If a container exceeds this, it is automatically terminated with an OOM error code and restarted. Recommended default: `512MB`. +Useful fields: -## Port Configurations +- **Root directory** — build a subdirectory of the repo (monorepos). +- **Port** — the port your app listens on inside the container. Railpack + usually detects this; override it when it does not. +- **Output directory** — for static sites, the folder served after build. +- **Project type** — `web` (long-running server) or `static` (built + assets served directly). +- **Compose service mapping** — for Compose builds, one row per + internet-facing service: service name, port, and subdomain. The first + row is the entry service on the project's main domain; the rest get + `.`. -Dequel's edge router (Caddy) uses reverse proxy routing to map incoming HTTP request traffic to your container. +## Ports and routing -
- ⚠️ Crucial Note: Internal Ports -

Your application inside the container must bind to the environment-injected PORT variable. Hardcoding listen ports to 3000 or 8080 without reading the environment variable will result in a 502 Bad Gateway error, as Caddy's dynamic routing target won't match.

-
+Your app must listen on the configured port on all interfaces +(`0.0.0.0`). A typical Node entrypoint reads it from the environment: -## Example: Express server +```js +const PORT = process.env.PORT || 3000; +app.listen(PORT, "0.0.0.0"); +``` -
-
- server.js -
- 
const express = require('express');
-const app = express();
+Caddy reverse-proxies the project domain to that port. If the app binds
+`localhost` only or a different port, the route returns `502 Bad
+Gateway` — check the [route files](/docs/agent-caddy-routes) and the
+container's runtime logs.
 
-// ALWAYS read from process.env.PORT
-const PORT = process.env.PORT || 3000;
+## Resource limits
+
+On the **Scaling** tab, the resource card sets per-container bounds:
+
+- **CPU limit (cores)** — for example `1.0` for one full core.
+- **Memory limit (MB)** — for example `512`. A container that exceeds it
+  is killed with an OOM error and restarted.
 
-app.get('/', (req, res) => {
-  res.send('Hello from Dequel!');
-});
+Autoscaling requires a CPU limit to be set, since the engine scales on
+CPU utilization. See [Scaling](/docs/scaling).
 
-app.listen(PORT, '0.0.0.0', () => {
-  console.log(`Server listening on port ${PORT}`);
-});
-
+## Next steps +- Read [Deployments](/docs/deployments) for the deploy lifecycle. +- Read [Environment Variables](/docs/env-vars) for config injection. diff --git a/apps/docs/docs/control-plane.md b/apps/docs/docs/control-plane.md deleted file mode 100644 index 8d8dccf..0000000 --- a/apps/docs/docs/control-plane.md +++ /dev/null @@ -1,55 +0,0 @@ ---- -title: Control Plane Setup -category: Getting Started -description: Install Dequel and connect remote servers for deployment. -slug: control-plane ---- - -The control plane (dashboard + API) runs on any machine with Docker. All builds, containers, and Caddy routing happen on the **servers** you connect; the control plane only orchestrates them over SSH or the P2P WireGuard agent. - -## Install - -The install script sets up all services (`api`, `web`, `caddy`, `redis`, `buildkit`, and the monitoring stack): - -```bash -curl -fsSL https://github.com/Lftobs/dequel/releases/latest/download/install.sh | bash -dequel start -``` - -Open `http://localhost` to access the dashboard. - -## Connecting servers - -### SSH mode (recommended) - -1. In **Settings → Servers**, add your server with its IP/hostname, SSH port, and SSH user (defaults to `root`). -2. The server only needs SSH access from the control plane — Docker, Caddy, and Railpack are installed for you. -3. Click **Prepare** on the server row. Dequel streams live progress as it: - - Connects over SSH and detects the OS - - Installs Docker if missing - - Installs Caddy and configures `/etc/caddy/routes/` - - Installs the Railpack CLI -4. When the stream reports success, the server is ready to deploy to. - -Prepare is idempotent — re-running it skips anything already installed. - -### Agent mode (firewalled nodes) - -For servers without open SSH ports, add the server as **Agent**, then click **Prepare**. Dequel generates a registration token and shows a `docker run` command — run it on the target server. The agent registers back over the control plane and establishes a direct WireGuard tunnel. The prepare stream waits until the agent reports connected. - -## Shared base domain - -All servers share a single base domain. Set it once on the control plane: - -``` -CADDY_BASE_DOMAIN=example.com -``` - -Every project defaults to `.` (e.g. `api.example.com`) **regardless of which server it is deployed to** — Dequel generates identical route hostnames everywhere. - -DNS setup: point `*.example.com` at your servers. With multiple servers behind one wildcard, either: - -- Point the wildcard record at a single ingress server, or -- Add per-deployment A records (`api.example.com → server-ip`) for finer control - -Projects with a custom domain configured bypass the base domain entirely. \ No newline at end of file diff --git a/apps/docs/docs/databases.md b/apps/docs/docs/databases.md index 6625c3d..4fa906c 100644 --- a/apps/docs/docs/databases.md +++ b/apps/docs/docs/databases.md @@ -39,6 +39,14 @@ To inject this secret into your container runtime, navigate to **Environment Var When **Public Access** is enabled, Dequel issues an external connection string that travels through the shared HTTPS port (443). The gateway reads the hostname from the TLS handshake and forwards the raw bytes to the database, so only ports 22, 80, and 443 ever need to be open — no per-database firewall rules, no random high ports. +### How exposing works for PostgreSQL, MongoDB, and Redis + +PostgreSQL, MongoDB, and Redis all expose through the gateway the same way. To expose one of them: + +1. Open the database in the dashboard and turn on **Public Access**. +2. Dequel writes a gateway route for `db-.` and Caddy automatically provisions a Let's Encrypt certificate for that hostname. This takes seconds and renews on its own. +3. Connect from any machine with the external string Dequel hands out. Your client opens TLS to port 443 and announces the hostname; the gateway checks the database status, the Public Access toggle, and the optional IP allowlist, then forwards raw bytes to the database — including databases that run on a remote server in your cluster. + Setup requirements: - Set `CADDY_BASE_DOMAIN` to your domain (for example `dequel.example.com`). @@ -52,7 +60,7 @@ Exposed databases receive strings that require TLS:
- mongodb://user:password@db-1a2b3c4d.dequel.example.com:443/dbname?authSource=admin&tls=true&tlsAllowInvalidCertificates=true + mongodb://user:password@db-1a2b3c4d.dequel.example.com:443/dbname?authSource=admin&tls=true
@@ -61,10 +69,33 @@ Exposed databases receive strings that require TLS: If **Public Access** is disabled, no external endpoint is issued — the database only receives its internal cluster string. +### MySQL — direct port instead + +MySQL is the one exception. Its wire protocol sends a server greeting before the client says anything, so there is no client-announced hostname for the gateway to route on. Exposed MySQL databases therefore keep the direct port path: Dequel publishes a random port on the database's own server, and the credentials panel shows the endpoint (`:`) along with a warning while the port is still blocked. Open that port in your server's firewall or security group, or use the dashboard SQL console instead. + Notes: -- MySQL is the exception: its wire protocol begins with a server greeting before the client sends anything, so there is no hostname for the gateway to route on. Exposed MySQL databases keep the direct port path (`:`); open that port in your firewall, or use the dashboard SQL console. -- The gateway terminates TLS with a platform certificate. MongoDB strings skip verification (`tlsAllowInvalidCertificates`). For PostgreSQL, `sslmode=require` skips verification in libpq/psql, but the Node `pg` driver treats it as full verification — there, either append `&uselibpqcompat=true` (Node only; psql rejects that flag) or set `ssl: { rejectUnauthorized: false }` in client code. Redis clients verify certificates by default, so set `rejectUnauthorized: false` in your client configuration. -- PostgreSQL clients work either way: classic clients send a plaintext `SSLRequest` first (the gateway answers it locally), and direct-TLS clients (`sslmode=direct`) announce the hostname immediately. +- The gateway terminates TLS with a Let's Encrypt certificate for the database + hostname. Caddy issues the certificate within seconds of enabling Public + Access and renews it automatically, so the strings above verify out of the + box. Right after you enable Public Access, or on a `localhost` setup with no + base domain, the gateway falls back to a self-signed certificate that clients + cannot verify. We recommend waiting for the real certificate and connecting + with verification on. If you must connect during that gap, disable + verification only as a temporary measure, and turn it back on once the real + certificate is in place. + +> **Warning:** A connection with certificate verification disabled is open to +> interception. Treat it as unencrypted until the real certificate is issued. + + - MongoDB: the string carries `tlsAllowInvalidCertificates=true`. + - PostgreSQL: set `ssl: { rejectUnauthorized: false }` in client code. For + Node-only tooling you can append `&uselibpqcompat=true`; psql rejects that + flag. + - Redis: clients verify certificates by default, so set + `rejectUnauthorized: false` in your client configuration. +- PostgreSQL clients work either way: classic clients send a plaintext + `SSLRequest` first (the gateway answers it locally), and direct-TLS clients + (`sslmode=direct`) announce the hostname immediately. - Every new connection is checked against the database status, the Public Access toggle, and the optional IP allowlist before any bytes are forwarded. diff --git a/apps/docs/docs/deployments.md b/apps/docs/docs/deployments.md index 811d51f..04a9387 100644 --- a/apps/docs/docs/deployments.md +++ b/apps/docs/docs/deployments.md @@ -1,31 +1,59 @@ --- title: Deployments -category: Core Architecture -description: Trigger builds, view active execution logs, and roll back code changes. +category: Projects +description: Trigger builds, follow live logs, and roll back releases. slug: deployments --- -Dequel supports two primary methods for deploying code changes: automated tracking via **Git repository branches**, and **manual zip uploads** directly from the local filesystem. +A deployment is one build-and-run cycle of your project. The project's +**Deployments** tab lists the full history with status, source, branch, +duration, and age, and lets you manage each one. -## Git Integration +## Deployment sources -Configure a Git repository URL (e.g. `github.com/my-org/my-project`) and specify a branch name (e.g. `main`). +**Git.** Point the project at a repository URL and branch (or pick a repo +through the GitHub integration). New pushes rebuild automatically when +the webhook is connected — toggle **Auto-deploy** on the Deployments +tab. **Manual Deploy** rebuilds from the latest commit or a specific +SHA, with an option to clear the BuildKit cache. -- **Automated webhook updates:** Dequel listens to push events to rebuild automatically. -- **Manual triggers:** Click the **Deploy Branch** CTA in the UI to manually queue a build of the latest commit. +**Upload.** Drop a `.zip` (at creation) or `.zip` / `.tar` / `.tar.gz` +(on the Deployments tab) to deploy local code. An upload project can be +migrated to Git later from the same tab without losing history. -## Manual File Directory Uploads +**Docker Compose.** Repos with a `docker-compose.yml` build all services +together; only the mapped ingress services get public URLs (see +[Project Configuration](/docs/configuration)). -If your code is not stored in a public repository, or you want to test local modifications immediately: +## Lifecycle -1. Drag and drop your local directory as a compressed `.zip` archive. -2. Dequel uploads the file payload directly to the builder API node. -3. The archive is unpacked and fed straight into the Docker build engine. +Deployments move through `pending` → `building` → `deploying` → +`running`. A failed build ends in `failed`; you can `cancel` a queued or +active build. Only one deployment is live per project — each successful +deploy replaces the previous container. -## Active Logs & Rollbacks +Click a row to expand its **build logs**, streamed live while the build +runs and stored afterwards. The **Runtime Logs** view (under the +**Logs** tab) shows the running container's stdout instead. -Each deployment creates an isolated, immutable build artifact. +## Redeploy, rollback, delete -- **Build Logs:** View the live compiler steps as Docker pulls layers, executes commands, and builds cache images. -- **Zero-Downtime Rollbacks:** Instantly switch traffic back to a previous build artifact by clicking the **Promote** button on any previously successful deployment. +- **Redeploy** clones the currently running deployment (same image and + config) — the standard way to apply changed env vars or volumes. +- **Rollback** points the route back at an earlier deployment's image. + Images are kept, so rollback is instant and does not rebuild. +- **Delete** destroys a deployment's containers. The running deployment + cannot be deleted. +## Failed builds and AI diagnosis + +If a deployment fails, the log header shows a **Diagnose** button. It +opens the AI diagnosis sheet: pick a configured LLM provider and model, +and the agent investigates the logs, decides whether the fault is your +code or the platform, and can open a fix pull request on your repo. See +[AI Diagnosis](/docs/ai-diagnosis). + +## Next steps + +- Read [AI Diagnosis](/docs/ai-diagnosis) for the failure workflow. +- Read [Observability](/docs/observability) for logs and metrics. diff --git a/apps/docs/docs/domains.mdx b/apps/docs/docs/domains.mdx index 168b905..0f497f1 100644 --- a/apps/docs/docs/domains.mdx +++ b/apps/docs/docs/domains.mdx @@ -1,31 +1,51 @@ --- title: Custom Domains -category: Networking & Security -description: Point custom web addresses to container deployment endpoints using DNS records. +category: Projects +description: Point your own domains at Dequel projects with automatic HTTPS. slug: domains --- -Dequel provides public routing endpoints. You can route external domain request traffic (e.g. `app.mycompany.com`) by pointing DNS records to Dequel cluster host IP addresses. - -## Step 1: Add Domain in Dequel Console - -Under the **Domains** tab, click **Add Domain**, enter your custom domain address (e.g., `blog.mydomain.com`), and save. - -## Step 2: Configure DNS Records - -In your DNS registrar's control panel (Cloudflare, GoDaddy, Namecheap, etc.), add the record displayed in the instructions box: - -- **For Subdomains:** Add a `CNAME` record pointing your subdomain (e.g. `blog`) to the cluster's base domain. -- **For Apex/Root Domains:** Add an `A` record pointing `@` directly to the Dequel cluster server IP. - -:::note[DNS Propagation] -DNS changes can take up to 24–48 hours to propagate globally. Dequel will run automated validation checks in the background until the records resolve correctly. +Every project gets a default URL on your base domain +(`.`). Add **custom domains** on the +project's **Domains** tab to serve the app on addresses you own, like +`app.mycompany.com`. + +## Adding a domain + +1. Click **Add Domain** and enter the domain. +2. For Docker Compose projects, pick the target service and port; for + single-service projects the running container is the target. +3. Dequel shows DNS instructions. Create the record at your registrar: + - **Subdomain** (`app.mycompany.com`): a `CNAME` pointing at your + base domain, or an `A` record pointing at the server IP. + - **Apex domain** (`mycompany.com`): an `A` record pointing at the + server IP (find it under **Settings → Servers** or + `GET /server/ip`). +4. Dequel verifies DNS in the background and marks the domain + `verified`. Caddy then serves it with an automatic Let's Encrypt + certificate (see [SSL](/docs/ssl)). + +The Domains tab shows live status per domain: validation state, SSL +state, whether DNS resolves, and whether TLS handshakes succeed (polled +continuously). + +:::note[DNS propagation] +New records can take minutes to hours to propagate. Dequel keeps +re-checking until they resolve — you don't need to click anything while +waiting. ::: +## Validation states + +- **Pending** — waiting for DNS to resolve to the server. +- **Verified** — DNS matches and the route is live. +- **Failed** — records point somewhere else. Fix the DNS values and + Dequel re-checks. -## Validation Statuses +Remove a domain any time with the delete action; its route is removed +on the next Caddy reload. -- **Pending:** Dequel is awaiting DNS propagation. -- **Verified:** DNS records resolve correctly. The proxy router is configured. -- **Failed:** Resolving record targets did not match the required target targets. Check your DNS values. +## Next steps +- Read [SSL](/docs/ssl) for certificate details. +- Read [Agent Caddy Routes](/docs/agent-caddy-routes) for the routing internals. diff --git a/apps/docs/docs/env-vars.mdx b/apps/docs/docs/env-vars.mdx index cb32ee8..3e8457f 100644 --- a/apps/docs/docs/env-vars.mdx +++ b/apps/docs/docs/env-vars.mdx @@ -1,28 +1,39 @@ --- title: Environment Variables -category: Core Architecture -description: Inject configuration values, API keys, and connection credentials securely into container environments. +category: Projects +description: Inject per-environment config and secrets into your containers. slug: env-vars --- -Environment variables decouple application configurations from source code repositories. Secrets (like payment gateways, SMTP credentials, or database passwords) must always be injected this way to preserve security. +Environment variables decouple configuration from code. API keys, +database URLs, and feature flags go here instead of the repository. +Values are encrypted at rest with `ENV_ENCRYPTION_KEY` and masked in +the UI until you reveal them. -## Adding Variables +## Managing variables -In the project console, click the **Environment Variables** tab: +Open the project and switch to the **Env Variables** tab: -- Enter a key name (e.g. `STRIPE_SECRET_KEY`). Key names should contain only alphanumeric characters and underscores. -- Enter the value. All values are securely stored and encrypted in the cluster database. -- Click **Add Variable**. +- **Add** one variable or several at once, each scoped to an environment + (for example `production`). Keys are conventionally uppercase. +- **Import** a local `.env` file to bulk-load variables. +- **Reveal** a masked value with the eye icon (decrypts on demand), or + copy it without revealing. +- **Edit** a value inline or **delete** the variable. -:::note[Masked Displays] -For security, all environment variable values are masked on load. Click the eye icon to reveal them, or copy them securely via the copy badge. -::: +Variables are injected when the container starts, so adding, editing, or +deleting one requires a restart. Dequel shows a redeploy banner at the +top of the tab after any change — click **Redeploy** to roll the new +config out. +## Shared variables -## Redeployment Prompts +Variables needed by several projects (a shared API key, a common feature +flag) live once in the **Shared Env** library and are linked into +projects. A project-level variable with the same key wins over the +shared one. See [Shared Env Vars](/docs/shared-env-vars). -Because environment variables are injected during container startup, modifying or deleting variables requires a container restart to take effect. - -Dequel displays a **Redeployment Warning Banner** at the top of the tab whenever variables are mutated. Click the **Redeploy Now** button on the banner to execute a zero-downtime rolling update with the new configurations. +## Next steps +- Read [Shared Env Vars](/docs/shared-env-vars) for cross-project config. +- Read [Deployments](/docs/deployments) for the redeploy flow. diff --git a/apps/docs/docs/github.md b/apps/docs/docs/github.md new file mode 100644 index 0000000..a9d8d5c --- /dev/null +++ b/apps/docs/docs/github.md @@ -0,0 +1,42 @@ +--- +title: GitHub Integration +category: Integrations +description: Repo picker, push-to-deploy webhooks, and fix pull requests. +slug: github +--- + +Connect a GitHub OAuth app once, and Dequel picks repos, installs +push-to-deploy webhooks, and opens AI fix pull requests on your behalf. + +## Setup + +Open **Settings → GitHub** and save the OAuth app credentials (client +ID, client secret, app name, webhook secret). The page shows the exact +callback URL to register in **GitHub → Settings → Developer settings → +OAuth Apps** and walks through creating the app. + +Then click **Connect GitHub** (in Settings or in the project wizard) to +authorize with `repo`, `read:org`, and `admin:repo_hook` scopes. +Dequel stores the token server-side and shows a connected badge. + +## What it unlocks + +- **Repo picker** — in the project wizard, choose **GitHub Account** + instead of pasting URLs: search your repos, and Dequel fills in the + clone URL, branch, and project name. +- **Push-to-deploy** — toggle **Auto-deploy** on the Deployments tab. + Dequel registers a webhook (`/api/github/webhook`, HMAC-verified with + your secret) that rebuilds the project on every push to the tracked + branch. +- **Fix pull requests** — [AI Diagnosis](/docs/ai-diagnosis) can open a + `dequel-fix/*` branch and PR against your repo when it determines the + failure is in your code. Requires the connected account to have push + access. + +Disconnect any time from the same tab; webhooks already installed keep +working until removed. + +## Next steps + +- Read [Deployments](/docs/deployments) for manual and auto deploys. +- Read [AI Diagnosis](/docs/ai-diagnosis) for the fix-PR flow. diff --git a/apps/docs/docs/index.mdx b/apps/docs/docs/index.mdx index 3539979..49b38c1 100644 --- a/apps/docs/docs/index.mdx +++ b/apps/docs/docs/index.mdx @@ -4,26 +4,37 @@ category: Getting Started slug: docs --- -Dequel provides developers with a Heroku-like platform experience on top of virtualized docker runtimes and cluster node providers. You get the simplicity of automated Git triggers, built-in let’s encrypt SSL proxy routing, and direct persistent volume attachments without wrestling with complex Kubernetes configurations. - -## Core Philosophy - -- **Zero-Config Routing:** Incoming traffic is automatically proxied to active deployments based on project configuration. -- **State Isolation:** Web containers are designed to be ephemeral. Persistent state is kept cleanly separated in managed databases or mapped node volumes. -- **Self-Healing Runtimes:** Failed container instances are automatically replaced, and scaling adjustments occur automatically based on CPU usage. - -:::note[Key Concept: Projects] -A Project in Dequel represents a single service (a container build) along with its environment configurations, database attachments, routing domains, and active deployments. +Dequel is a self-hosted deployment platform. You connect your servers, +push code from Git or a ZIP file, and Dequel builds, runs, routes, and +monitors your apps — with automatic HTTPS, managed databases, horizontal +autoscaling, and AI-assisted failure diagnosis built in. + +If you have used Heroku, Render, or Railway, Dequel gives you that same +push-to-deploy workflow on infrastructure you own. There is no control +plane phoning home: the dashboard, API, builders, and observability stack +all run as containers on your machine. + +## How Dequel works + +When you create a project and trigger a deployment: + +1. Dequel fetches your code from a Git repository (or unpacks your ZIP + upload) into a build workspace. +2. It detects your framework automatically with Railpack — no Dockerfile + required — or follows your `docker-compose.yml` for multi-container + projects, then builds the image with BuildKit. +3. It starts your container on the target server (the control plane + itself, a server you connected over SSH, or a WireGuard agent node) + and injects your environment variables and volume mounts. +4. It writes a Caddy route file so your app is reachable at + `https://.` with an automatic Let's Encrypt + certificate, or at `http://.localhost` in local setups. +5. It keeps watching: request logs, runtime logs, CPU and memory metrics, + alert rules, and autoscaling policies all run against the deployment. + +:::note[Key concept: projects] +A project in Dequel is a single service — its source, build settings, +environment variables, volumes, domains, scaling policy, alert rules, +and deployment history. Databases are separate managed resources you +attach to projects. ::: - - -## How Dequel Works - -When you connect a repository or upload a folder: - -1. Dequel reads your project's code directory and looks for a `Dockerfile` or builder configurations. -2. It initiates a secure cluster builder environment to package your container image. -3. The container image is pushed to the local cluster registry and distributed across running nodes. -4. Caddy routing endpoints are automatically updated to direct traffic to the newly created instance ports. - - diff --git a/apps/docs/docs/installation.md b/apps/docs/docs/installation.md index 72edfde..49c8bfd 100644 --- a/apps/docs/docs/installation.md +++ b/apps/docs/docs/installation.md @@ -5,155 +5,129 @@ description: Install Dequel on your own infrastructure. slug: installation --- +You install Dequel with one command. The installer sets up the control +plane (dashboard, API, builders, databases engine, and monitoring) as +Docker containers on your machine. Your apps can run on that same machine, +or on additional servers you connect later (see +[Servers](/docs/servers)). + ## Prerequisites -- **Docker** installed on your system -- **Docker Compose v2** (`docker compose` command) +- **Docker** and **Docker Compose v2** (`docker compose`) +- A Linux server with ports `80` and `443` reachable if you want public + HTTPS URLs (not needed for local-only use) -## Install Script +## Install -The quickest way to get Dequel running: +Run the install script: ```bash curl -fsSL https://github.com/Lftobs/dequel/releases/latest/download/install.sh | bash ``` -This will: - -1. Check that Docker and Docker Compose are installed -2. Create a config directory at `~/.dequel` -3. Download `docker-compose.yml` and monitoring configs -4. Pull pre-built Docker images from GitHub Container Registry -5. Install the `dequel` CLI to `/usr/local/bin` +The script checks for Docker and Compose, creates a config directory at +`~/.dequel`, generates a unique `ENV_ENCRYPTION_KEY`, downloads the +`docker-compose.yml` and monitoring configs, pulls the pre-built images +from GitHub Container Registry, installs the `dequel` CLI, and asks for +an optional admin email (Let's Encrypt notifications) and base domain +(public app URLs). -After installation, start the platform: +Then start the platform: ```bash dequel start ``` -Open `http://localhost` to access the dashboard (or the configured `CADDY_BASE_DOMAIN` in production). +Open `https://` to access the dashboard. If you configured a base +domain during install, the dashboard is also served on it. + +![Dequel login page](/screenshots/login.png) -## Post-Installation: User Setup +## Create your login user -Dequel authenticates against Linux system users in the `dequel` group. After install, create a user: +Dequel authenticates against Linux system users in the `dequel` group. +Create the group and a user after installing: ```bash +sudo groupadd -f dequel sudo useradd -m -s /bin/bash sudo passwd sudo usermod -aG dequel ``` -See [Authentication & Access Control](/docs/auth) for details on session management and API keys. - -## Manual Setup (no install script) - -If the installer fails, set up the platform manually with just Docker Compose and the config files: - -```bash -# Create the installation directory -mkdir -p ~/.dequel/data ~/.dequel/workspace \ - ~/.dequel/infra/caddy/routes \ - ~/.dequel/infra/monitoring/grafana/datasources - -# Download config files -curl -fsSL https://raw.githubusercontent.com/Lftobs/dequel/main/docker-compose.yml \ - -o ~/.dequel/docker-compose.yml -curl -fsSL https://raw.githubusercontent.com/Lftobs/dequel/main/infra/caddy/Caddyfile \ - -o ~/.dequel/infra/caddy/Caddyfile -curl -fsSL https://raw.githubusercontent.com/Lftobs/dequel/main/infra/monitoring/prometheus.yml \ - -o ~/.dequel/infra/monitoring/prometheus.yml -curl -fsSL https://raw.githubusercontent.com/Lftobs/dequel/main/infra/monitoring/loki-config.yml \ - -o ~/.dequel/infra/monitoring/loki-config.yml -curl -fsSL https://raw.githubusercontent.com/Lftobs/dequel/main/infra/monitoring/promtail-config.yml \ - -o ~/.dequel/infra/monitoring/promtail-config.yml -curl -fsSL https://raw.githubusercontent.com/Lftobs/dequel/main/infra/monitoring/grafana/datasources/loki.yml \ - -o ~/.dequel/infra/monitoring/grafana/datasources/loki.yml -curl -fsSL https://raw.githubusercontent.com/Lftobs/dequel/main/infra/monitoring/grafana/datasources/prometheus.yml \ - -o ~/.dequel/infra/monitoring/grafana/datasources/prometheus.yml - -# Start all services -docker compose -f ~/.dequel/docker-compose.yml up -d -``` - -The compose file uses pre-built images from GitHub Container Registry, so no source code checkout is needed. Access the dashboard at `http://localhost` (or your configured `CADDY_BASE_DOMAIN`). - -## Manual Setup with Docker Compose (with source) +Sign in with that username and password. See +[Authentication](/docs/auth) for sessions, API keys, and programmatic +access. -Clone the repository and build from source: +## Production setup -```bash -git clone https://github.com/Lftobs/dequel.git -cd dequel -docker compose up -d --build -``` - -## Production Setup +To serve public HTTPS traffic: -To make Dequel publicly accessible with auto-SSL: - -1. Set `CADDY_BASE_DOMAIN` and `CADDY_EMAIL` in `~/.dequel/.env`: - ``` +1. Set the base domain and contact email in `~/.dequel/.env` (the + installer already wrote these if you provided them): + ```bash CADDY_BASE_DOMAIN=example.com CADDY_EMAIL=admin@example.com ``` -2. Configure a wildcard DNS `*.example.com` pointing to your server's IP. +2. Point a wildcard DNS record (`*.example.com`) at your server's IP. 3. Ensure ports `80` and `443` are open in your firewall. -4. Restart Dequel: `dequel restart` +4. Restart with `dequel restart`. -Deployed apps will be reachable at `https://.example.com` with auto-provisioned Let's Encrypt certificates. The dashboard is available at both `http://example.com` and `https://example.com`. - -## The `dequel` CLI - -The `dequel` command manages the platform lifecycle: - -| Command | Description | -|---------|-------------| -| `dequel start` | Start all Dequel services | -| `dequel stop` | Stop all services | -| `dequel status` | Show service status | -| `dequel logs` | Follow service logs | -| `dequel update` | Pull latest images and restart | -| `dequel restart` | Restart all services | -| `dequel --help` | Show all commands | +Deployed apps become reachable at `https://.example.com` +with automatic Let's Encrypt certificates. -## Configuration Directory +## Configuration directory -Dequel stores its configuration in `~/.dequel` (or `$DEQUEL_HOME` if set): +Dequel stores its state in `~/.dequel` (override with `DEQUEL_HOME`): -``` +```text ~/.dequel/ ├── docker-compose.yml -├── .env # Optional environment overrides -├── data/ # SQLite database +├── .env # CADDY_BASE_DOMAIN, CADDY_EMAIL, ENV_ENCRYPTION_KEY +├── data/ +│ ├── dequel.json # File-based platform config (0600) +│ └── ... # Postgres data, backups, instance id ├── workspace/ # Build staging area +├── scripts/auth/ # PAM auth server files └── infra/ ├── caddy/ │ ├── Caddyfile - │ └── routes/ - └── monitoring/ - ├── prometheus.yml - ├── loki-config.yml - └── grafana/ + │ └── routes/ # Generated per-project route files + └── monitoring/ # Prometheus, Loki, Grafana configs ``` -## Building from Source +PostgreSQL holds all platform state (projects, deployments, env vars, +databases, users). It runs as the `postgres` container with a persisted +volume — there is no SQLite file to manage. -For local development, run the API and web dashboard directly: +## The `dequel` CLI -```bash -# Terminal 1 — API -export DATABASE_PATH=./data/dequel.db \ - WORKSPACE_ROOT=./workspace \ - CADDY_ROUTES_DIR=./infra/caddy/routes \ - CADDY_BASE_DOMAIN=localhost \ - DOCKER_NETWORK=dequel_net \ - APP_INTERNAL_PORT=3000 -bun apps/api/src/index.ts - -# Terminal 2 — Web -bun apps/web/src/main.tsx -``` +The CLI manages the platform lifecycle. See the full +[CLI reference](/docs/cli) for update channels and backups. + +| Command | Description | +|---------|-------------| +| `dequel start` | Start all services | +| `dequel stop` | Stop all services | +| `dequel restart` | Restart all services | +| `dequel status` | Show service status | +| `dequel logs` | Follow service logs | +| `dequel update` | Update to the latest release | +| `dequel backup` | Back up the internal database | +| `dequel uninstall` | Remove everything (prompts first) | + +## Manual setup + +If the installer does not fit your environment, replicate what it does: +create the directory layout above, download `docker-compose.yml` and the +`infra/` configs from the latest GitHub release, write your `.env` with +`CADDY_BASE_DOMAIN`, `CADDY_EMAIL`, and a generated +`ENV_ENCRYPTION_KEY`, then run +`docker compose -f ~/.dequel/docker-compose.yml up -d`. + +To build from source instead, clone the repository and run +`docker compose up -d --build`. For local development of the API and +dashboard directly, see the commands in `AGENTS.md`. ## Updating @@ -161,5 +135,10 @@ bun apps/web/src/main.tsx dequel update ``` -This pulls the latest images from GitHub Container Registry and recreates the services. +This pulls the latest images and recreates services with a rolling +restart. Pin a version with `dequel update vX.Y.Z`. + +## Next steps +- Follow the [Quickstart](/docs/quickstart) to deploy your first app. +- Connect more machines under [Servers](/docs/servers). diff --git a/apps/docs/docs/keys.md b/apps/docs/docs/keys.md new file mode 100644 index 0000000..a331d17 --- /dev/null +++ b/apps/docs/docs/keys.md @@ -0,0 +1,44 @@ +--- +title: Keys & Tokens +category: Security +description: API tokens for automation and the SSH key pool for servers. +slug: keys +--- + +The **Keys** page is the credential control center: API tokens for +programmatic access, and the SSH key pool that server connections draw +from. + +![Keys page](/screenshots/keys.png) + +## API tokens + +Generate a token with a label and a scope preset (CI/CD pipeline, +monitoring and logs, or full admin). The `dql_...` token is shown +**once** — copy it immediately; afterwards only its hash prefix is +listed. Send it as a Bearer token: + +```bash +curl -H "Authorization: Bearer dql_" https://dequel.example.com/api/projects +``` + +Scopes are enforced per HTTP method (read vs write). Revoke a token +from the table when it leaks or is no longer needed. The **CLI guide** +tab shows copy-paste `curl` snippets and secret-handling practices. + +## SSH key pool + +Servers connected over SSH authenticate with a key from this pool, not +with per-server pastes. Add a PEM/OpenSSH private key (paste or upload +`.pem` / `.key`); Dequel detects Ed25519/RSA formats, fingerprints the +key, and rejects duplicates. Private material is never displayed again +— only name, fingerprint, tags, and date. + +When you add a server (**Settings → Servers**), pick its key from the +pool dropdown. Rotating a server's access means updating the pool entry +it points at. + +## Next steps + +- Read [Authentication](/docs/auth) for login sessions. +- Read [Servers](/docs/servers) to attach keys to machines. diff --git a/apps/docs/docs/observability.md b/apps/docs/docs/observability.md new file mode 100644 index 0000000..d777d01 --- /dev/null +++ b/apps/docs/docs/observability.md @@ -0,0 +1,49 @@ +--- +title: Observability +category: Observability +description: Metrics, request logs, and runtime logs for every project. +slug: observability +--- + +Each project has an **Observability** tab (metrics) and a **Logs** tab +(request + runtime logs). The dashboard also links out to the full +Grafana instance at `/grafana`. + +![Project observability](/screenshots/observability.png) + +## Metrics + +The Observability tab charts the last 6 hours from Prometheus (fed by +cAdvisor): memory, CPU %, network in/out per container, plus an edge +request-load chart from the API's own Loki-backed request matrix. Cards +refresh every 15 seconds. An empty state means nothing is deployed yet +— deploy to render telemetry. + +Behind the scenes, `GET /projects/:id/status` aggregates health over a +configurable window, and `GET /prometheus/query...` proxies arbitrary +PromQL for custom dashboards. + +## Request logs + +The Logs tab's **Request Monitoring** view tails the Caddy access log +for the project domain via Loki: method, path, status, latency, and a +30×1-minute histogram. Filter by date range (pauses the live tail), +full-text search, and level checkboxes, or click a row for the +slide-over detail. A live/paused toggle controls the SSE stream. + +## Runtime logs + +The **Runtime Logs** view follows the running container's stdout +(`docker logs -f`) over SSE — what your app actually printed, as +opposed to what the edge proxy saw. Same search and filter controls. + +## Grafana + +**Open Grafana** (sidebar, under Observability) jumps to the platform +Grafana with Prometheus and Loki data sources pre-wired, for ad-hoc +queries across all projects and infrastructure. + +## Next steps + +- Read [Alerts](/docs/alerts) to get notified on CPU, memory, or downtime. +- Read [Deployments](/docs/deployments) for build logs (as opposed to runtime logs). diff --git a/apps/docs/docs/quickstart.mdx b/apps/docs/docs/quickstart.mdx index 6ce6c57..3435e1d 100644 --- a/apps/docs/docs/quickstart.mdx +++ b/apps/docs/docs/quickstart.mdx @@ -1,50 +1,74 @@ --- title: Quickstart Guide category: Getting Started -description: Deploy your first containerized application to Dequel in under 5 minutes. +description: Deploy your first application to Dequel in under 5 minutes. slug: quickstart --- -In this guide, we'll configure a project, push the codebase, and verify that the container is running with active routing. +This guide takes you from an empty dashboard to a live HTTPS app. You +need a running Dequel installation and a login user (see +[Installation](/docs/installation)). No Dockerfile is required — Dequel +detects your framework automatically. -## Step 1: Create a Project +## Step 1: Create a project -Navigate to the Dequel dashboard (at `http://localhost` or your configured `CADDY_BASE_DOMAIN`) and click the **+ Create Project** button in the upper right corner. +Open the dashboard and click **New Project**. -- Enter a unique project name (e.g. `my-web-app`). -- Configure CPU limit boundaries (e.g., `0.5 cores`) and Memory bounds (e.g., `512MB`). -- Click **Create** to initialize the project namespace. +![Dequel dashboard](/screenshots/dashboard.png) -## Step 2: Add a Dockerfile +Fill in the wizard: -Dequel builds applications using a standard container blueprint. Ensure your project includes a valid `Dockerfile` in its root directory: +1. **Source** — pick **Git** (paste an HTTPS clone URL and branch, or + connect your GitHub account to pick a repo) or **Upload** (a `.zip` + of your code). +2. **Name** — a unique project name. It becomes your default URL: + `.`. +3. **Target server** — where the app runs. `local` (this machine) works + out of the box; other servers appear after you connect them (see + [Servers](/docs/servers)). +4. **Build strategy** — **Automatic (Railpack)** for a single service, + or **Docker Compose** if your repo root has a `docker-compose.yml`. +5. For Railpack builds, confirm the detected framework settings (install + and start commands, output directory, port). -```dockerfile Dockerfile -FROM node:18-alpine -WORKDIR /app -COPY package*.json ./ -RUN npm install -COPY . . -RUN npm run build -ENV PORT=3000 -EXPOSE 3000 -CMD ["npm", "run", "start"] -``` +![New project wizard](/screenshots/create-project.png) -:::tip[Custom Ports] -Dequel reads the `PORT` environment variable inside your container to automatically route incoming traffic. Ensure your app listens on the port specified by this variable. -::: +Click **Launch Deployment**. Dequel creates the project, links any +selected shared env vars, and immediately starts the first deployment. +## Step 2: Watch the build -## Step 3: Trigger the Deploy +The project page opens on the **Deployments** tab. Build logs stream +live as Railpack detects the framework, BuildKit builds the image, and +the container starts. Deployments move through `pending` → `building` → +`deploying` → `running` (or `failed` if something breaks). -On the deployments page, you can choose between connecting your GitHub repository or performing a local directory upload: +![Deployment build logs](/screenshots/deployments.png) -1. Enter your repository URL and set the branch to `main`. -2. Click **Deploy Branch**. -3. Follow the build stages in real-time under the **Active Build Logs** window. +If the build fails, click **Diagnose** in the log header to hand the +failure to an AI agent — see [AI Diagnosis](/docs/ai-diagnosis). -## Step 4: Verify Routing +## Step 3: Open your app -Once the deployment changes status from `BUILDING` to `READY`, click the external link button near your custom domain or the auto-generated base domain to see your running web container. +Once the deployment is `running`, click the live URL pill in the +project header. Locally that is +`http://.localhost`; with a base domain configured it is +`https://.` with a valid certificate. +## Step 4: Push updates + +- **Git projects:** push to the tracked branch and Dequel rebuilds + automatically (via the GitHub webhook), or click **Manual Deploy** to + rebuild from the latest commit or a specific SHA. +- **Upload projects:** drop a new archive on the Deployments tab. + +Previous builds keep their images, so you can **Rollback** to any +earlier deployment from the history table. + +## Next steps + +- Add [environment variables](/docs/env-vars) and link + [shared env vars](/docs/shared-env-vars). +- Attach a [custom domain](/docs/domains) with automatic SSL. +- Provision a [managed database](/docs/databases). +- Set [resource limits and autoscaling](/docs/scaling). diff --git a/apps/docs/docs/scaling.md b/apps/docs/docs/scaling.md index cafb7d8..21ccc2a 100644 --- a/apps/docs/docs/scaling.md +++ b/apps/docs/docs/scaling.md @@ -1,30 +1,49 @@ --- -title: Scaling Policies -category: Core Architecture -description: Adjust running container replicas automatically to handle high traffic and preserve cluster efficiency. +title: Scaling +category: Projects +description: Resource limits and CPU-based autoscaling for your containers. slug: scaling --- -Dequel supports two types of container instance scaling: **Manual Scaling** (fixed replica count) and **Auto-scaling** (load-based replica counts). +The project's **Scaling** tab holds two cards: resource limits and the +autoscaling policy. -## Manual Scaling +![Scaling tab](/screenshots/scaling.png) -Configure a static replica limit for your service (e.g. `3 replicas`). The orchestrator will verify that exactly three instances are running on cluster nodes. If an instance crashes, it will be automatically replaced immediately. +## Resource limits -## Auto-scaling Policies +Set per-container bounds that are enforced on every deployment: -Auto-scaling lets you scale compute bounds in response to real-time resource demands: +- **CPU limit (cores)** — for example `1.0` for one full core. +- **Memory limit (MB)** — for example `512`. A container that exceeds it + is killed with an OOM error and restarted. -- **Min Replicas:** The baseline floor of instances always running. -- **Max Replicas:** The maximum horizontal scale ceiling allowed under load. -- **Target CPU Utilization (%):** The load threshold trigger. If average CPU utilization exceeds this value (e.g. `70%`), new container replicas are provisioned. If the load falls below the target, containers are scaled down. +Autoscaling requires a CPU limit, because the engine scales on CPU +utilization. -
- ⚠️ Warning: Stateless Design Required -

To scale horizontally, your application container must be completely stateless. Avoid writing file records directly to the local filesystem; use persistent volumes or managed databases instead.

-
+## Autoscaling policy -## Configuration Modal +Click **Enable autoscaling** and configure: -To disable auto-scaling policies or switch back to static scaling, click **Delete Policy** in the scaling panel. Confirm the change in the custom Radix confirmation dialog to update the scaling configuration safely. +- **Min replicas** — the floor of instances always running (at least 1). +- **Max replicas** — the ceiling under load. +- **CPU threshold (%)** — when average CPU utilization across replicas + exceeds this for the evaluation window, Dequel adds a replica; when + load falls, it removes replicas down to the floor. +- **Cooldown** — quiet period between scaling actions so the system does + not flap. +Disable the policy any time to return to a single replica. The engine +evaluates policies continuously in the background +(`ALERT_EVAL_INTERVAL_MS` controls the platform-wide cadence). + +:::note[Stateless services scale best] +Horizontal replicas do not share local files. Keep uploads and caches on +[persistent volumes](/docs/volumes) or in [managed databases](/docs/databases) +so any replica can serve any request. +::: + +## Next steps + +- Read [Observability](/docs/observability) to watch CPU and memory. +- Read [Alerts](/docs/alerts) to get notified on resource pressure. diff --git a/apps/docs/docs/servers.md b/apps/docs/docs/servers.md new file mode 100644 index 0000000..ebd7f99 --- /dev/null +++ b/apps/docs/docs/servers.md @@ -0,0 +1,70 @@ +--- +title: Servers +category: Multi-Server +description: Run apps on multiple machines over SSH or WireGuard agents. +slug: servers +--- + +Dequel is not limited to the machine it is installed on. Connect extra +servers and pick a target per project or database: builds run there, +containers live there, and Caddy routes point at them. The control plane +itself is just the default `local` target. + +![Servers settings](/screenshots/servers.png) + +## Connection modes + +**Direct SSH (recommended).** The control plane shells into the server +and runs Docker commands remotely. The server needs outbound internet +and SSH reachable from the control plane — Docker, Caddy, BuildKit, and +Railpack are installed for you by the Prepare step. + +**WireGuard agent (firewalled nodes).** For servers with no open SSH +port, Dequel shows a `docker run` command for the Dequel agent. Run it +on the target: the agent registers back with a one-time token and opens +a WireGuard tunnel to the control plane. All orchestration then flows +through the mesh. + +## Adding a server + +Open **Settings → Servers** and click **Connect Server**: + +1. Choose **Direct SSH** (host, port, user, and an SSH key from your + [key pool](/docs/keys)) or **WireGuard P2P Agent** (a name — Dequel + mints the registration token). +2. Click **Prepare Node**. Dequel streams live progress: OS detection, + Docker install, Caddy + `/etc/caddy/routes/` setup, BuildKit, and + Railpack CLI. Prepare is idempotent — re-run it any time to heal a + node. +3. When the stream reports success, the server appears as a deployment + target in the project wizard and the database provisioner. + +Use **Test** to check connectivity without preparing, and remove a +server with the delete action when it is decommissioned. + +## Ingress and shared domains + +All servers share one base domain (`CADDY_BASE_DOMAIN`). A project gets +the same hostname (`.`) no matter which server runs it — +Dequel renders that hostname into the route file on the owning server. +Point the wildcard DNS at your servers (see [Installation](/docs/installation)), +or add per-deployment A records for finer control. Custom domains bypass +the base domain entirely. The **Settings → Servers** view (or +**Settings → Ingress** state) controls which server is the ingress +entrypoint; route internals are documented under +[Agent Caddy Routes](/docs/agent-caddy-routes). + +## Failover + +A project can fail over to an alternate server: the control plane +redeploys the project's Git source on the new target and removes the +old containers. Trigger it from the project API +(`POST /projects/:id/failover`). Automatic failover runs on the +`FAILOVER_MIN_INTERVAL_MS` cadence unless `FAILOVER_DISABLED` is set +(see [System Configuration](/docs/system-config)). + +## Next steps + +- Read [Keys](/docs/keys) for the SSH key pool servers draw from. +- Read [Agent Caddy Routes](/docs/agent-caddy-routes) for route files. +- Read [Deployments](/docs/deployments) for targeting a server per deploy. diff --git a/apps/docs/docs/settings.md b/apps/docs/docs/settings.md new file mode 100644 index 0000000..b49e105 --- /dev/null +++ b/apps/docs/docs/settings.md @@ -0,0 +1,58 @@ +--- +title: Platform Settings +category: Integrations +description: SMTP, ingress, backup storage, GitHub, servers, and AI keys. +slug: settings +--- + +**Settings** (sidebar) is where the platform itself is configured. A +banner counts registered nodes and flags integrations that still need +attention. + +## Servers + +Connect machines over SSH or WireGuard agents, prepare them with Docker +/ Caddy / BuildKit, and pick deployment targets. Documented in full +under [Servers](/docs/servers). + +## Backups + +Local-path or S3 storage plus the internal database schedule. See +[Backups](/docs/backups). + +## GitHub + +OAuth app credentials, connect/disconnect, and the callback URL. See +[GitHub](/docs/github). + +## SMTP + +The relay for alert emails and failure notifications: host, port, user, +password (encrypted at rest), and sender address, with presets for +Resend, SendGrid, Postmark, Mailgun, and AWS SES. **Send Test Email** +verifies the setup before you depend on it. + +Without SMTP, email-channel [alert rules](/docs/alerts) cannot deliver +— Slack and webhook channels work independently. + +## AI + +Provider keys for [AI Diagnosis](/docs/ai-diagnosis): OpenAI, Anthropic, +Gemini, Groq, self-hosted Ollama, or any custom OpenAI-compatible +endpoint (base URL). Keys are AES-256-GCM encrypted and never shown +after saving; leaving the field blank keeps the stored key. **Save & +Sync Models** pulls the provider's model catalog so the diagnosis sheet +offers real model names. + +![AI provider settings](/screenshots/settings-ai.png) + +## Ingress + +The ingress view shows which server terminates public traffic and the +failover state. App hostnames stay identical across servers regardless +(see [Servers](/docs/servers)). + +## Next steps + +- Read [System Configuration](/docs/system-config) for env/file settings. +- Read [Alerts](/docs/alerts) to wire SMTP into notifications. diff --git a/apps/docs/docs/shared-env-vars.md b/apps/docs/docs/shared-env-vars.md new file mode 100644 index 0000000..f541d62 --- /dev/null +++ b/apps/docs/docs/shared-env-vars.md @@ -0,0 +1,40 @@ +--- +title: Shared Env Vars +category: Projects +description: One secret library linked into many projects. +slug: shared-env-vars +--- + +**Shared Env** is a global variable library. Define a value once — a +third-party API key, a common feature flag, a shared endpoint — and +link it into any number of projects instead of pasting it everywhere. +Rotating the shared value updates every linked project on its next +redeploy. + +![Shared env vars](/screenshots/shared-env.png) + +## Creating shared variables + +Open **Shared Env** in the sidebar and add a variable with a key, +value, description, and optional tags. Values are encrypted at rest and +masked in the UI, exactly like project variables — reveal or copy them +per row. + +## Linking into projects + +Link variables from the project's **Env Variables** tab (linked section) +or while creating a project (the env step lists the library). Linking +is by reference: editing the shared value changes what linked projects +receive next time they start. + +When a project-level variable and a shared variable share a key, the +**project value wins**. Use that for per-project overrides of a shared +default. + +Unlink any time from the same section; the project keeps its own +variables untouched. + +## Next steps + +- Read [Environment Variables](/docs/env-vars) for project-level config. +- Read [Deployments](/docs/deployments) to roll out rotated values. diff --git a/apps/docs/docs/ssl.mdx b/apps/docs/docs/ssl.mdx index f788b35..eeeb3bc 100644 --- a/apps/docs/docs/ssl.mdx +++ b/apps/docs/docs/ssl.mdx @@ -1,34 +1,43 @@ --- title: SSL Certificates -category: Networking & Security -description: Secure external traffic automatically with Let's Encrypt TLS/SSL certificates. +category: Security +description: Automatic HTTPS via Let's Encrypt for every routed domain. slug: ssl --- -Dequel enforces HTTPS encryption for all public endpoints. The cluster edge routing layer manages keys, handshakes, and automatic certificate acquisitions behind the scenes. +Dequel serves every routed hostname over HTTPS with certificates from +**Let's Encrypt**. There is nothing to buy, upload, or renew manually. -## How Certificates are Provisioned +## How provisioning works -Dequel interfaces directly with **Let's Encrypt** (a free, automated, open Certificate Authority): +1. A project domain (default `.` or a verified + [custom domain](/docs/domains)) appears in a Caddy route file. +2. On first request, Caddy completes an ACME HTTP-01 challenge and + obtains a certificate. +3. Caddy stores the certificate and renews it automatically before the + 90-day expiry. -1. When a custom domain is successfully verified, Dequel triggers Caddy's ACME certificate client. -2. Caddy performs an HTTP-01 challenge verification with Let's Encrypt servers. -3. Upon challenge completion, Let's Encrypt signs a valid SSL certificate. -4. Caddy saves the certificate to the cluster storage path and starts enforcing TLS/HTTPS handshakes. +For this to work, the domain must resolve to the server and ports `80` +and `443` must be reachable — the challenge runs over plain HTTP. If +you terminate TLS in front of Dequel (for example Cloudflare proxy), +use DNS-01 challenges or pause proxying until the first certificate is +issued. -## Automatic Renewals +## Local development -Let's Encrypt certificates are valid for 90 days. Dequel's router checks certificates once daily and schedules automatic renewals 30 days before expiration, ensuring zero downtime and avoiding manual keys management. +On `localhost` there is no public CA involved: Caddy serves plain HTTP +for `.localhost` URLs, and the dashboard is available over both +HTTP and HTTPS when a base domain is configured. -:::note[Key Security Spec] -Dequel configures modern TLS 1.3 as the default cipher suite block, providing maximum security, fast TLS handshakes, and strict protection against legacy protocol exploits. -::: +## Troubleshooting +- **Domain shows TLS pending** — confirm DNS resolves to the server IP + and that no CDN is intercepting `/.well-known/acme-challenge/`. +- **Certificate errors on managed databases** — database TLS uses the + gateway's own certificate, not Let's Encrypt. See + [Managed Databases](/docs/databases) for the client flags. -## Troubleshooting SSL - -If your domain status is stuck on `PENDING_SSL`: - -- Verify that your DNS CNAME/A records have fully propagated. -- Ensure that no CDN service (e.g. Cloudflare proxy) is blocking HTTP ACME challenge endpoints (`/.well-known/acme-challenge/`). +## Next steps +- Read [Custom Domains](/docs/domains) to add domains. +- Read [Managed Databases](/docs/databases) for database TLS behavior. diff --git a/apps/docs/docs/system-config.md b/apps/docs/docs/system-config.md index e258942..4784726 100644 --- a/apps/docs/docs/system-config.md +++ b/apps/docs/docs/system-config.md @@ -1,82 +1,106 @@ --- title: System Configuration -category: Core Architecture -description: Configure the Dequel platform via environment variables or the dequel.json config file. +category: Platform +description: Environment variables and the dequel.json file for platform settings. slug: system-config --- -Dequel's platform-level settings are configured through environment variables or a `dequel.json` config file. Environment variables take precedence over file values. +The API reads platform settings from environment variables first, then +from a `dequel.json` file. Environment variables always win. -## Config File Location +## Config file location -Dequel looks for `dequel.json` in the following order: +The API loads the first `dequel.json` it finds: -1. `DEQUEL_CONFIG` environment variable pointing to an explicit path -2. `~/.config/dequel/dequel.json` -3. `./dequel.json` (next to the binary) -4. `./data/dequel.json` +1. The path in `DEQUEL_CONFIG` +2. `$XDG_CONFIG_HOME/dequel/dequel.json` (usually + `~/.config/dequel/dequel.json`) +3. `./dequel.json` (next to the process working directory) +4. `./data/dequel.json` (what the installer writes, mode `0600`) + +The installer generates `data/dequel.json` with your base domain and a +random `ENV_ENCRYPTION_KEY`. ## Reference -### Security +### Identity and domains -| Variable | Default | Description | -|----------|---------|-------------| -| `ENV_ENCRYPTION_KEY` | `dev-env-key-change-me` | Key used to encrypt environment variable values and SMTP passwords at rest | +| Variable | Default | Source | Description | +|----------|---------|--------|-------------| +| `CADDY_BASE_DOMAIN` | `localhost` | env or file | Base domain for app URLs (`.`). Set a real domain with wildcard DNS for auto-SSL. | +| `CADDY_EMAIL` | _(empty)_ | env (Caddy) | Contact for Let's Encrypt notifications. Read by Caddy, not the API. | +| `CONTROL_PLANE_URL` | _(empty)_ | env or file | Public URL of this control plane, used for links and agent registration. | +| `DATABASE_URL` | _(empty)_ | env | PostgreSQL connection string. Empty fails fast with a clear error. | -### GitHub Integration +### Secrets -Set these to enable the repo picker in the project creation dialog: +| Variable | Default | Source | Description | +|----------|---------|--------|-------------| +| `ENV_ENCRYPTION_KEY` | `dev-env-key-change-me` | env | Encrypts env vars, SMTP passwords, and LLM keys at rest. The API **refuses to start in production** with the default — set a unique value. | + +### Background engines | Variable | Default | Description | |----------|---------|-------------| -| `GITHUB_CLIENT_ID` | `""` | GitHub OAuth App client ID | -| `GITHUB_CLIENT_SECRET` | `""` | GitHub OAuth App client secret | -| `GITHUB_APP_NAME` | `Dequel` | GitHub App name displayed during OAuth | -| `GITHUB_WEBHOOK_SECRET` | `""` | Secret for GitHub webhook verification | - -Config file equivalent: - -```json -{ - "githubClientId": "Iv1...", - "githubClientSecret": "...", - "githubAppName": "Dequel", - "githubWebhookSecret": "..." -} -``` - -On boot, these values seed the `github_integrations` table. You can also update them from the Settings page in the dashboard. +| `QUEUE_CONCURRENCY` | `3` | Parallel jobs (builds, provisions, failovers). | +| `QUEUE_RETRY_MAX` | `5` | Retries per failed job. | +| `QUEUE_RETRY_BASE_MS` | `5000` | Base backoff between retries. | +| `ALERT_EVAL_INTERVAL_MS` | `60000` | Alert-rule evaluation cadence. | +| `FAILURE_SWEEP_INTERVAL_MS` | `60000` | Failed-deployment sweep cadence. | +| `FAILOVER_DISABLED` | _(empty)_ | Set to disable automatic failover. | +| `FAILOVER_MIN_INTERVAL_MS` | `600000` | Minimum time between failovers. | -### SMTP +### Observability -SMTP settings are not read from environment variables or the config file. Configure them from the Settings page in the dashboard, where you can also send a test email to verify the setup. The password is encrypted at rest using `ENV_ENCRYPTION_KEY`. +| Variable | Default | Description | +|----------|---------|-------------| +| `GRAFANA_URL` | `http://grafana:3000` | Grafana origin for dashboard links. | +| `GRAFANA_USER` / `GRAFANA_PASS` | `admin` / `admin` | Grafana credentials. | -### Ingress +### Agent networking | Variable | Default | Description | |----------|---------|-------------| -| `CADDY_BASE_DOMAIN` | `localhost` | Base domain for deployment subdomains. Deployed apps are reachable at `https://.`. GitHub webhook URLs are derived from this automatically. Set to a real domain (e.g. `example.com`) and configure wildcard DNS for auto-SSL. | -| `CADDY_EMAIL` | `""` | Email address for Let's Encrypt SSL certificate expiration notifications | +| `AGENT_TUNNEL_URL` | _(empty)_ | Tunnel endpoint agents dial back on. | +| `WIREGUARD_SERVER_CONTAINER` | _(empty)_ | WireGuard server container name. | +| `WIREGUARD_SERVER_PUBLIC_KEY` | _(empty)_ | Server public key for agent peers. | +| `WIREGUARD_SERVER_ENDPOINT` | _(empty)_ | Public endpoint agents connect to. | +| `WIREGUARD_SERVER_IP` | `10.200.0.1` | Server address inside the mesh. | +| `WIREGUARD_PEER_CIDR` | `10.200.0.0/24` | Agent address range. | -Config file equivalent: +### Telemetry -```json -{ - "caddyBaseDomain": "example.com", - "caddyEmail": "admin@example.com" -} +Dequel collects anonymous usage telemetry (PostHog) to guide +development. Disable it with either variable: + +```bash +DEQUEL_TELEMETRY_DISABLED=1 +DO_NOT_TRACK=1 ``` -## Full Config File Example +### What is configured in the dashboard instead + +These have no environment equivalent — set them under +[Settings](/docs/settings): + +- GitHub OAuth app (client ID/secret, webhook secret) +- SMTP relay and sender address +- Backup storage (local path or S3) +- LLM provider keys and models +- Ingress server selection + +## Full file example ```json { - "githubClientId": "Iv1...", - "githubClientSecret": "...", - "githubAppName": "MyDequel", - "githubWebhookSecret": "...", - "envEncryptionKey": "your-secure-key-here" + "CADDY_BASE_DOMAIN": "example.com", + "CONTROL_PLANE_URL": "https://dequel.example.com", + "QUEUE_CONCURRENCY": 3, + "FAILOVER_MIN_INTERVAL_MS": 600000 } ``` +## Next steps + +- Read [Settings](/docs/settings) for dashboard-managed config. +- Read [Servers](/docs/servers) for multi-server networking. diff --git a/apps/docs/docs/volumes.mdx b/apps/docs/docs/volumes.mdx index 9dde140..1cb7828 100644 --- a/apps/docs/docs/volumes.mdx +++ b/apps/docs/docs/volumes.mdx @@ -1,28 +1,43 @@ --- title: Persistent Volumes -category: Cluster Storage & Data -description: Mount persistent cluster disk pathways inside container runtimes to preserve storage across updates. +category: Projects +description: Keep data across deployments with mounted Docker volumes. slug: volumes --- -Dequel container instances are ephemeral by default. When a new deployment occurs, the old container is deleted, taking all local filesystem changes with it. +Containers are ephemeral: each new deployment replaces the old +container, and anything written to its local filesystem disappears with +it. Mount a **persistent volume** for uploads, caches, SQLite files, or +any data that must survive redeploys. -To store uploaded assets, databases, or local file caches, mount a **Persistent Volume**. +## Adding a volume -## Creating a Volume Mount +Open the project and switch to the **Volumes** tab, then click **Add +Volume**: -Under the **Volumes** tab, click **Add Volume**: +1. Enter the mount path inside the container (for example + `/app/uploads`). +2. Save. Dequel creates a Docker volume on the deployment's server and + records the mount. -1. Enter a mount path inside the container environment (e.g. `/app/uploads`). -2. Click **Add Volume**. -3. Dequel allocates block storage space on the cluster nodes and maps it to your container's target folder. +Mount changes apply on the next container start. The tab shows a +redeploy banner after you add or delete a volume — click **Redeploy** to +roll it out. Deleting a volume unmounts it; the volume record can be +removed separately. -:::note[Redeployment Requirement] -Adding or deleting volumes modifies the low-level container host bindings. To apply these mount pathways, you must trigger a container restart via the **Redeploy Now** prompt banner at the top of the volumes tab. +:::note[Volumes live on the deployment's server] +Volumes are created on whichever server runs the deployment. If you move +a project to another server (see [Servers](/docs/servers)), its data +does not move with it — plan a copy step for stateful moves. ::: +## Databases vs volumes -## Data Security & Backups +- Use a [managed database](/docs/databases) for structured data with + querying, credentials, backups, and remote access. +- Use a volume for flat files your app reads and writes directly. -Because data is tied directly to cluster block volumes, data persists even if you update environment variables, scale container replicas, or deploy code rollbacks. +## Next steps +- Read [Managed Databases](/docs/databases) for the database option. +- Read [Backups](/docs/backups) for snapshot and restore flows. diff --git a/apps/docs/pages/index.astro b/apps/docs/pages/index.astro index 7f356c6..497a883 100644 --- a/apps/docs/pages/index.astro +++ b/apps/docs/pages/index.astro @@ -43,99 +43,87 @@ import CTA from "../components/CTA.astro"; Dequel — Self-Hosted PaaS Engine & Container Platform - + + +
@@ -146,82 +134,27 @@ import CTA from "../components/CTA.astro";
-