diff --git a/apps/dokploy/__test__/server/docker-script-fakes.ts b/apps/dokploy/__test__/server/docker-script-fakes.ts new file mode 100644 index 00000000000..923e24e5f46 --- /dev/null +++ b/apps/dokploy/__test__/server/docker-script-fakes.ts @@ -0,0 +1,156 @@ +import { spawnSync } from "node:child_process"; +import { PINNED_VERSIONS } from "@dokploy/server/setup/component-versions"; + +// Stand-ins for the host tools the generated update scripts call. Each one +// records its arguments in $CALL_LOG and keeps Docker's state in $FAKE_STATE. +export const BASH = spawnSync("sh", ["-c", "command -v bash"], { + encoding: "utf8", +}).stdout.trim(); + +export const FAKE_TOOLS: Record = { + docker: `#!/bin/sh +printf 'docker %s\\n' "$*" >> "$CALL_LOG" +case "$1" in + pull) [ "$FAKE_PULL_FAILS" = 1 ] && exit 1 ;; + create) + [ "$FAKE_CREATE_FAILS" = 1 ] && exit 1 + for traefik_arg in "$@"; do traefik_image_arg="$traefik_arg"; done + printf '%s\\n' "$traefik_image_arg" > "$FAKE_STATE/traefik-image" + ;; + version) cat "$FAKE_STATE/engine" ;; + info) echo "\${FAKE_SWARM_STATE:-active}" ;; + ps) echo cid ;; + exec) + case "$*" in + *redis-cli*) echo PONG ;; + esac + ;; + network) + case "$2" in + inspect) case " $FAKE_MISSING_NETWORKS " in *" $3 "*) exit 1 ;; esac ;; + connect) case " $FAKE_CONNECT_FAILS " in *" $3 "*) exit 1 ;; esac ;; + esac + ;; + inspect) + case "$*" in + *State.Health*) echo "\${FAKE_PANEL_HEALTH:-healthy}" ;; + *Config.Image*) echo "$FAKE_OLD_IMAGE" ;; + *NetworkSettings*) echo "$FAKE_TRAEFIK_NETWORKS" ;; + *PortBindings*) echo "$FAKE_PORT_BINDINGS" ;; + *RestartCount*) + traefik_now="$(cat "$FAKE_STATE/traefik-image" 2>/dev/null || echo "$FAKE_OLD_IMAGE")" + if [ "$FAKE_NOT_RUNNING" = 1 ] && [ "$traefik_now" != "$FAKE_OLD_IMAGE" ]; then echo "false 0"; else echo "true 0"; fi + ;; + *State.Running*) + traefik_now="$(cat "$FAKE_STATE/traefik-image" 2>/dev/null || echo "$FAKE_OLD_IMAGE")" + if [ "$FAKE_NOT_RUNNING" = 1 ] && [ "$traefik_now" != "$FAKE_OLD_IMAGE" ]; then echo false; else echo true; fi + ;; + esac + ;; +esac +exit 0 +`, + "apt-get": `#!/bin/sh +printf 'apt-get %s\\n' "$*" >> "$CALL_LOG" +case "$1" in + update) + [ "$FAKE_UPDATE_FAILS" = 1 ] && exit 1 + ;; + download) + [ "$FAKE_DOWNLOAD_FAILS" = 1 ] && exit 1 + printf 'fake package\\n' > "\${2%%=*}.deb" + ;; + install) + case "$*" in + *--download-only*) + [ "$FAKE_PREDOWNLOAD_FAILS" = 1 ] && exit 1 + ;; + *--allow-downgrades*) + [ "$FAKE_ROLLBACK_FAILS" = 1 ] && exit 1 + printf '%s\\n' "$FAKE_PREVIOUS_ENGINE" > "$FAKE_STATE/engine" + ;; + *--no-download*) + if [ "$FAKE_UPGRADE_KEEPS_OLD" != 1 ]; then + printf '%s\\n' "$FAKE_TARGET_ENGINE" > "$FAKE_STATE/engine" + fi + ;; + esac + ;; +esac +exit 0 +`, + "apt-cache": `#!/bin/sh +printf 'apt-cache %s\\n' "$*" >> "$CALL_LOG" +case "$1" in + madison) + [ "$FAKE_NO_CANDIDATE" = 1 ] && exit 0 + if [ "$FAKE_CANDIDATE_AFTER_REPO" = 1 ] && [ ! -f "$DOKPLOY_APT_SOURCES_DIR/docker.list" ]; then + exit 0 + fi + echo " docker-ce | 5:${PINNED_VERSIONS.docker}-1~ubuntu.24.04~noble | https://download.docker.com/linux/ubuntu noble/stable amd64 Packages" + ;; + policy) + case "$2" in + containerd.io) + candidate=1.7.27-1 + if [ "$FAKE_CONTAINERD_CHANGES" = 1 ]; then candidate=2.0.0-1; fi + printf 'containerd.io:\\n Installed: 1.7.27-1\\n Candidate: %s\\n' "$candidate" + ;; + docker-buildx-plugin) + printf 'docker-buildx-plugin:\\n Installed: 0.25.0-1~ubuntu.24.04~noble\\n Candidate: 0.25.0-1~ubuntu.24.04~noble\\n' + ;; + docker-compose-plugin) + printf 'docker-compose-plugin:\\n Installed: 2.40.0-1~ubuntu.24.04~noble\\n Candidate: 2.40.0-1~ubuntu.24.04~noble\\n' + ;; + esac + ;; +esac +exit 0 +`, + "dpkg-query": `#!/bin/sh +printf 'dpkg-query %s\\n' "$*" >> "$CALL_LOG" +case "$2" in + *Status-Status*) + case "$3" in + docker-ce) [ "$FAKE_NO_DOCKER_CE" = 1 ] || echo installed ;; + docker-ce-cli | containerd.io | docker-buildx-plugin | docker-compose-plugin) echo installed ;; + esac + ;; + *Version*) + case "$3" in + docker-ce | docker-ce-cli | docker-ce-rootless-extras) echo "5:28.3.0-1~ubuntu.24.04~noble" ;; + containerd.io) echo "1.7.27-1" ;; + docker-buildx-plugin) echo "0.25.0-1~ubuntu.24.04~noble" ;; + docker-compose-plugin) echo "2.40.0-1~ubuntu.24.04~noble" ;; + esac + ;; +esac +exit 0 +`, + dpkg: `#!/bin/sh +echo amd64 +`, + systemctl: `#!/bin/sh +printf 'systemctl %s\\n' "$*" >> "$CALL_LOG" +`, + curl: `#!/bin/sh +printf 'curl %s\\n' "$*" >> "$CALL_LOG" +if [ -n "$FAKE_CURL_FAILS_FOR" ]; then + case "$*" in + *"$FAKE_CURL_FAILS_FOR"*) exit 22 ;; + esac +fi +out="" +while [ "$#" -gt 0 ]; do + if [ "$1" = "-o" ]; then out="$2"; fi + shift +done +printf '%s\\n' 'echo fake get-docker "$@"' > "$out" +`, + sudo: `#!/bin/sh +exec "$@" +`, + sleep: `#!/bin/sh +exit 0 +`, +}; diff --git a/apps/dokploy/__test__/server/host-docker-upgrade.test.ts b/apps/dokploy/__test__/server/host-docker-upgrade.test.ts new file mode 100644 index 00000000000..3b53e1f2132 --- /dev/null +++ b/apps/dokploy/__test__/server/host-docker-upgrade.test.ts @@ -0,0 +1,339 @@ +import { spawnSync } from "node:child_process"; +import { + chmodSync, + existsSync, + mkdirSync, + mkdtempSync, + readFileSync, + rmSync, + statSync, + utimesSync, + writeFileSync, +} from "node:fs"; +import { tmpdir } from "node:os"; +import path from "node:path"; +import { PINNED_VERSIONS } from "@dokploy/server/setup/component-versions"; +import { + buildDockerUpgradeLauncherArgs, + buildHostDockerUpgradeScript, + DOCKER_UPGRADE_UNIT, + readHostDockerUpgradeStatus, + startHostDockerUpgrade, +} from "@dokploy/server/setup/host-docker-upgrade"; +import { afterEach, describe, expect, it } from "vitest"; +import { BASH, FAKE_TOOLS } from "./docker-script-fakes"; + +const PREVIOUS_ENGINE = "28.3.0"; + +const tempDirs: string[] = []; + +const makeTempDir = () => { + const dir = mkdtempSync(path.join(tmpdir(), "host-docker-upgrade-")); + tempDirs.push(dir); + return dir; +}; + +afterEach(() => { + for (const dir of tempDirs.splice(0)) { + rmSync(dir, { force: true, recursive: true }); + } +}); + +const DOCKER_LIST_NOBLE = + "deb [arch=amd64 signed-by=/etc/apt/keyrings/docker.asc] https://download.docker.com/linux/ubuntu noble stable\n"; + +const runHostScript = (env: Record = {}) => { + const dir = makeTempDir(); + const updates = path.join(dir, "host-updates"); + const sources = path.join(dir, "sources"); + const callLog = path.join(dir, "calls.log"); + mkdirSync(updates); + mkdirSync(sources); + writeFileSync(path.join(sources, "docker.list"), DOCKER_LIST_NOBLE); + writeFileSync( + path.join(dir, "os-release"), + "ID=ubuntu\nVERSION_CODENAME=noble\n", + ); + writeFileSync(path.join(dir, "engine"), `${PREVIOUS_ENGINE}\n`); + writeFileSync(callLog, ""); + for (const [name, body] of Object.entries(FAKE_TOOLS)) { + writeFileSync(path.join(dir, name), body); + chmodSync(path.join(dir, name), 0o755); + } + const result = spawnSync(BASH, [], { + input: buildHostDockerUpgradeScript(updates), + encoding: "utf8", + env: { + ...process.env, + PATH: `${dir}:${process.env.PATH}`, + CALL_LOG: callLog, + FAKE_STATE: dir, + FAKE_PREVIOUS_ENGINE: PREVIOUS_ENGINE, + FAKE_TARGET_ENGINE: PINNED_VERSIONS.docker, + FAKE_OLD_IMAGE: "traefik:v3.7.16", + TMPDIR: dir, + DOKPLOY_BACKUP_DIR: path.join(dir, "backups"), + DOKPLOY_OS_RELEASE: path.join(dir, "os-release"), + DOKPLOY_APT_SOURCES_DIR: sources, + DOKPLOY_DOCKER_KEYRING: path.join(dir, "keyrings", "docker.asc"), + DOKPLOY_DOCKER_VERIFY_TIMEOUT: "3", + ...env, + }, + }); + const statusFile = path.join(updates, "docker-engine-upgrade.json"); + return { + status: result.status, + stdout: result.stdout, + log: readFileSync(path.join(updates, "docker-engine-upgrade.log"), "utf8"), + stored: existsSync(statusFile) + ? JSON.parse(readFileSync(statusFile, "utf8")) + : null, + calls: readFileSync(callLog, "utf8").split("\n").filter(Boolean), + engine: readFileSync(path.join(dir, "engine"), "utf8").trim(), + }; +}; + +// Each test runs the generated script in a real shell. +describe("buildHostDockerUpgradeScript", { timeout: 60_000 }, () => { + it("generates valid bash", () => { + const result = spawnSync("bash", ["-n"], { + input: buildHostDockerUpgradeScript(), + }); + + expect(result.status, result.stderr?.toString()).toBe(0); + }); + + it("upgrades Docker, waits for the panel, Postgres and Redis, and records success", () => { + const run = runHostScript(); + + expect(run.status, run.log).toBe(0); + expect(run.stdout).toBe(""); + expect(run.engine).toBe(PINNED_VERSIONS.docker); + expect(run.stored).toMatchObject({ + state: "succeeded", + from: PREVIOUS_ENGINE, + to: PINNED_VERSIONS.docker, + exitCode: 0, + }); + expect(run.stored.finishedAt).toMatch(/^\d{4}-\d{2}-\d{2}T/); + expect(run.log).toContain( + `Docker Engine upgrade started on the host at ${run.stored.startedAt}: ${PREVIOUS_ENGINE} -> ${PINNED_VERSIONS.docker}`, + ); + expect(run.log).toContain("Docker Engine upgrade finished ✅"); + expect(run.calls).toContain( + "docker info --format {{.Swarm.LocalNodeState}}", + ); + expect( + run.calls.some((call) => + call.startsWith("docker exec cid pg_isready -U dokploy -d dokploy"), + ), + ).toBe(true); + expect(run.calls).toContain("docker exec cid redis-cli ping"); + expect(run.calls.some((call) => call.includes("--allow-downgrades"))).toBe( + false, + ); + }); + + it("rolls Docker back and records the failure when the panel does not come back healthy", () => { + const run = runHostScript({ FAKE_PANEL_HEALTH: "starting" }); + + expect(run.status).not.toBe(0); + expect(run.engine).toBe(PREVIOUS_ENGINE); + expect(run.calls.some((call) => call.includes("--allow-downgrades"))).toBe( + true, + ); + expect(run.log).toContain( + `Error: Docker ${PINNED_VERSIONS.docker} did not come up healthy after the install.`, + ); + expect(run.log).toContain( + `Not ready on Docker ${PINNED_VERSIONS.docker} after 3 seconds: the panel container is starting`, + ); + expect(run.log).toContain(`Rolling Docker back to ${PREVIOUS_ENGINE}`); + expect(run.stored).toMatchObject({ + state: "failed", + from: PREVIOUS_ENGINE, + exitCode: 1, + }); + }); + + it("accepts a panel container without a health check", () => { + const run = runHostScript({ FAKE_PANEL_HEALTH: "none" }); + + expect(run.status, run.log).toBe(0); + expect(run.stored.state).toBe("succeeded"); + }); + + it("fails without touching Docker when the pinned version is not in apt", () => { + const run = runHostScript({ FAKE_NO_CANDIDATE: "1" }); + + expect(run.status).not.toBe(0); + expect(run.engine).toBe(PREVIOUS_ENGINE); + expect(run.log).toContain("Error: Docker not upgraded:"); + expect(run.log).toContain("Nothing was changed."); + expect(run.calls.some((call) => call.includes("--no-download"))).toBe( + false, + ); + expect(run.stored).toMatchObject({ state: "failed", exitCode: 1 }); + }); +}); + +describe("buildDockerUpgradeLauncherArgs", () => { + it("starts the script as a host systemd unit from a privileged one-off container", () => { + const args = buildDockerUpgradeLauncherArgs( + "ghcr.io/agenthits/dokploy:agenthits-dev@sha256:abc", + "/etc/dokploy/host-updates", + ); + const hostCommand = args.at(-1) ?? ""; + + expect(args.slice(0, 9)).toEqual([ + "run", + "--rm", + "--pull=never", + "--privileged", + "--pid=host", + "--network=host", + "--user=0:0", + "--label=com.dokploy.host-update=docker-engine", + "--entrypoint=nsenter", + ]); + expect(args).toContain( + "ghcr.io/agenthits/dokploy:agenthits-dev@sha256:abc", + ); + expect(args).toContain("--target=1"); + expect(hostCommand).toContain( + `exec systemd-run --quiet --collect --unit=${DOCKER_UPGRADE_UNIT}`, + ); + expect(hostCommand).toContain( + "/bin/bash '/etc/dokploy/host-updates/docker-engine-upgrade.sh'", + ); + expect(spawnSync("sh", ["-n"], { input: hostCommand }).status).toBe(0); + }); + + it("stops before anything runs when the host cannot see the script the panel wrote", () => { + const dir = makeTempDir(); + const hostCommand = + buildDockerUpgradeLauncherArgs("image", dir).at(-1) ?? ""; + + const missingScript = spawnSync("sh", ["-c", hostCommand], { + encoding: "utf8", + }); + expect(missingScript.status).toBe(4); + expect(missingScript.stderr).toContain("not mounted from the host"); + }); +}); + +describe("startHostDockerUpgrade", () => { + it("writes the script, a starting status and a fresh log, then launches with the panel image", async () => { + const dir = path.join(makeTempDir(), "host-updates"); + const launches: string[][] = []; + + await startHostDockerUpgrade("29.8.2", { + dir, + readImage: async () => "panel-image:test", + launch: async (args) => { + launches.push(args); + }, + }); + + expect(launches).toHaveLength(1); + expect(launches[0]).toContain("panel-image:test"); + expect( + statSync(path.join(dir, "docker-engine-upgrade.sh")).mode & 0o777, + ).toBe(0o700); + expect(statSync(dir).mode & 0o777).toBe(0o700); + const status = await readHostDockerUpgradeStatus(dir); + expect(status).toMatchObject({ + state: "starting", + from: "29.8.2", + to: PINNED_VERSIONS.docker, + finishedAt: null, + exitCode: null, + }); + expect(status?.log).toContain( + `Starting the Docker Engine upgrade to ${PINNED_VERSIONS.docker} from the panel`, + ); + }); + + it("records a failed start with the launcher's error and nothing changed", async () => { + const dir = path.join(makeTempDir(), "host-updates"); + const failure = Object.assign(new Error("exit 3"), { + stderr: + "This host does not run systemd, so the upgrade cannot run outside Docker from the panel.\n", + }); + + await expect( + startHostDockerUpgrade("29.8.2", { + dir, + readImage: async () => "panel-image:test", + launch: async () => { + throw failure; + }, + }), + ).rejects.toThrow("This host does not run systemd"); + + const status = await readHostDockerUpgradeStatus(dir); + expect(status?.state).toBe("failed"); + expect(status?.finishedAt).not.toBeNull(); + expect(status?.log).toContain("Nothing was changed."); + }); +}); + +describe("readHostDockerUpgradeStatus", () => { + const writeStatus = (dir: string, state: string) => { + mkdirSync(dir, { recursive: true }); + writeFileSync( + path.join(dir, "docker-engine-upgrade.json"), + JSON.stringify({ + state, + from: "29.8.2", + to: "29.9.2", + startedAt: "2026-10-29T10:00:00Z", + finishedAt: null, + exitCode: null, + }), + ); + writeFileSync(path.join(dir, "docker-engine-upgrade.log"), "line\n"); + }; + + it("returns null before the first run and for an unreadable status", async () => { + const dir = makeTempDir(); + expect(await readHostDockerUpgradeStatus(dir)).toBeNull(); + + writeFileSync(path.join(dir, "docker-engine-upgrade.json"), "{broken"); + expect(await readHostDockerUpgradeStatus(dir)).toBeNull(); + }); + + it("reports a running upgrade that stopped writing for half an hour as interrupted", async () => { + const dir = makeTempDir(); + writeStatus(dir, "running"); + const now = Date.now(); + + expect((await readHostDockerUpgradeStatus(dir, now))?.state).toBe( + "running", + ); + + const old = new Date(now - 31 * 60 * 1000); + for (const name of [ + "docker-engine-upgrade.json", + "docker-engine-upgrade.log", + ]) { + utimesSync(path.join(dir, name), old, old); + } + expect((await readHostDockerUpgradeStatus(dir, now))?.state).toBe( + "interrupted", + ); + }); + + it("keeps only the tail of a long log", async () => { + const dir = makeTempDir(); + writeStatus(dir, "succeeded"); + writeFileSync( + path.join(dir, "docker-engine-upgrade.log"), + `${"x".repeat(70 * 1024)}last line\n`, + ); + + const status = await readHostDockerUpgradeStatus(dir); + expect(status?.log.length).toBe(64 * 1024); + expect(status?.log.endsWith("last line\n")).toBe(true); + }); +}); diff --git a/apps/dokploy/__test__/server/host-stack-rows.test.ts b/apps/dokploy/__test__/server/host-stack-rows.test.ts index a3c303e44c3..540152dc8aa 100644 --- a/apps/dokploy/__test__/server/host-stack-rows.test.ts +++ b/apps/dokploy/__test__/server/host-stack-rows.test.ts @@ -53,14 +53,14 @@ describe("buildHostStackRows", () => { } }); - it("keeps the reason of the maintenance-window steps even when they are current", () => { + it("keeps the downtime and maintenance-window reasons even when the rows are current", () => { const rows = rowsFor({}); expect(rowNamed(rows, "docker")).toMatchObject({ - action: "manual", + action: "docker", command: "DOCKER_ENGINE_UPGRADE=1 bash install-agenthits.sh update", }); - expect(rowNamed(rows, "docker").reason).toContain("maintenance window"); + expect(rowNamed(rows, "docker").reason).toContain("restarts Docker"); expect(rowNamed(rows, "postgres")).toMatchObject({ action: "manual", command: null, @@ -83,17 +83,17 @@ describe("buildHostStackRows", () => { } }); - it("marks Docker Engine behind its pin as a maintenance-window step with the engine upgrade command", () => { + it("offers the panel's own upgrade for Docker Engine behind its pin and keeps the manual command", () => { const docker = rowNamed(rowsFor({ docker: "29.4.1" }), "docker"); expect(docker).toMatchObject({ installed: "29.4.1", target: PINNED_VERSIONS.docker, outdated: true, - action: "manual", + action: "docker", command: "DOCKER_ENGINE_UPGRADE=1 bash install-agenthits.sh update", }); - expect(docker.reason).toContain("maintenance window"); + expect(docker.reason).toContain("rolls Docker back"); }); it("compares Docker Engine versions numerically", () => { @@ -295,9 +295,7 @@ describe("host stack update rules", () => { ]); expect(refusals).toHaveLength(3); - expect(refusals[0]).toContain( - "DOCKER_ENGINE_UPGRADE=1 bash install-agenthits.sh update", - ); + expect(refusals[0]).toContain("its own button"); expect(refusals[1]).toContain("POSTGRES_IMAGE"); expect(refusals[2]).toContain("Web Server Update"); expect(hostStackRefusals(["traefik", "redis"])).toEqual([]); diff --git a/apps/dokploy/__test__/server/server-components.test.ts b/apps/dokploy/__test__/server/server-components.test.ts index 82d2e260ace..e19d359bd19 100644 --- a/apps/dokploy/__test__/server/server-components.test.ts +++ b/apps/dokploy/__test__/server/server-components.test.ts @@ -37,6 +37,7 @@ import { TRAEFIK_VERSION, } from "@dokploy/server/setup/traefik-setup"; import { describe, expect, it } from "vitest"; +import { BASH, FAKE_TOOLS } from "./docker-script-fakes"; const PORT_80_PUBLICATION = /(?:-p|--publish)[=\s]+\S*\b80\b|published=80\b|\b80\/tcp\b/; @@ -365,14 +366,17 @@ describe("buildComponentUpdateScript", () => { ); }); - it("waits 30 seconds for the new Docker before rolling back", () => { + it("waits 30 seconds for the new Docker before rolling back unless the host sets a longer wait", () => { const script = buildComponentUpdateScript(["docker"]); const wait = script.slice( script.indexOf("docker_wait_ready() {"), script.indexOf("docker_roll_back() {"), ); - expect(wait).toContain('if [ "$attempt" -ge 30 ]; then'); + expect(wait).toContain( + 'local timeout="${DOKPLOY_DOCKER_VERIFY_TIMEOUT:-30}"', + ); + expect(wait).toContain('if [ "$(date +%s)" -ge "$deadline" ]; then'); expect(wait).toContain("sleep 1"); expect(wait).not.toContain("sleep 2"); }); @@ -442,150 +446,6 @@ describe("filterDockerUpgrade", () => { }); }); -const BASH = spawnSync("sh", ["-c", "command -v bash"], { - encoding: "utf8", -}).stdout.trim(); - -const FAKE_TOOLS: Record = { - docker: `#!/bin/sh -printf 'docker %s\\n' "$*" >> "$CALL_LOG" -case "$1" in - pull) [ "$FAKE_PULL_FAILS" = 1 ] && exit 1 ;; - create) - [ "$FAKE_CREATE_FAILS" = 1 ] && exit 1 - for traefik_arg in "$@"; do traefik_image_arg="$traefik_arg"; done - printf '%s\\n' "$traefik_image_arg" > "$FAKE_STATE/traefik-image" - ;; - version) cat "$FAKE_STATE/engine" ;; - network) - case "$2" in - inspect) case " $FAKE_MISSING_NETWORKS " in *" $3 "*) exit 1 ;; esac ;; - connect) case " $FAKE_CONNECT_FAILS " in *" $3 "*) exit 1 ;; esac ;; - esac - ;; - inspect) - case "$*" in - *Config.Image*) echo "$FAKE_OLD_IMAGE" ;; - *NetworkSettings*) echo "$FAKE_TRAEFIK_NETWORKS" ;; - *PortBindings*) echo "$FAKE_PORT_BINDINGS" ;; - *RestartCount*) - traefik_now="$(cat "$FAKE_STATE/traefik-image" 2>/dev/null || echo "$FAKE_OLD_IMAGE")" - if [ "$FAKE_NOT_RUNNING" = 1 ] && [ "$traefik_now" != "$FAKE_OLD_IMAGE" ]; then echo "false 0"; else echo "true 0"; fi - ;; - *State.Running*) - traefik_now="$(cat "$FAKE_STATE/traefik-image" 2>/dev/null || echo "$FAKE_OLD_IMAGE")" - if [ "$FAKE_NOT_RUNNING" = 1 ] && [ "$traefik_now" != "$FAKE_OLD_IMAGE" ]; then echo false; else echo true; fi - ;; - esac - ;; -esac -exit 0 -`, - "apt-get": `#!/bin/sh -printf 'apt-get %s\\n' "$*" >> "$CALL_LOG" -case "$1" in - update) - [ "$FAKE_UPDATE_FAILS" = 1 ] && exit 1 - ;; - download) - [ "$FAKE_DOWNLOAD_FAILS" = 1 ] && exit 1 - printf 'fake package\\n' > "\${2%%=*}.deb" - ;; - install) - case "$*" in - *--download-only*) - [ "$FAKE_PREDOWNLOAD_FAILS" = 1 ] && exit 1 - ;; - *--allow-downgrades*) - [ "$FAKE_ROLLBACK_FAILS" = 1 ] && exit 1 - printf '%s\\n' "$FAKE_PREVIOUS_ENGINE" > "$FAKE_STATE/engine" - ;; - *--no-download*) - if [ "$FAKE_UPGRADE_KEEPS_OLD" != 1 ]; then - printf '%s\\n' "$FAKE_TARGET_ENGINE" > "$FAKE_STATE/engine" - fi - ;; - esac - ;; -esac -exit 0 -`, - "apt-cache": `#!/bin/sh -printf 'apt-cache %s\\n' "$*" >> "$CALL_LOG" -case "$1" in - madison) - [ "$FAKE_NO_CANDIDATE" = 1 ] && exit 0 - if [ "$FAKE_CANDIDATE_AFTER_REPO" = 1 ] && [ ! -f "$DOKPLOY_APT_SOURCES_DIR/docker.list" ]; then - exit 0 - fi - echo " docker-ce | 5:${PINNED_VERSIONS.docker}-1~ubuntu.24.04~noble | https://download.docker.com/linux/ubuntu noble/stable amd64 Packages" - ;; - policy) - case "$2" in - containerd.io) - candidate=1.7.27-1 - if [ "$FAKE_CONTAINERD_CHANGES" = 1 ]; then candidate=2.0.0-1; fi - printf 'containerd.io:\\n Installed: 1.7.27-1\\n Candidate: %s\\n' "$candidate" - ;; - docker-buildx-plugin) - printf 'docker-buildx-plugin:\\n Installed: 0.25.0-1~ubuntu.24.04~noble\\n Candidate: 0.25.0-1~ubuntu.24.04~noble\\n' - ;; - docker-compose-plugin) - printf 'docker-compose-plugin:\\n Installed: 2.40.0-1~ubuntu.24.04~noble\\n Candidate: 2.40.0-1~ubuntu.24.04~noble\\n' - ;; - esac - ;; -esac -exit 0 -`, - "dpkg-query": `#!/bin/sh -printf 'dpkg-query %s\\n' "$*" >> "$CALL_LOG" -case "$2" in - *Status-Status*) - case "$3" in - docker-ce) [ "$FAKE_NO_DOCKER_CE" = 1 ] || echo installed ;; - docker-ce-cli | containerd.io | docker-buildx-plugin | docker-compose-plugin) echo installed ;; - esac - ;; - *Version*) - case "$3" in - docker-ce | docker-ce-cli | docker-ce-rootless-extras) echo "5:28.3.0-1~ubuntu.24.04~noble" ;; - containerd.io) echo "1.7.27-1" ;; - docker-buildx-plugin) echo "0.25.0-1~ubuntu.24.04~noble" ;; - docker-compose-plugin) echo "2.40.0-1~ubuntu.24.04~noble" ;; - esac - ;; -esac -exit 0 -`, - dpkg: `#!/bin/sh -echo amd64 -`, - systemctl: `#!/bin/sh -printf 'systemctl %s\\n' "$*" >> "$CALL_LOG" -`, - curl: `#!/bin/sh -printf 'curl %s\\n' "$*" >> "$CALL_LOG" -if [ -n "$FAKE_CURL_FAILS_FOR" ]; then - case "$*" in - *"$FAKE_CURL_FAILS_FOR"*) exit 22 ;; - esac -fi -out="" -while [ "$#" -gt 0 ]; do - if [ "$1" = "-o" ]; then out="$2"; fi - shift -done -printf '%s\\n' 'echo fake get-docker "$@"' > "$out" -`, - sudo: `#!/bin/sh -exec "$@" -`, - sleep: `#!/bin/sh -exit 0 -`, -}; - const SYSTEM_TOOLS = [ "id", "awk", @@ -731,6 +591,8 @@ const runComponentScript = ( FAKE_OLD_IMAGE: "traefik:v3.6.25", TMPDIR: dir, TRAEFIK_SETTLE_SECONDS: "0", + // The fake sleep returns at once, so a failed wait spins until this deadline. + DOKPLOY_DOCKER_VERIFY_TIMEOUT: "2", DOKPLOY_BACKUP_DIR: backups, DOKPLOY_OS_RELEASE: osRelease, DOKPLOY_APT_SOURCES_DIR: sources, diff --git a/apps/dokploy/__test__/settings/docker-server-boundary.test.ts b/apps/dokploy/__test__/settings/docker-server-boundary.test.ts index 9f003944eac..837b6f0bb16 100644 --- a/apps/dokploy/__test__/settings/docker-server-boundary.test.ts +++ b/apps/dokploy/__test__/settings/docker-server-boundary.test.ts @@ -70,6 +70,9 @@ const mocks = vi.hoisted(() => ({ writeMainConfig: vi.fn(), writeTraefikConfigInPath: vi.fn(), writeTraefikSetup: vi.fn(), + readHostDockerUpgradeStatus: vi.fn(async (): Promise => null), + readHostDockerVersion: vi.fn(async (): Promise => "29.8.2"), + startHostDockerUpgrade: vi.fn(async () => {}), })); const redactWebServerSettings = (settings: T) => settings; @@ -91,6 +94,12 @@ vi.mock("@dokploy/server", async () => ({ component === "traefik" || component === "redis", readHostStackRows: vi.fn(), updateHostStackComponents: vi.fn(() => Promise.resolve()), + dockerRow: (await import("@dokploy/server/setup/host-stack-rows")).dockerRow, + isHostDockerUpgradeActive: (status: { state: string } | null) => + status?.state === "starting" || status?.state === "running", + readHostDockerUpgradeStatus: mocks.readHostDockerUpgradeStatus, + readHostDockerVersion: mocks.readHostDockerVersion, + startHostDockerUpgrade: mocks.startHostDockerUpgrade, checkGPUStatus: mocks.checkGPUStatus, checkPortInUse: mocks.checkPortInUse, checkPostgresHealth: mocks.checkPostgresHealth, @@ -930,6 +939,98 @@ describe("panel host update lock", () => { }); }); +describe("panel host Docker upgrade", () => { + const RUNNING = { + state: "running", + from: "29.8.2", + to: "29.9.2", + startedAt: "2026-10-29T10:00:00Z", + finishedAt: null, + exitCode: null, + log: "", + }; + + beforeEach(() => { + vi.clearAllMocks(); + mocks.isServerUpdateRunning.mockReturnValue(false); + mocks.audit.mockResolvedValue(undefined); + mocks.readHostDockerVersion.mockResolvedValue("29.8.2"); + mocks.readHostDockerUpgradeStatus.mockResolvedValue(null); + mocks.startHostDockerUpgrade.mockResolvedValue(undefined); + mocks.getUpdateData.mockResolvedValue({ + updateAvailable: true, + updateSource: "official", + latestVersion: "v2.0.0", + latestOfficialVersion: "v2.0.0", + }); + }); + + it("starts the host upgrade of an outdated Docker Engine and returns its status", async () => { + mocks.readHostDockerUpgradeStatus + .mockResolvedValueOnce(null) + .mockResolvedValueOnce({ ...RUNNING, state: "starting" }); + + await expect(createCaller().upgradeHostDocker()).resolves.toMatchObject({ + state: "starting", + }); + expect(mocks.startHostDockerUpgrade).toHaveBeenCalledWith("29.8.2"); + expect(mocks.audit).toHaveBeenCalledWith( + expect.anything(), + expect.objectContaining({ + resourceName: "panel-host", + metadata: { components: ["docker"] }, + }), + ); + }); + + it("refuses when Docker Engine already runs the pinned version or cannot be read", async () => { + mocks.readHostDockerVersion.mockResolvedValueOnce("99.0.0"); + await expect(createCaller().upgradeHostDocker()).rejects.toMatchObject({ + code: "BAD_REQUEST", + message: "Docker Engine already runs 99.0.0.", + }); + + mocks.readHostDockerVersion.mockResolvedValueOnce(null); + await expect(createCaller().upgradeHostDocker()).rejects.toMatchObject({ + code: "BAD_REQUEST", + }); + expect(mocks.startHostDockerUpgrade).not.toHaveBeenCalled(); + }); + + it("refuses a second upgrade while one runs on the host", async () => { + mocks.readHostDockerUpgradeStatus.mockResolvedValue(RUNNING); + + await expect(createCaller().upgradeHostDocker()).rejects.toMatchObject({ + code: "CONFLICT", + }); + expect(mocks.startHostDockerUpgrade).not.toHaveBeenCalled(); + }); + + it("keeps the panel update and the Traefik and Redis update out while Docker is upgraded", async () => { + mocks.readHostDockerUpgradeStatus.mockResolvedValue(RUNNING); + + await expect(createCaller().updateServer()).rejects.toMatchObject({ + code: "CONFLICT", + }); + await expect( + createCaller().updateHostStack({ components: ["redis"] }), + ).rejects.toMatchObject({ code: "CONFLICT" }); + expect(mocks.startServerUpdate).not.toHaveBeenCalled(); + }); + + it("frees the host-stack lock when the upgrade cannot start", async () => { + mocks.startHostDockerUpgrade.mockRejectedValueOnce( + new Error("The upgrade could not start on the host: no systemd"), + ); + + await expect(createCaller().upgradeHostDocker()).rejects.toMatchObject({ + code: "INTERNAL_SERVER_ERROR", + message: "The upgrade could not start on the host: no systemd", + }); + await expect(createCaller().upgradeHostDocker()).resolves.toBeNull(); + }); +}); + describe("Traefik operations next to a certificate deletion", () => { const ports = { additionalPorts: [ diff --git a/apps/dokploy/components/dashboard/settings/servers/panel-host-stack.tsx b/apps/dokploy/components/dashboard/settings/servers/panel-host-stack.tsx index 89c72ec16f5..eea12ccf42f 100644 --- a/apps/dokploy/components/dashboard/settings/servers/panel-host-stack.tsx +++ b/apps/dokploy/components/dashboard/settings/servers/panel-host-stack.tsx @@ -1,3 +1,4 @@ +import type { HostDockerUpgradeStatus } from "@dokploy/server/setup/host-docker-upgrade"; import type { HostStackComponent, HostStackRow, @@ -8,8 +9,9 @@ import { Loader2, RefreshCw, Server, + XCircle, } from "lucide-react"; -import { useRef, useState } from "react"; +import { useEffect, useRef, useState } from "react"; import { toast } from "sonner"; import { AlertDialog, @@ -62,6 +64,85 @@ const WARNINGS: Partial> = { redis: "Redis is restarted; it is unavailable for a few seconds.", }; +const DOCKER_POLL_MS = 2000; +const RECENT_RUN_MS = 24 * 60 * 60 * 1000; + +const isDockerUpgradeActive = (status: HostDockerUpgradeStatus | null) => + status?.state === "starting" || status?.state === "running"; + +const isRecentRun = (status: HostDockerUpgradeStatus) => { + const finished = Date.parse(status.finishedAt ?? status.startedAt); + return Number.isFinite(finished) && Date.now() - finished < RECENT_RUN_MS; +}; + +const DockerUpgradeHeadline = ({ + status, + connectionLost, +}: { + status: HostDockerUpgradeStatus; + connectionLost: boolean; +}) => { + const versions = `${status.from ?? "unknown"} → ${status.to}`; + if (isDockerUpgradeActive(status)) { + return ( +

+ + + {connectionLost + ? "Docker is restarting, and this panel with it. Waiting for the panel to come back; this window reconnects by itself." + : `Upgrading Docker Engine ${versions} on the host. Sites and this panel stop for a minute or two.`} + +

+ ); + } + if (status.state === "succeeded") { + return ( +

+ + Docker Engine upgraded: {versions}. +

+ ); + } + if (status.state === "interrupted") { + return ( +

+ + + The Docker Engine upgrade stopped reporting before it finished. Check + the host with: journalctl -u dokploy-docker-engine-upgrade + +

+ ); + } + return ( +

+ + + The Docker Engine upgrade failed + {status.exitCode !== null ? ` (exit code ${status.exitCode})` : ""}. The + log below says whether Docker was rolled back to {status.from}. + +

+ ); +}; + +const DockerUpgradeProgress = ({ + status, + connectionLost, +}: { + status: HostDockerUpgradeStatus; + connectionLost: boolean; +}) => ( +
+ + {status.log && ( + 
+				{status.log}
+
+ )} +
+); + const StatusLine = ({ row }: { row: HostStackRow }) => { if (row.status === "outdated") { return ( @@ -87,11 +168,15 @@ const HostStackRowItem = ({ checked, onCheckedChange, onOpenWebUpdate, + dockerUpgradeBusy, + onUpgradeDocker, }: { row: HostStackRow; checked: boolean; onCheckedChange: (checked: boolean) => void; onOpenWebUpdate: () => void; + dockerUpgradeBusy: boolean; + onUpgradeDocker: () => void; }) => { const checkboxId = `host-stack-${row.name}`; return ( @@ -128,8 +213,21 @@ const HostStackRowItem = ({ )} + {row.action === "docker" && row.outdated && ( + + )} {row.reason && - (row.action === "manual" || row.status === "unknown") && ( + (row.action === "manual" || + row.status === "unknown" || + (row.action === "docker" && row.outdated)) && (

{row.reason}

)} {row.action === "manual" && row.command && ( @@ -137,6 +235,18 @@ const HostStackRowItem = ({ {row.command} )} + {row.action === "docker" && + row.command && + (row.outdated || row.status === "unknown") && ( +
+ + Or run it on the host yourself + + + {row.command} + +
+ )} ); @@ -159,6 +269,51 @@ const PanelHostCard = ({ const [runComponents, setRunComponents] = useState([]); const [logs, setLogs] = useState(""); const logBuffer = useRef(""); + const [isDockerConfirmOpen, setIsDockerConfirmOpen] = useState(false); + const [isWatchingDocker, setIsWatchingDocker] = useState(false); + const utils = api.useUtils(); + + // The panel itself goes down while Docker restarts, so polling keeps going + // through failed requests until the host reports the run finished. + const dockerUpgrade = api.settings.hostDockerUpgrade.useQuery(undefined, { + refetchOnWindowFocus: false, + retry: false, + refetchInterval: (query) => + isWatchingDocker || isDockerUpgradeActive(query.state.data ?? null) + ? DOCKER_POLL_MS + : false, + }); + const dockerStatus = dockerUpgrade.data ?? null; + const dockerActive = isDockerUpgradeActive(dockerStatus); + const dockerRow = rows.find((row) => row.name === "docker"); + + const upgradeDocker = api.settings.upgradeHostDocker.useMutation({ + onSuccess(status) { + utils.settings.hostDockerUpgrade.setData(undefined, status); + setIsWatchingDocker(true); + }, + onError(err) { + toast.error(err.message); + void dockerUpgrade.refetch(); + }, + }); + + useEffect(() => { + if (dockerActive) { + setIsWatchingDocker(true); + return; + } + if (!isWatchingDocker || !dockerStatus) { + return; + } + setIsWatchingDocker(false); + if (dockerStatus.state === "succeeded") { + toast.success(`Docker Engine upgraded to ${dockerStatus.to}`); + } else { + toast.error("Docker Engine upgrade did not finish, see the log"); + } + void refetch(); + }, [dockerActive, dockerStatus, isWatchingDocker, refetch]); const outdatedCount = rows.filter((row) => row.outdated).length; const unknownCount = rows.filter((row) => row.status === "unknown").length; @@ -277,10 +432,26 @@ const PanelHostCard = ({ setIsOpen(false); setIsWebUpdateOpen(true); }} + dockerUpgradeBusy={ + dockerActive || + isWatchingDocker || + upgradeDocker.isPending + } + onUpgradeDocker={() => setIsDockerConfirmOpen(true)} /> ))} + {dockerStatus && + (dockerActive || + isWatchingDocker || + isRecentRun(dockerStatus)) && ( + + )} + {selected.map((name) => WARNINGS[name] ? (

@@ -297,7 +468,7 @@ const PanelHostCard = ({ {isUpdating && } {selected.length > 0 ? `Update ${selected.map((name) => LABELS[name]).join(" and ")}` - : "Nothing to update here"} + : "Traefik and Redis: nothing to update"} {logs && ( @@ -338,6 +509,45 @@ const PanelHostCard = ({ + + + + + + Upgrade Docker Engine to {dockerRow?.target} on the panel host? + + +

+ + Docker restarts, and every container on this server restarts + with it: all sites and this panel are unavailable for a minute + or two. + + + The upgrade runs on the host, outside the panel. If the + services do not come back healthy, it rolls Docker back to{" "} + {dockerRow?.installed} by itself. Keep this window open; it + reconnects when the panel is back. + +
+ + + + Cancel + { + setIsDockerConfirmOpen(false); + upgradeDocker.mutate(); + }} + > + Upgrade Docker + + + + ); }; diff --git a/apps/dokploy/server/api/routers/settings.ts b/apps/dokploy/server/api/routers/settings.ts index 0113b1263cd..3be6c914e86 100644 --- a/apps/dokploy/server/api/routers/settings.ts +++ b/apps/dokploy/server/api/routers/settings.ts @@ -18,6 +18,7 @@ import { DOKPLOY_KEEP_IMAGES_MIN, diskHistoryInputSchema, dockerDiskUsageDetailLimitSchema, + dockerRow, filterProtectedTraefikEntries, findServerById, getAgentHitsUpdateCommand, @@ -40,6 +41,7 @@ import { HOST_STACK_COMPONENTS, hostStackRefusals, IS_CLOUD, + isHostDockerUpgradeActive, isHostStackUiComponent, isHostStackUpdateRunning, isServerUpdateRunning, @@ -53,6 +55,8 @@ import { readDirectory, readEnvironmentVariableList, readEnvironmentVariables, + readHostDockerUpgradeStatus, + readHostDockerVersion, readHostStackRows, readMainConfig, readMonitoringConfig, @@ -65,6 +69,7 @@ import { resolveDockerDiskUsageDetailLimit, sendDockerCleanupNotifications, setupGPUSupport, + startHostDockerUpgrade, startLogCleanup, startServerUpdate, stopLogCleanup, @@ -258,6 +263,13 @@ export const settingsRouter = createTRPCRouter({ message: refusals.join(" "), }); } + if (isHostDockerUpgradeActive(await readHostDockerUpgradeStatus())) { + throw new TRPCError({ + code: "CONFLICT", + message: + "Docker Engine is being upgraded on this host. Try again when it finishes.", + }); + } const releaseHostStackLock = isServerUpdateRunning() ? null : acquireHostStackUpdateLock(); @@ -295,6 +307,77 @@ export const settingsRouter = createTRPCRouter({ }); }); }), + hostDockerUpgrade: adminProcedure.query(async ({ ctx }) => { + await assertLocalHostAccess(ctx); + if (IS_CLOUD) { + return null; + } + return await readHostDockerUpgradeStatus(); + }), + upgradeHostDocker: adminProcedure + .meta({ + openapi: { + path: "/settings/upgrade-host-docker", + method: "POST", + override: true, + enabled: false, + }, + }) + .mutation(async ({ ctx }) => { + await assertLocalHostAccess(ctx); + if (IS_CLOUD) { + throw new TRPCError({ + code: "FORBIDDEN", + message: "The panel host components are not managed here.", + }); + } + const docker = dockerRow(await readHostDockerVersion()); + if (!docker.outdated) { + throw new TRPCError({ + code: "BAD_REQUEST", + message: + docker.installed === null + ? "The installed Docker Engine version could not be read on the panel host." + : `Docker Engine already runs ${docker.installed}.`, + }); + } + const releaseHostStackLock = isServerUpdateRunning() + ? null + : acquireHostStackUpdateLock(); + if (!releaseHostStackLock) { + throw new TRPCError({ + code: "CONFLICT", + message: + "Another update of this panel host is running. Try again when it finishes.", + }); + } + try { + if (isHostDockerUpgradeActive(await readHostDockerUpgradeStatus())) { + throw new TRPCError({ + code: "CONFLICT", + message: "Docker Engine is already being upgraded on this host.", + }); + } + await audit(ctx, { + action: "update", + resourceType: "settings", + resourceName: "panel-host", + metadata: { components: ["docker"] }, + }); + await startHostDockerUpgrade(docker.installed); + } catch (error) { + if (error instanceof TRPCError) { + throw error; + } + throw new TRPCError({ + code: "INTERNAL_SERVER_ERROR", + message: error instanceof Error ? error.message : String(error), + }); + } finally { + releaseHostStackLock(); + } + return await readHostDockerUpgradeStatus(); + }), toggleDashboard: adminProcedure .input(apiEnableDashboard) .mutation(async ({ input, ctx }) => { @@ -793,7 +876,10 @@ export const settingsRouter = createTRPCRouter({ const keepImages = input?.keepImages; const data = await getUpdateData(packageInfo.version); // After the await: a host-stack update can start while the update check runs. - if (isHostStackUpdateRunning()) { + if ( + isHostStackUpdateRunning() || + isHostDockerUpgradeActive(await readHostDockerUpgradeStatus()) + ) { throw new TRPCError({ code: "CONFLICT", message: diff --git a/apps/dokploy/server/api/utils/procedure-access.ts b/apps/dokploy/server/api/utils/procedure-access.ts index 5eea0c3bc22..4a7ff84ffb7 100644 --- a/apps/dokploy/server/api/utils/procedure-access.ts +++ b/apps/dokploy/server/api/utils/procedure-access.ts @@ -626,6 +626,7 @@ export const PROCEDURE_ACCESS: Record = { "settings.haveActivateRequests": "read", "settings.haveTraefikDashboardPortEnabled": "read", "settings.health": "read", + "settings.hostDockerUpgrade": "read", "settings.hostStack": "read", "settings.isCloud": "read", "settings.isUserSubscribed": "read", @@ -657,6 +658,7 @@ export const PROCEDURE_ACCESS: Record = { "settings.updateTraefik": "dangerous", "settings.updateTraefikPorts": "dangerous", "settings.updateWebServerTraefikConfig": "dangerous", + "settings.upgradeHostDocker": "dangerous", "settings.writeTraefikEnv": "dangerous", "sshKey.all": "read", "sshKey.allForApps": "read", diff --git a/docs/server-maintenance.md b/docs/server-maintenance.md index 40743d3f9a5..cb72e17979f 100644 --- a/docs/server-maintenance.md +++ b/docs/server-maintenance.md @@ -7,13 +7,29 @@ В карточке **Panel host (this server)** есть два способа обновления: - **Components** обновляет **Traefik** (веб-сервер, через который идёт весь HTTPS-трафик) и **Redis** (кэш панели). Во время замены Traefik на несколько секунд недоступны все сайты и админка. Во время замены Redis он на несколько секунд недоступен. +- В том же окне **Components** у строки **Docker** есть своя кнопка **Upgrade to <версия>**. Она обновляет Docker Engine; подробности ниже. - **Web Server Update** обновляет саму **панель**. +### Обновление Docker из панели + +При обновлении Docker перезапускается его демон, а вместе с ним все контейнеры сервера: сайты и сама панель недоступны минуту-две. Поэтому панель не обновляет Docker сама изнутри своего контейнера. Она записывает скрипт в `/etc/dokploy/host-updates/` и запускает его на сервере как отдельную службу systemd `dokploy-docker-engine-upgrade`. Эта служба работает вне Docker и переживает его перезапуск. + +Что делает скрипт: + +1. Скачивает новые пакеты Docker и сохраняет текущие для отката в `/var/backups/dokploy`. Если скачать не удалось, сервер не меняется. +2. Ставит новые пакеты. Docker перезапускается. +3. До пяти минут ждёт, пока снова заработают Traefik, панель, Postgres и Redis. +4. Если они не поднялись, сам возвращает прежнюю версию Docker из сохранённых пакетов. + +Окно с кнопкой можно не закрывать: пока панель перезапускается, оно ждёт и потом само показывает журнал и итог. Журнал лежит в `/etc/dokploy/host-updates/docker-engine-upgrade.log`, на сервере его видно и через `journalctl -u dokploy-docker-engine-upgrade`. + +Кнопка работает на серверах с systemd и apt (Ubuntu, Debian, машина OrbStack на Mac). На других серверах панель покажет причину отказа, и тогда Docker обновляют командой из раздела 2. + ## 2. Что обновляется только в плановое окно -Docker и Postgres кнопкой не обновляются. Карточка показывает для них команду, которую запускают в заранее выбранное время. +Postgres кнопкой не обновляется. Для Docker кнопка есть (см. выше), но простой при обновлении такой же, как при ручном запуске. Поэтому его тоже лучше запускать в тихое время и заранее предупреждать пользователей. -**Docker Engine.** При обновлении перезапускается демон Docker. Вместе с ним останавливаются все контейнеры сервера, включая панель. Поэтому это делают один раз, в объявленное окно, и предупреждают пользователей о простое. Команда: `DOCKER_ENGINE_UPGRADE=1 bash install-agenthits.sh update`. +**Docker Engine вручную.** Если кнопка недоступна, Docker обновляют командой на сервере: `DOCKER_ENGINE_UPGRADE=1 bash install-agenthits.sh update`. Эта же команда видна в карточке под строкой Docker. **Postgres (база панели).** Смена major-версии (например, 17 → 18) требует переноса данных. Это отдельная работа, а не обновление. Поэтому сервер сохраняет текущую major-версию: установщик её не меняет. Сейчас с Postgres ничего делать не нужно. Обновление внутри одной major-версии (например, 18.5 → 18.6) выполняет та же команда обновления. Перед заменой она сохраняет резервную копию базы. diff --git a/packages/server/src/index.ts b/packages/server/src/index.ts index 27b07df27f5..279fc1575df 100644 --- a/packages/server/src/index.ts +++ b/packages/server/src/index.ts @@ -73,6 +73,7 @@ export * from "./services/web-server-settings"; export * from "./services/web-server-update"; export * from "./setup/config-paths"; export * from "./setup/forward-auth-setup"; +export * from "./setup/host-docker-upgrade"; export * from "./setup/host-stack"; export * from "./setup/host-stack-lock"; export * from "./setup/host-stack-rows"; diff --git a/packages/server/src/setup/host-docker-upgrade.ts b/packages/server/src/setup/host-docker-upgrade.ts new file mode 100644 index 00000000000..7cd4c0b18b9 --- /dev/null +++ b/packages/server/src/setup/host-docker-upgrade.ts @@ -0,0 +1,328 @@ +import { + appendFile, + chmod, + mkdir, + readFile, + rename, + stat, + writeFile, +} from "node:fs/promises"; +import path from "node:path"; +import { spawnAsync } from "../utils/process/spawnAsync"; +import { PINNED_VERSIONS } from "./component-versions"; +import { buildDockerEngineUpgradeSteps } from "./server-components"; + +// The panel and the host share /etc/dokploy, so the panel writes the script and +// reads its log and status here while the script itself runs on the host. +export const HOST_UPDATES_DIR = "/etc/dokploy/host-updates"; +export const DOCKER_UPGRADE_UNIT = "dokploy-docker-engine-upgrade"; + +const SCRIPT_NAME = "docker-engine-upgrade.sh"; +const LOG_NAME = "docker-engine-upgrade.log"; +const STATUS_NAME = "docker-engine-upgrade.json"; +const LOG_TAIL_BYTES = 64 * 1024; +// The longest silent stretch of a run is the health wait after the restart and +// the same wait after a rollback; a run quiet for longer than this has died. +const STALE_AFTER_MS = 30 * 60 * 1000; + +export type HostDockerUpgradeState = + | "starting" + | "running" + | "succeeded" + | "failed" + | "interrupted"; + +export type HostDockerUpgradeStatus = { + state: HostDockerUpgradeState; + from: string | null; + to: string; + startedAt: string; + finishedAt: string | null; + exitCode: number | null; + log: string; +}; + +type StoredStatus = Omit; + +const STATES: readonly HostDockerUpgradeState[] = [ + "starting", + "running", + "succeeded", + "failed", + "interrupted", +]; + +export const isHostDockerUpgradeActive = ( + status: Pick | null, +) => status?.state === "starting" || status?.state === "running"; + +const filesIn = (dir: string) => ({ + script: path.join(dir, SCRIPT_NAME), + log: path.join(dir, LOG_NAME), + status: path.join(dir, STATUS_NAME), +}); + +const shellQuote = (value: string) => `'${value.replaceAll("'", "'\\''")}'`; + +const HOST_READY_CHECK = ` + DOCKER_NOT_READY="this node is not an active swarm node" + [ "$(docker info --format '{{.Swarm.LocalNodeState}}' 2>/dev/null)" = "active" ] || return 1 + local service="" container="" health="" + for service in dokploy dokploy-postgres dokploy-redis; do + DOCKER_NOT_READY="$service has no running task" + container="$(docker ps -q --no-trunc --filter "label=com.docker.swarm.service.name=$service" --filter status=running | head -n1)" + [ -n "$container" ] || return 1 + case "$service" in + dokploy) + health="$(docker inspect --format '{{if .State.Health}}{{.State.Health.Status}}{{else}}none{{end}}' "$container" 2>/dev/null)" + DOCKER_NOT_READY="the panel container is \${health:-not inspectable}" + case "$health" in + healthy | none) ;; + *) return 1 ;; + esac + ;; + dokploy-postgres) + DOCKER_NOT_READY="Postgres does not accept connections" + docker exec "$container" pg_isready -U dokploy -d dokploy >/dev/null 2>&1 || return 1 + ;; + dokploy-redis) + DOCKER_NOT_READY="Redis does not answer PING" + [ "$(docker exec "$container" redis-cli ping 2>/dev/null)" = "PONG" ] || return 1 + ;; + esac + done + DOCKER_NOT_READY=""`; + +export const buildHostDockerUpgradeScript = (dir = HOST_UPDATES_DIR) => { + const files = filesIn(dir); + return `#!/usr/bin/env bash +# Runs on the panel host as a transient systemd unit, outside Docker, so the +# daemon restart below does not stop it the way it stops the panel. +STATUS_FILE=${shellQuote(files.status)} +LOG_FILE=${shellQuote(files.log)} +TARGET_VERSION=${shellQuote(PINNED_VERSIONS.docker)} +exec >>"$LOG_FILE" 2>&1 +export DEBIAN_FRONTEND=noninteractive +# needrestart would otherwise restart unrelated host services after apt-get. +export NEEDRESTART_MODE=l +DOKPLOY_DOCKER_VERIFY_TIMEOUT="\${DOKPLOY_DOCKER_VERIFY_TIMEOUT:-300}" +started_at="$(date -u +%Y-%m-%dT%H:%M:%SZ)" +previous_version="$(docker version --format '{{.Server.Version}}' 2>/dev/null | tr -cd '0-9A-Za-z.+~-' || true)" +write_status() { + local finished="null" from="null" + if [ -n "$3" ]; then + finished="\\"$3\\"" + fi + if [ -n "$previous_version" ]; then + from="\\"$previous_version\\"" + fi + printf '{"state":"%s","from":%s,"to":"%s","startedAt":"%s","finishedAt":%s,"exitCode":%s}\\n' "$1" "$from" "$TARGET_VERSION" "$started_at" "$finished" "$2" >"$STATUS_FILE.tmp" && mv -f "$STATUS_FILE.tmp" "$STATUS_FILE" +} +finish_status() { + local code=$? + if [ "$code" = 0 ]; then + write_status succeeded 0 "$(date -u +%Y-%m-%dT%H:%M:%SZ)" + else + write_status failed "$code" "$(date -u +%Y-%m-%dT%H:%M:%SZ)" + fi +} +write_status running null "" +trap finish_status EXIT +echo "Docker Engine upgrade started on the host at $started_at: \${previous_version:-unknown} -> $TARGET_VERSION" +set -e +SUDO_CMD="" +${buildDockerEngineUpgradeSteps(HOST_READY_CHECK)} +echo "Docker Engine upgrade finished ✅" +`; +}; + +// Runs inside a one-off privileged container that enters the host's +// namespaces, so systemd-run starts the script as a host unit that outlives +// both that container and the Docker daemon restart. +export const buildDockerUpgradeLauncherArgs = ( + image: string, + dir = HOST_UPDATES_DIR, +) => { + const script = filesIn(dir).script; + const hostCommand = [ + `[ -f ${shellQuote(script)} ] || { echo "The upgrade script is not on the host at ${script}: the panel's /etc/dokploy is not mounted from the host." >&2; exit 4; }`, + `[ -d /run/systemd/system ] && command -v systemd-run >/dev/null 2>&1 || { echo "This host does not run systemd, so the upgrade cannot run outside Docker from the panel." >&2; exit 3; }`, + `exec systemd-run --quiet --collect --unit=${DOCKER_UPGRADE_UNIT} --description="Dokploy Docker Engine upgrade" /bin/bash ${shellQuote(script)}`, + ].join("\n"); + return [ + "run", + "--rm", + "--pull=never", + "--privileged", + "--pid=host", + "--network=host", + "--user=0:0", + "--label=com.dokploy.host-update=docker-engine", + "--entrypoint=nsenter", + image, + "--target=1", + "--mount", + "--uts", + "--ipc", + "--net", + "--pid", + "--", + "/bin/sh", + "-c", + hostCommand, + ]; +}; + +const parseStoredStatus = (raw: string): StoredStatus | null => { + let value: unknown; + try { + value = JSON.parse(raw); + } catch { + return null; + } + if (typeof value !== "object" || value === null) { + return null; + } + const record = value as Record; + const state = STATES.find((candidate) => candidate === record.state); + if (!state || typeof record.to !== "string") { + return null; + } + return { + state, + from: typeof record.from === "string" ? record.from : null, + to: record.to, + startedAt: typeof record.startedAt === "string" ? record.startedAt : "", + finishedAt: + typeof record.finishedAt === "string" ? record.finishedAt : null, + exitCode: typeof record.exitCode === "number" ? record.exitCode : null, + }; +}; + +const readIfExists = async (file: string) => { + try { + return await readFile(file, "utf8"); + } catch (error) { + if ((error as NodeJS.ErrnoException).code === "ENOENT") { + return null; + } + throw error; + } +}; + +const lastModified = async (file: string) => { + try { + return (await stat(file)).mtimeMs; + } catch { + return 0; + } +}; + +const writeStoredStatus = async (dir: string, status: StoredStatus) => { + const file = filesIn(dir).status; + await writeFile(`${file}.tmp`, `${JSON.stringify(status)}\n`, { + mode: 0o600, + }); + await rename(`${file}.tmp`, file); +}; + +export const readHostDockerUpgradeStatus = async ( + dir = HOST_UPDATES_DIR, + now = Date.now(), +): Promise => { + const files = filesIn(dir); + const raw = await readIfExists(files.status); + const stored = raw === null ? null : parseStoredStatus(raw); + if (!stored) { + return null; + } + const log = (await readIfExists(files.log)) ?? ""; + const status: HostDockerUpgradeStatus = { + ...stored, + log: log.length > LOG_TAIL_BYTES ? log.slice(-LOG_TAIL_BYTES) : log, + }; + if (isHostDockerUpgradeActive(status)) { + const touched = Math.max( + await lastModified(files.status), + await lastModified(files.log), + ); + if (now - touched > STALE_AFTER_MS) { + return { ...status, state: "interrupted" }; + } + } + return status; +}; + +const readPanelImage = async () => { + const output = await spawnAsync("docker", [ + "service", + "inspect", + "dokploy", + "--format", + "{{.Spec.TaskTemplate.ContainerSpec.Image}}", + ]); + const image = output.toString().trim(); + if (!image) { + throw new Error("The image of the dokploy service could not be read."); + } + return image; +}; + +const errorText = (error: unknown) => { + const stderr = (error as { stderr?: { toString(): string } }).stderr; + const text = stderr?.toString().trim(); + if (text) { + return text; + } + return error instanceof Error ? error.message : String(error); +}; + +// Callers check that no run is active and hold the host-stack lock until this +// returns; the run itself goes on in the host unit after that. +export const startHostDockerUpgrade = async ( + from: string | null, + { + dir = HOST_UPDATES_DIR, + readImage = readPanelImage, + launch = (args: string[]) => spawnAsync("docker", args), + }: { + dir?: string; + readImage?: () => Promise; + launch?: (args: string[]) => Promise; + } = {}, +) => { + const files = filesIn(dir); + await mkdir(dir, { recursive: true, mode: 0o700 }); + await writeFile(files.script, buildHostDockerUpgradeScript(dir), { + mode: 0o700, + }); + await chmod(files.script, 0o700); + const startedAt = new Date().toISOString(); + await writeFile( + files.log, + `Starting the Docker Engine upgrade to ${PINNED_VERSIONS.docker} from the panel at ${startedAt}\n`, + { mode: 0o600 }, + ); + const starting: StoredStatus = { + state: "starting", + from, + to: PINNED_VERSIONS.docker, + startedAt, + finishedAt: null, + exitCode: null, + }; + await writeStoredStatus(dir, starting); + try { + const image = await readImage(); + await launch(buildDockerUpgradeLauncherArgs(image, dir)); + } catch (error) { + const message = `The upgrade could not start on the host: ${errorText(error)}`; + await appendFile(files.log, `Error: ${message}\nNothing was changed.\n`); + await writeStoredStatus(dir, { + ...starting, + state: "failed", + finishedAt: new Date().toISOString(), + }); + throw new Error(message); + } +}; diff --git a/packages/server/src/setup/host-stack-rows.ts b/packages/server/src/setup/host-stack-rows.ts index 14ecc22eb23..12640d3d433 100644 --- a/packages/server/src/setup/host-stack-rows.ts +++ b/packages/server/src/setup/host-stack-rows.ts @@ -15,7 +15,7 @@ export const HOST_STACK_UI_COMPONENTS = ["traefik", "redis"] as const; export type HostStackUiComponent = (typeof HOST_STACK_UI_COMPONENTS)[number]; -export type HostStackAction = "panel" | "ui" | "manual"; +export type HostStackAction = "panel" | "ui" | "docker" | "manual"; export type HostStackStatus = "current" | "outdated" | "unknown"; @@ -54,7 +54,7 @@ const UNKNOWN_REASON = const PANEL_UNKNOWN_REASON = "The update check did not return a version for this panel."; const DOCKER_REASON = - "Docker Engine upgrades restart the daemon, which stops this panel. Run them only in the maintenance window."; + "Upgrading restarts Docker: every site on this server and this panel stop for a minute or two. The upgrade runs on the host and rolls Docker back by itself if the services do not come back healthy."; const POSTGRES_REASON = "Postgres changes only in the maintenance window, and keeps its major version."; @@ -63,7 +63,8 @@ const HOST_STACK_REFUSALS: Record< string > = { panel: "The panel is updated from the Web Server Update dialog.", - docker: `Docker Engine is not upgraded from the panel because the daemon restart stops it. In the maintenance window run: ${DOCKER_UPGRADE_COMMAND}`, + docker: + "Docker Engine is upgraded with its own button in the Panel host components list, because the daemon restart stops this panel.", postgres: "Postgres is not updated from the panel, and its major version never changes here. In the maintenance window run the host update with POSTGRES_IMAGE set to the image Postgres runs now (see the Panel host components list).", }; @@ -145,12 +146,12 @@ const panelRow = ({ }; }; -const dockerRow = (readout: string | null): HostStackRow => { +export const dockerRow = (readout: string | null): HostStackRow => { const target = PINNED_VERSIONS.docker; const installed = versionOf(readout); if (installed === null) { return { - ...unknownRow("docker", target, "manual"), + ...unknownRow("docker", target, "docker"), command: DOCKER_UPGRADE_COMMAND, }; } @@ -161,7 +162,7 @@ const dockerRow = (readout: string | null): HostStackRow => { target, status: outdated ? "outdated" : "current", outdated, - action: "manual", + action: "docker", reason: DOCKER_REASON, command: DOCKER_UPGRADE_COMMAND, }; diff --git a/packages/server/src/setup/host-stack.ts b/packages/server/src/setup/host-stack.ts index 0b40b4f46b1..0895bb531ae 100644 --- a/packages/server/src/setup/host-stack.ts +++ b/packages/server/src/setup/host-stack.ts @@ -41,12 +41,15 @@ const checkPanelUpdate = async (currentVersion: string) => { } }; +export const readHostDockerVersion = () => + readCommand("docker version --format '{{.Server.Version}}'"); + export const readHostStackRows = async ( currentVersion: string, ): Promise => { const [docker, traefikImage, postgresImage, redisImage, update] = await Promise.all([ - readCommand("docker version --format '{{.Server.Version}}'"), + readHostDockerVersion(), readCommand("docker inspect -f '{{.Config.Image}}' dokploy-traefik"), readCommand( "docker service inspect dokploy-postgres --format '{{.Spec.TaskTemplate.ContainerSpec.Image}}'", diff --git a/packages/server/src/setup/server-components.ts b/packages/server/src/setup/server-components.ts index 9dca2c9b643..fefa188d5fe 100644 --- a/packages/server/src/setup/server-components.ts +++ b/packages/server/src/setup/server-components.ts @@ -323,20 +323,37 @@ docker_prepare_apt() { return 0 } +# The panel host replaces this with a check of its own swarm services. +docker_host_ready() { + return 0 +} + docker_ready_on() { + DOCKER_NOT_READY="Docker Engine does not report $1 yet" [ "$($SUDO_CMD docker version --format '{{.Server.Version}}' 2>/dev/null)" = "$1" ] || return 1 if [ "$2" = 1 ]; then + DOCKER_NOT_READY="dokploy-traefik is not running" [ "$($SUDO_CMD docker inspect -f '{{.State.Running}}' dokploy-traefik 2>/dev/null)" = "true" ] || return 1 fi + DOCKER_NOT_READY="" + docker_host_ready } +# A deadline, not a count of attempts: on a loaded host one round of checks +# takes seconds, so counted attempts stretched the wait far past its limit. docker_wait_ready() { - local attempt=0 + local timeout="\${DOKPLOY_DOCKER_VERIFY_TIMEOUT:-30}" + local deadline=$(($(date +%s) + timeout)) + local next_report=$(($(date +%s) + 30)) until docker_ready_on "$1" "$2"; do - attempt=$((attempt + 1)) - if [ "$attempt" -ge 30 ]; then + if [ "$(date +%s)" -ge "$deadline" ]; then + echo "Not ready on Docker $1 after $timeout seconds: \${DOCKER_NOT_READY:-the services are not back}" >&2 return 1 fi + if [ "$(date +%s)" -ge "$next_report" ]; then + echo "Waiting for the services on Docker $1: \${DOCKER_NOT_READY:-the services are not back}" + next_report=$(($(date +%s) + 30)) + fi sleep 1 done } @@ -607,6 +624,23 @@ ${predownloadGate(components)} ${components.map(updateStepFor).join("\n")} `; +// On the panel host a skipped Docker upgrade is the whole job failing, and the +// host is ready only when its own services are back, so both differ from the +// remote-server step above. +export const buildDockerEngineUpgradeSteps = (hostReadyCheck: string) => ` +${predownloadGate(["docker"])} +if [ "$DOCKER_SKIPPED" = 1 ]; then + echo "Error: Docker not upgraded: $DOCKER_SKIP_REASON. Nothing was changed." >&2 + exit 1 +fi +docker_host_ready() { +${hostReadyCheck} +} +echo "Upgrading Docker Engine to ${PINNED_VERSIONS.docker}. The daemon restarts, and every container on this server restarts with it." +docker_update_from_apt +echo "Docker version ${PINNED_VERSIONS.docker} installed ✅" +`; + export const wrapComponentUpdateSteps = (steps: string) => ` set -e if [ "$(id -u)" -eq 0 ]; then SUDO_CMD=""; else SUDO_CMD="sudo"; fi