From 8d5dee23c23d1835379f0c89b81d0036ebdbc811 Mon Sep 17 00:00:00 2001 From: merge-check <140916359+agentHits@users.noreply.github.com> Date: Thu, 29 Oct 2026 07:21:56 -0400 Subject: [PATCH 1/2] feat(panel-host): upgrade Docker Engine from the Panel host components dialog The Docker row of Panel host components only showed a shell command, because upgrading Docker restarts the daemon and with it the panel that would run the upgrade. The row now has an "Upgrade to " button. The panel writes the upgrade script to /etc/dokploy/host-updates, which the host shares with the panel container, and starts it through a one-off privileged container that enters the host's namespaces and calls systemd-run. The script then runs as the host unit dokploy-docker-engine-upgrade, outside Docker, so the daemon restart does not stop it. The script reuses the remote-server Docker steps: it pre-downloads the pinned packages, saves the installed ones for rollback, installs, and waits up to five minutes for Traefik, the panel (healthy), Postgres (pg_isready) and Redis (PONG). If they do not come back, it reinstalls the saved packages. A skipped upgrade (no apt, no pinned candidate, a Docker apt source for another release) fails the run with nothing changed. The script writes its log and a JSON status next to itself. The dialog polls that status every two seconds, keeps polling while the panel is down, and shows the log and the result when the panel is back. A run that stops writing for 30 minutes is reported as interrupted. A running upgrade blocks a second one, the Traefik and Redis update and the panel update. The new procedures are settings.hostDockerUpgrade (read) and settings.upgradeHostDocker (dangerous, not in the OpenAPI document). Hosts without systemd get a clear refusal and keep the manual command, which stays visible under the button. The Docker test fakes move to __test__/server/docker-script-fakes.ts so the new host script tests run the generated bash the same way as the remote-server tests. Co-Authored-By: Claude Opus 5.5 --- .../__test__/server/docker-script-fakes.ts | 156 ++++++++ .../server/host-docker-upgrade.test.ts | 336 ++++++++++++++++++ .../__test__/server/host-stack-rows.test.ts | 16 +- .../__test__/server/server-components.test.ts | 151 +------- .../settings/docker-server-boundary.test.ts | 101 ++++++ .../settings/servers/panel-host-stack.tsx | 216 ++++++++++- apps/dokploy/server/api/routers/settings.ts | 88 ++++- .../server/api/utils/procedure-access.ts | 2 + docs/server-maintenance.md | 20 +- packages/server/src/index.ts | 1 + .../server/src/setup/host-docker-upgrade.ts | 321 +++++++++++++++++ packages/server/src/setup/host-stack-rows.ts | 13 +- packages/server/src/setup/host-stack.ts | 5 +- .../server/src/setup/server-components.ts | 25 +- 14 files changed, 1282 insertions(+), 169 deletions(-) create mode 100644 apps/dokploy/__test__/server/docker-script-fakes.ts create mode 100644 apps/dokploy/__test__/server/host-docker-upgrade.test.ts create mode 100644 packages/server/src/setup/host-docker-upgrade.ts diff --git a/apps/dokploy/__test__/server/docker-script-fakes.ts b/apps/dokploy/__test__/server/docker-script-fakes.ts new file mode 100644 index 00000000000..923e24e5f46 --- /dev/null +++ b/apps/dokploy/__test__/server/docker-script-fakes.ts @@ -0,0 +1,156 @@ +import { spawnSync } from "node:child_process"; +import { PINNED_VERSIONS } from "@dokploy/server/setup/component-versions"; + +// Stand-ins for the host tools the generated update scripts call. Each one +// records its arguments in $CALL_LOG and keeps Docker's state in $FAKE_STATE. +export const BASH = spawnSync("sh", ["-c", "command -v bash"], { + encoding: "utf8", +}).stdout.trim(); + +export const FAKE_TOOLS: Record = { + docker: `#!/bin/sh +printf 'docker %s\\n' "$*" >> "$CALL_LOG" +case "$1" in + pull) [ "$FAKE_PULL_FAILS" = 1 ] && exit 1 ;; + create) + [ "$FAKE_CREATE_FAILS" = 1 ] && exit 1 + for traefik_arg in "$@"; do traefik_image_arg="$traefik_arg"; done + printf '%s\\n' "$traefik_image_arg" > "$FAKE_STATE/traefik-image" + ;; + version) cat "$FAKE_STATE/engine" ;; + info) echo "\${FAKE_SWARM_STATE:-active}" ;; + ps) echo cid ;; + exec) + case "$*" in + *redis-cli*) echo PONG ;; + esac + ;; + network) + case "$2" in + inspect) case " $FAKE_MISSING_NETWORKS " in *" $3 "*) exit 1 ;; esac ;; + connect) case " $FAKE_CONNECT_FAILS " in *" $3 "*) exit 1 ;; esac ;; + esac + ;; + inspect) + case "$*" in + *State.Health*) echo "\${FAKE_PANEL_HEALTH:-healthy}" ;; + *Config.Image*) echo "$FAKE_OLD_IMAGE" ;; + *NetworkSettings*) echo "$FAKE_TRAEFIK_NETWORKS" ;; + *PortBindings*) echo "$FAKE_PORT_BINDINGS" ;; + *RestartCount*) + traefik_now="$(cat "$FAKE_STATE/traefik-image" 2>/dev/null || echo "$FAKE_OLD_IMAGE")" + if [ "$FAKE_NOT_RUNNING" = 1 ] && [ "$traefik_now" != "$FAKE_OLD_IMAGE" ]; then echo "false 0"; else echo "true 0"; fi + ;; + *State.Running*) + traefik_now="$(cat "$FAKE_STATE/traefik-image" 2>/dev/null || echo "$FAKE_OLD_IMAGE")" + if [ "$FAKE_NOT_RUNNING" = 1 ] && [ "$traefik_now" != "$FAKE_OLD_IMAGE" ]; then echo false; else echo true; fi + ;; + esac + ;; +esac +exit 0 +`, + "apt-get": `#!/bin/sh +printf 'apt-get %s\\n' "$*" >> "$CALL_LOG" +case "$1" in + update) + [ "$FAKE_UPDATE_FAILS" = 1 ] && exit 1 + ;; + download) + [ "$FAKE_DOWNLOAD_FAILS" = 1 ] && exit 1 + printf 'fake package\\n' > "\${2%%=*}.deb" + ;; + install) + case "$*" in + *--download-only*) + [ "$FAKE_PREDOWNLOAD_FAILS" = 1 ] && exit 1 + ;; + *--allow-downgrades*) + [ "$FAKE_ROLLBACK_FAILS" = 1 ] && exit 1 + printf '%s\\n' "$FAKE_PREVIOUS_ENGINE" > "$FAKE_STATE/engine" + ;; + *--no-download*) + if [ "$FAKE_UPGRADE_KEEPS_OLD" != 1 ]; then + printf '%s\\n' "$FAKE_TARGET_ENGINE" > "$FAKE_STATE/engine" + fi + ;; + esac + ;; +esac +exit 0 +`, + "apt-cache": `#!/bin/sh +printf 'apt-cache %s\\n' "$*" >> "$CALL_LOG" +case "$1" in + madison) + [ "$FAKE_NO_CANDIDATE" = 1 ] && exit 0 + if [ "$FAKE_CANDIDATE_AFTER_REPO" = 1 ] && [ ! -f "$DOKPLOY_APT_SOURCES_DIR/docker.list" ]; then + exit 0 + fi + echo " docker-ce | 5:${PINNED_VERSIONS.docker}-1~ubuntu.24.04~noble | https://download.docker.com/linux/ubuntu noble/stable amd64 Packages" + ;; + policy) + case "$2" in + containerd.io) + candidate=1.7.27-1 + if [ "$FAKE_CONTAINERD_CHANGES" = 1 ]; then candidate=2.0.0-1; fi + printf 'containerd.io:\\n Installed: 1.7.27-1\\n Candidate: %s\\n' "$candidate" + ;; + docker-buildx-plugin) + printf 'docker-buildx-plugin:\\n Installed: 0.25.0-1~ubuntu.24.04~noble\\n Candidate: 0.25.0-1~ubuntu.24.04~noble\\n' + ;; + docker-compose-plugin) + printf 'docker-compose-plugin:\\n Installed: 2.40.0-1~ubuntu.24.04~noble\\n Candidate: 2.40.0-1~ubuntu.24.04~noble\\n' + ;; + esac + ;; +esac +exit 0 +`, + "dpkg-query": `#!/bin/sh +printf 'dpkg-query %s\\n' "$*" >> "$CALL_LOG" +case "$2" in + *Status-Status*) + case "$3" in + docker-ce) [ "$FAKE_NO_DOCKER_CE" = 1 ] || echo installed ;; + docker-ce-cli | containerd.io | docker-buildx-plugin | docker-compose-plugin) echo installed ;; + esac + ;; + *Version*) + case "$3" in + docker-ce | docker-ce-cli | docker-ce-rootless-extras) echo "5:28.3.0-1~ubuntu.24.04~noble" ;; + containerd.io) echo "1.7.27-1" ;; + docker-buildx-plugin) echo "0.25.0-1~ubuntu.24.04~noble" ;; + docker-compose-plugin) echo "2.40.0-1~ubuntu.24.04~noble" ;; + esac + ;; +esac +exit 0 +`, + dpkg: `#!/bin/sh +echo amd64 +`, + systemctl: `#!/bin/sh +printf 'systemctl %s\\n' "$*" >> "$CALL_LOG" +`, + curl: `#!/bin/sh +printf 'curl %s\\n' "$*" >> "$CALL_LOG" +if [ -n "$FAKE_CURL_FAILS_FOR" ]; then + case "$*" in + *"$FAKE_CURL_FAILS_FOR"*) exit 22 ;; + esac +fi +out="" +while [ "$#" -gt 0 ]; do + if [ "$1" = "-o" ]; then out="$2"; fi + shift +done +printf '%s\\n' 'echo fake get-docker "$@"' > "$out" +`, + sudo: `#!/bin/sh +exec "$@" +`, + sleep: `#!/bin/sh +exit 0 +`, +}; diff --git a/apps/dokploy/__test__/server/host-docker-upgrade.test.ts b/apps/dokploy/__test__/server/host-docker-upgrade.test.ts new file mode 100644 index 00000000000..069c1034c6f --- /dev/null +++ b/apps/dokploy/__test__/server/host-docker-upgrade.test.ts @@ -0,0 +1,336 @@ +import { spawnSync } from "node:child_process"; +import { + chmodSync, + existsSync, + mkdirSync, + mkdtempSync, + readFileSync, + rmSync, + statSync, + utimesSync, + writeFileSync, +} from "node:fs"; +import { tmpdir } from "node:os"; +import path from "node:path"; +import { PINNED_VERSIONS } from "@dokploy/server/setup/component-versions"; +import { + buildDockerUpgradeLauncherArgs, + buildHostDockerUpgradeScript, + DOCKER_UPGRADE_UNIT, + readHostDockerUpgradeStatus, + startHostDockerUpgrade, +} from "@dokploy/server/setup/host-docker-upgrade"; +import { afterEach, describe, expect, it } from "vitest"; +import { BASH, FAKE_TOOLS } from "./docker-script-fakes"; + +const PREVIOUS_ENGINE = "28.3.0"; + +const tempDirs: string[] = []; + +const makeTempDir = () => { + const dir = mkdtempSync(path.join(tmpdir(), "host-docker-upgrade-")); + tempDirs.push(dir); + return dir; +}; + +afterEach(() => { + for (const dir of tempDirs.splice(0)) { + rmSync(dir, { force: true, recursive: true }); + } +}); + +const DOCKER_LIST_NOBLE = + "deb [arch=amd64 signed-by=/etc/apt/keyrings/docker.asc] https://download.docker.com/linux/ubuntu noble stable\n"; + +const runHostScript = (env: Record = {}) => { + const dir = makeTempDir(); + const updates = path.join(dir, "host-updates"); + const sources = path.join(dir, "sources"); + const callLog = path.join(dir, "calls.log"); + mkdirSync(updates); + mkdirSync(sources); + writeFileSync(path.join(sources, "docker.list"), DOCKER_LIST_NOBLE); + writeFileSync( + path.join(dir, "os-release"), + "ID=ubuntu\nVERSION_CODENAME=noble\n", + ); + writeFileSync(path.join(dir, "engine"), `${PREVIOUS_ENGINE}\n`); + writeFileSync(callLog, ""); + for (const [name, body] of Object.entries(FAKE_TOOLS)) { + writeFileSync(path.join(dir, name), body); + chmodSync(path.join(dir, name), 0o755); + } + const result = spawnSync(BASH, [], { + input: buildHostDockerUpgradeScript(updates), + encoding: "utf8", + env: { + ...process.env, + PATH: `${dir}:${process.env.PATH}`, + CALL_LOG: callLog, + FAKE_STATE: dir, + FAKE_PREVIOUS_ENGINE: PREVIOUS_ENGINE, + FAKE_TARGET_ENGINE: PINNED_VERSIONS.docker, + FAKE_OLD_IMAGE: "traefik:v3.7.16", + TMPDIR: dir, + DOKPLOY_BACKUP_DIR: path.join(dir, "backups"), + DOKPLOY_OS_RELEASE: path.join(dir, "os-release"), + DOKPLOY_APT_SOURCES_DIR: sources, + DOKPLOY_DOCKER_KEYRING: path.join(dir, "keyrings", "docker.asc"), + DOKPLOY_DOCKER_VERIFY_TIMEOUT: "3", + ...env, + }, + }); + const statusFile = path.join(updates, "docker-engine-upgrade.json"); + return { + status: result.status, + stdout: result.stdout, + log: readFileSync(path.join(updates, "docker-engine-upgrade.log"), "utf8"), + stored: existsSync(statusFile) + ? JSON.parse(readFileSync(statusFile, "utf8")) + : null, + calls: readFileSync(callLog, "utf8").split("\n").filter(Boolean), + engine: readFileSync(path.join(dir, "engine"), "utf8").trim(), + }; +}; + +// Each test runs the generated script in a real shell. +describe("buildHostDockerUpgradeScript", { timeout: 60_000 }, () => { + it("generates valid bash", () => { + const result = spawnSync("bash", ["-n"], { + input: buildHostDockerUpgradeScript(), + }); + + expect(result.status, result.stderr?.toString()).toBe(0); + }); + + it("upgrades Docker, waits for the panel, Postgres and Redis, and records success", () => { + const run = runHostScript(); + + expect(run.status, run.log).toBe(0); + expect(run.stdout).toBe(""); + expect(run.engine).toBe(PINNED_VERSIONS.docker); + expect(run.stored).toMatchObject({ + state: "succeeded", + from: PREVIOUS_ENGINE, + to: PINNED_VERSIONS.docker, + exitCode: 0, + }); + expect(run.stored.finishedAt).toMatch(/^\d{4}-\d{2}-\d{2}T/); + expect(run.log).toContain( + `Docker Engine upgrade started on the host at ${run.stored.startedAt}: ${PREVIOUS_ENGINE} -> ${PINNED_VERSIONS.docker}`, + ); + expect(run.log).toContain("Docker Engine upgrade finished ✅"); + expect(run.calls).toContain( + "docker info --format {{.Swarm.LocalNodeState}}", + ); + expect( + run.calls.some((call) => + call.startsWith("docker exec cid pg_isready -U dokploy -d dokploy"), + ), + ).toBe(true); + expect(run.calls).toContain("docker exec cid redis-cli ping"); + expect(run.calls.some((call) => call.includes("--allow-downgrades"))).toBe( + false, + ); + }); + + it("rolls Docker back and records the failure when the panel does not come back healthy", () => { + const run = runHostScript({ FAKE_PANEL_HEALTH: "starting" }); + + expect(run.status).not.toBe(0); + expect(run.engine).toBe(PREVIOUS_ENGINE); + expect(run.calls.some((call) => call.includes("--allow-downgrades"))).toBe( + true, + ); + expect(run.log).toContain( + `Error: Docker ${PINNED_VERSIONS.docker} did not come up healthy after the install.`, + ); + expect(run.log).toContain(`Rolling Docker back to ${PREVIOUS_ENGINE}`); + expect(run.stored).toMatchObject({ + state: "failed", + from: PREVIOUS_ENGINE, + exitCode: 1, + }); + }); + + it("accepts a panel container without a health check", () => { + const run = runHostScript({ FAKE_PANEL_HEALTH: "none" }); + + expect(run.status, run.log).toBe(0); + expect(run.stored.state).toBe("succeeded"); + }); + + it("fails without touching Docker when the pinned version is not in apt", () => { + const run = runHostScript({ FAKE_NO_CANDIDATE: "1" }); + + expect(run.status).not.toBe(0); + expect(run.engine).toBe(PREVIOUS_ENGINE); + expect(run.log).toContain("Error: Docker not upgraded:"); + expect(run.log).toContain("Nothing was changed."); + expect(run.calls.some((call) => call.includes("--no-download"))).toBe( + false, + ); + expect(run.stored).toMatchObject({ state: "failed", exitCode: 1 }); + }); +}); + +describe("buildDockerUpgradeLauncherArgs", () => { + it("starts the script as a host systemd unit from a privileged one-off container", () => { + const args = buildDockerUpgradeLauncherArgs( + "ghcr.io/agenthits/dokploy:agenthits-dev@sha256:abc", + "/etc/dokploy/host-updates", + ); + const hostCommand = args.at(-1) ?? ""; + + expect(args.slice(0, 9)).toEqual([ + "run", + "--rm", + "--pull=never", + "--privileged", + "--pid=host", + "--network=host", + "--user=0:0", + "--label=com.dokploy.host-update=docker-engine", + "--entrypoint=nsenter", + ]); + expect(args).toContain( + "ghcr.io/agenthits/dokploy:agenthits-dev@sha256:abc", + ); + expect(args).toContain("--target=1"); + expect(hostCommand).toContain( + `exec systemd-run --quiet --collect --unit=${DOCKER_UPGRADE_UNIT}`, + ); + expect(hostCommand).toContain( + "/bin/bash '/etc/dokploy/host-updates/docker-engine-upgrade.sh'", + ); + expect(spawnSync("sh", ["-n"], { input: hostCommand }).status).toBe(0); + }); + + it("stops before anything runs when the host cannot see the script the panel wrote", () => { + const dir = makeTempDir(); + const hostCommand = + buildDockerUpgradeLauncherArgs("image", dir).at(-1) ?? ""; + + const missingScript = spawnSync("sh", ["-c", hostCommand], { + encoding: "utf8", + }); + expect(missingScript.status).toBe(4); + expect(missingScript.stderr).toContain("not mounted from the host"); + }); +}); + +describe("startHostDockerUpgrade", () => { + it("writes the script, a starting status and a fresh log, then launches with the panel image", async () => { + const dir = path.join(makeTempDir(), "host-updates"); + const launches: string[][] = []; + + await startHostDockerUpgrade("29.8.2", { + dir, + readImage: async () => "panel-image:test", + launch: async (args) => { + launches.push(args); + }, + }); + + expect(launches).toHaveLength(1); + expect(launches[0]).toContain("panel-image:test"); + expect( + statSync(path.join(dir, "docker-engine-upgrade.sh")).mode & 0o777, + ).toBe(0o700); + expect(statSync(dir).mode & 0o777).toBe(0o700); + const status = await readHostDockerUpgradeStatus(dir); + expect(status).toMatchObject({ + state: "starting", + from: "29.8.2", + to: PINNED_VERSIONS.docker, + finishedAt: null, + exitCode: null, + }); + expect(status?.log).toContain( + `Starting the Docker Engine upgrade to ${PINNED_VERSIONS.docker} from the panel`, + ); + }); + + it("records a failed start with the launcher's error and nothing changed", async () => { + const dir = path.join(makeTempDir(), "host-updates"); + const failure = Object.assign(new Error("exit 3"), { + stderr: + "This host does not run systemd, so the upgrade cannot run outside Docker from the panel.\n", + }); + + await expect( + startHostDockerUpgrade("29.8.2", { + dir, + readImage: async () => "panel-image:test", + launch: async () => { + throw failure; + }, + }), + ).rejects.toThrow("This host does not run systemd"); + + const status = await readHostDockerUpgradeStatus(dir); + expect(status?.state).toBe("failed"); + expect(status?.finishedAt).not.toBeNull(); + expect(status?.log).toContain("Nothing was changed."); + }); +}); + +describe("readHostDockerUpgradeStatus", () => { + const writeStatus = (dir: string, state: string) => { + mkdirSync(dir, { recursive: true }); + writeFileSync( + path.join(dir, "docker-engine-upgrade.json"), + JSON.stringify({ + state, + from: "29.8.2", + to: "29.9.2", + startedAt: "2026-10-29T10:00:00Z", + finishedAt: null, + exitCode: null, + }), + ); + writeFileSync(path.join(dir, "docker-engine-upgrade.log"), "line\n"); + }; + + it("returns null before the first run and for an unreadable status", async () => { + const dir = makeTempDir(); + expect(await readHostDockerUpgradeStatus(dir)).toBeNull(); + + writeFileSync(path.join(dir, "docker-engine-upgrade.json"), "{broken"); + expect(await readHostDockerUpgradeStatus(dir)).toBeNull(); + }); + + it("reports a running upgrade that stopped writing for half an hour as interrupted", async () => { + const dir = makeTempDir(); + writeStatus(dir, "running"); + const now = Date.now(); + + expect((await readHostDockerUpgradeStatus(dir, now))?.state).toBe( + "running", + ); + + const old = new Date(now - 31 * 60 * 1000); + for (const name of [ + "docker-engine-upgrade.json", + "docker-engine-upgrade.log", + ]) { + utimesSync(path.join(dir, name), old, old); + } + expect((await readHostDockerUpgradeStatus(dir, now))?.state).toBe( + "interrupted", + ); + }); + + it("keeps only the tail of a long log", async () => { + const dir = makeTempDir(); + writeStatus(dir, "succeeded"); + writeFileSync( + path.join(dir, "docker-engine-upgrade.log"), + `${"x".repeat(70 * 1024)}last line\n`, + ); + + const status = await readHostDockerUpgradeStatus(dir); + expect(status?.log.length).toBe(64 * 1024); + expect(status?.log.endsWith("last line\n")).toBe(true); + }); +}); diff --git a/apps/dokploy/__test__/server/host-stack-rows.test.ts b/apps/dokploy/__test__/server/host-stack-rows.test.ts index a3c303e44c3..540152dc8aa 100644 --- a/apps/dokploy/__test__/server/host-stack-rows.test.ts +++ b/apps/dokploy/__test__/server/host-stack-rows.test.ts @@ -53,14 +53,14 @@ describe("buildHostStackRows", () => { } }); - it("keeps the reason of the maintenance-window steps even when they are current", () => { + it("keeps the downtime and maintenance-window reasons even when the rows are current", () => { const rows = rowsFor({}); expect(rowNamed(rows, "docker")).toMatchObject({ - action: "manual", + action: "docker", command: "DOCKER_ENGINE_UPGRADE=1 bash install-agenthits.sh update", }); - expect(rowNamed(rows, "docker").reason).toContain("maintenance window"); + expect(rowNamed(rows, "docker").reason).toContain("restarts Docker"); expect(rowNamed(rows, "postgres")).toMatchObject({ action: "manual", command: null, @@ -83,17 +83,17 @@ describe("buildHostStackRows", () => { } }); - it("marks Docker Engine behind its pin as a maintenance-window step with the engine upgrade command", () => { + it("offers the panel's own upgrade for Docker Engine behind its pin and keeps the manual command", () => { const docker = rowNamed(rowsFor({ docker: "29.4.1" }), "docker"); expect(docker).toMatchObject({ installed: "29.4.1", target: PINNED_VERSIONS.docker, outdated: true, - action: "manual", + action: "docker", command: "DOCKER_ENGINE_UPGRADE=1 bash install-agenthits.sh update", }); - expect(docker.reason).toContain("maintenance window"); + expect(docker.reason).toContain("rolls Docker back"); }); it("compares Docker Engine versions numerically", () => { @@ -295,9 +295,7 @@ describe("host stack update rules", () => { ]); expect(refusals).toHaveLength(3); - expect(refusals[0]).toContain( - "DOCKER_ENGINE_UPGRADE=1 bash install-agenthits.sh update", - ); + expect(refusals[0]).toContain("its own button"); expect(refusals[1]).toContain("POSTGRES_IMAGE"); expect(refusals[2]).toContain("Web Server Update"); expect(hostStackRefusals(["traefik", "redis"])).toEqual([]); diff --git a/apps/dokploy/__test__/server/server-components.test.ts b/apps/dokploy/__test__/server/server-components.test.ts index 82d2e260ace..2f7c2e5a202 100644 --- a/apps/dokploy/__test__/server/server-components.test.ts +++ b/apps/dokploy/__test__/server/server-components.test.ts @@ -37,6 +37,7 @@ import { TRAEFIK_VERSION, } from "@dokploy/server/setup/traefik-setup"; import { describe, expect, it } from "vitest"; +import { BASH, FAKE_TOOLS } from "./docker-script-fakes"; const PORT_80_PUBLICATION = /(?:-p|--publish)[=\s]+\S*\b80\b|published=80\b|\b80\/tcp\b/; @@ -365,14 +366,16 @@ describe("buildComponentUpdateScript", () => { ); }); - it("waits 30 seconds for the new Docker before rolling back", () => { + it("waits 30 seconds for the new Docker before rolling back unless the host sets a longer wait", () => { const script = buildComponentUpdateScript(["docker"]); const wait = script.slice( script.indexOf("docker_wait_ready() {"), script.indexOf("docker_roll_back() {"), ); - expect(wait).toContain('if [ "$attempt" -ge 30 ]; then'); + expect(wait).toContain( + 'if [ "$attempt" -ge "${DOKPLOY_DOCKER_VERIFY_TIMEOUT:-30}" ]; then', + ); expect(wait).toContain("sleep 1"); expect(wait).not.toContain("sleep 2"); }); @@ -442,150 +445,6 @@ describe("filterDockerUpgrade", () => { }); }); -const BASH = spawnSync("sh", ["-c", "command -v bash"], { - encoding: "utf8", -}).stdout.trim(); - -const FAKE_TOOLS: Record = { - docker: `#!/bin/sh -printf 'docker %s\\n' "$*" >> "$CALL_LOG" -case "$1" in - pull) [ "$FAKE_PULL_FAILS" = 1 ] && exit 1 ;; - create) - [ "$FAKE_CREATE_FAILS" = 1 ] && exit 1 - for traefik_arg in "$@"; do traefik_image_arg="$traefik_arg"; done - printf '%s\\n' "$traefik_image_arg" > "$FAKE_STATE/traefik-image" - ;; - version) cat "$FAKE_STATE/engine" ;; - network) - case "$2" in - inspect) case " $FAKE_MISSING_NETWORKS " in *" $3 "*) exit 1 ;; esac ;; - connect) case " $FAKE_CONNECT_FAILS " in *" $3 "*) exit 1 ;; esac ;; - esac - ;; - inspect) - case "$*" in - *Config.Image*) echo "$FAKE_OLD_IMAGE" ;; - *NetworkSettings*) echo "$FAKE_TRAEFIK_NETWORKS" ;; - *PortBindings*) echo "$FAKE_PORT_BINDINGS" ;; - *RestartCount*) - traefik_now="$(cat "$FAKE_STATE/traefik-image" 2>/dev/null || echo "$FAKE_OLD_IMAGE")" - if [ "$FAKE_NOT_RUNNING" = 1 ] && [ "$traefik_now" != "$FAKE_OLD_IMAGE" ]; then echo "false 0"; else echo "true 0"; fi - ;; - *State.Running*) - traefik_now="$(cat "$FAKE_STATE/traefik-image" 2>/dev/null || echo "$FAKE_OLD_IMAGE")" - if [ "$FAKE_NOT_RUNNING" = 1 ] && [ "$traefik_now" != "$FAKE_OLD_IMAGE" ]; then echo false; else echo true; fi - ;; - esac - ;; -esac -exit 0 -`, - "apt-get": `#!/bin/sh -printf 'apt-get %s\\n' "$*" >> "$CALL_LOG" -case "$1" in - update) - [ "$FAKE_UPDATE_FAILS" = 1 ] && exit 1 - ;; - download) - [ "$FAKE_DOWNLOAD_FAILS" = 1 ] && exit 1 - printf 'fake package\\n' > "\${2%%=*}.deb" - ;; - install) - case "$*" in - *--download-only*) - [ "$FAKE_PREDOWNLOAD_FAILS" = 1 ] && exit 1 - ;; - *--allow-downgrades*) - [ "$FAKE_ROLLBACK_FAILS" = 1 ] && exit 1 - printf '%s\\n' "$FAKE_PREVIOUS_ENGINE" > "$FAKE_STATE/engine" - ;; - *--no-download*) - if [ "$FAKE_UPGRADE_KEEPS_OLD" != 1 ]; then - printf '%s\\n' "$FAKE_TARGET_ENGINE" > "$FAKE_STATE/engine" - fi - ;; - esac - ;; -esac -exit 0 -`, - "apt-cache": `#!/bin/sh -printf 'apt-cache %s\\n' "$*" >> "$CALL_LOG" -case "$1" in - madison) - [ "$FAKE_NO_CANDIDATE" = 1 ] && exit 0 - if [ "$FAKE_CANDIDATE_AFTER_REPO" = 1 ] && [ ! -f "$DOKPLOY_APT_SOURCES_DIR/docker.list" ]; then - exit 0 - fi - echo " docker-ce | 5:${PINNED_VERSIONS.docker}-1~ubuntu.24.04~noble | https://download.docker.com/linux/ubuntu noble/stable amd64 Packages" - ;; - policy) - case "$2" in - containerd.io) - candidate=1.7.27-1 - if [ "$FAKE_CONTAINERD_CHANGES" = 1 ]; then candidate=2.0.0-1; fi - printf 'containerd.io:\\n Installed: 1.7.27-1\\n Candidate: %s\\n' "$candidate" - ;; - docker-buildx-plugin) - printf 'docker-buildx-plugin:\\n Installed: 0.25.0-1~ubuntu.24.04~noble\\n Candidate: 0.25.0-1~ubuntu.24.04~noble\\n' - ;; - docker-compose-plugin) - printf 'docker-compose-plugin:\\n Installed: 2.40.0-1~ubuntu.24.04~noble\\n Candidate: 2.40.0-1~ubuntu.24.04~noble\\n' - ;; - esac - ;; -esac -exit 0 -`, - "dpkg-query": `#!/bin/sh -printf 'dpkg-query %s\\n' "$*" >> "$CALL_LOG" -case "$2" in - *Status-Status*) - case "$3" in - docker-ce) [ "$FAKE_NO_DOCKER_CE" = 1 ] || echo installed ;; - docker-ce-cli | containerd.io | docker-buildx-plugin | docker-compose-plugin) echo installed ;; - esac - ;; - *Version*) - case "$3" in - docker-ce | docker-ce-cli | docker-ce-rootless-extras) echo "5:28.3.0-1~ubuntu.24.04~noble" ;; - containerd.io) echo "1.7.27-1" ;; - docker-buildx-plugin) echo "0.25.0-1~ubuntu.24.04~noble" ;; - docker-compose-plugin) echo "2.40.0-1~ubuntu.24.04~noble" ;; - esac - ;; -esac -exit 0 -`, - dpkg: `#!/bin/sh -echo amd64 -`, - systemctl: `#!/bin/sh -printf 'systemctl %s\\n' "$*" >> "$CALL_LOG" -`, - curl: `#!/bin/sh -printf 'curl %s\\n' "$*" >> "$CALL_LOG" -if [ -n "$FAKE_CURL_FAILS_FOR" ]; then - case "$*" in - *"$FAKE_CURL_FAILS_FOR"*) exit 22 ;; - esac -fi -out="" -while [ "$#" -gt 0 ]; do - if [ "$1" = "-o" ]; then out="$2"; fi - shift -done -printf '%s\\n' 'echo fake get-docker "$@"' > "$out" -`, - sudo: `#!/bin/sh -exec "$@" -`, - sleep: `#!/bin/sh -exit 0 -`, -}; - const SYSTEM_TOOLS = [ "id", "awk", diff --git a/apps/dokploy/__test__/settings/docker-server-boundary.test.ts b/apps/dokploy/__test__/settings/docker-server-boundary.test.ts index 9f003944eac..837b6f0bb16 100644 --- a/apps/dokploy/__test__/settings/docker-server-boundary.test.ts +++ b/apps/dokploy/__test__/settings/docker-server-boundary.test.ts @@ -70,6 +70,9 @@ const mocks = vi.hoisted(() => ({ writeMainConfig: vi.fn(), writeTraefikConfigInPath: vi.fn(), writeTraefikSetup: vi.fn(), + readHostDockerUpgradeStatus: vi.fn(async (): Promise => null), + readHostDockerVersion: vi.fn(async (): Promise => "29.8.2"), + startHostDockerUpgrade: vi.fn(async () => {}), })); const redactWebServerSettings = (settings: T) => settings; @@ -91,6 +94,12 @@ vi.mock("@dokploy/server", async () => ({ component === "traefik" || component === "redis", readHostStackRows: vi.fn(), updateHostStackComponents: vi.fn(() => Promise.resolve()), + dockerRow: (await import("@dokploy/server/setup/host-stack-rows")).dockerRow, + isHostDockerUpgradeActive: (status: { state: string } | null) => + status?.state === "starting" || status?.state === "running", + readHostDockerUpgradeStatus: mocks.readHostDockerUpgradeStatus, + readHostDockerVersion: mocks.readHostDockerVersion, + startHostDockerUpgrade: mocks.startHostDockerUpgrade, checkGPUStatus: mocks.checkGPUStatus, checkPortInUse: mocks.checkPortInUse, checkPostgresHealth: mocks.checkPostgresHealth, @@ -930,6 +939,98 @@ describe("panel host update lock", () => { }); }); +describe("panel host Docker upgrade", () => { + const RUNNING = { + state: "running", + from: "29.8.2", + to: "29.9.2", + startedAt: "2026-10-29T10:00:00Z", + finishedAt: null, + exitCode: null, + log: "", + }; + + beforeEach(() => { + vi.clearAllMocks(); + mocks.isServerUpdateRunning.mockReturnValue(false); + mocks.audit.mockResolvedValue(undefined); + mocks.readHostDockerVersion.mockResolvedValue("29.8.2"); + mocks.readHostDockerUpgradeStatus.mockResolvedValue(null); + mocks.startHostDockerUpgrade.mockResolvedValue(undefined); + mocks.getUpdateData.mockResolvedValue({ + updateAvailable: true, + updateSource: "official", + latestVersion: "v2.0.0", + latestOfficialVersion: "v2.0.0", + }); + }); + + it("starts the host upgrade of an outdated Docker Engine and returns its status", async () => { + mocks.readHostDockerUpgradeStatus + .mockResolvedValueOnce(null) + .mockResolvedValueOnce({ ...RUNNING, state: "starting" }); + + await expect(createCaller().upgradeHostDocker()).resolves.toMatchObject({ + state: "starting", + }); + expect(mocks.startHostDockerUpgrade).toHaveBeenCalledWith("29.8.2"); + expect(mocks.audit).toHaveBeenCalledWith( + expect.anything(), + expect.objectContaining({ + resourceName: "panel-host", + metadata: { components: ["docker"] }, + }), + ); + }); + + it("refuses when Docker Engine already runs the pinned version or cannot be read", async () => { + mocks.readHostDockerVersion.mockResolvedValueOnce("99.0.0"); + await expect(createCaller().upgradeHostDocker()).rejects.toMatchObject({ + code: "BAD_REQUEST", + message: "Docker Engine already runs 99.0.0.", + }); + + mocks.readHostDockerVersion.mockResolvedValueOnce(null); + await expect(createCaller().upgradeHostDocker()).rejects.toMatchObject({ + code: "BAD_REQUEST", + }); + expect(mocks.startHostDockerUpgrade).not.toHaveBeenCalled(); + }); + + it("refuses a second upgrade while one runs on the host", async () => { + mocks.readHostDockerUpgradeStatus.mockResolvedValue(RUNNING); + + await expect(createCaller().upgradeHostDocker()).rejects.toMatchObject({ + code: "CONFLICT", + }); + expect(mocks.startHostDockerUpgrade).not.toHaveBeenCalled(); + }); + + it("keeps the panel update and the Traefik and Redis update out while Docker is upgraded", async () => { + mocks.readHostDockerUpgradeStatus.mockResolvedValue(RUNNING); + + await expect(createCaller().updateServer()).rejects.toMatchObject({ + code: "CONFLICT", + }); + await expect( + createCaller().updateHostStack({ components: ["redis"] }), + ).rejects.toMatchObject({ code: "CONFLICT" }); + expect(mocks.startServerUpdate).not.toHaveBeenCalled(); + }); + + it("frees the host-stack lock when the upgrade cannot start", async () => { + mocks.startHostDockerUpgrade.mockRejectedValueOnce( + new Error("The upgrade could not start on the host: no systemd"), + ); + + await expect(createCaller().upgradeHostDocker()).rejects.toMatchObject({ + code: "INTERNAL_SERVER_ERROR", + message: "The upgrade could not start on the host: no systemd", + }); + await expect(createCaller().upgradeHostDocker()).resolves.toBeNull(); + }); +}); + describe("Traefik operations next to a certificate deletion", () => { const ports = { additionalPorts: [ diff --git a/apps/dokploy/components/dashboard/settings/servers/panel-host-stack.tsx b/apps/dokploy/components/dashboard/settings/servers/panel-host-stack.tsx index 89c72ec16f5..eea12ccf42f 100644 --- a/apps/dokploy/components/dashboard/settings/servers/panel-host-stack.tsx +++ b/apps/dokploy/components/dashboard/settings/servers/panel-host-stack.tsx @@ -1,3 +1,4 @@ +import type { HostDockerUpgradeStatus } from "@dokploy/server/setup/host-docker-upgrade"; import type { HostStackComponent, HostStackRow, @@ -8,8 +9,9 @@ import { Loader2, RefreshCw, Server, + XCircle, } from "lucide-react"; -import { useRef, useState } from "react"; +import { useEffect, useRef, useState } from "react"; import { toast } from "sonner"; import { AlertDialog, @@ -62,6 +64,85 @@ const WARNINGS: Partial> = { redis: "Redis is restarted; it is unavailable for a few seconds.", }; +const DOCKER_POLL_MS = 2000; +const RECENT_RUN_MS = 24 * 60 * 60 * 1000; + +const isDockerUpgradeActive = (status: HostDockerUpgradeStatus | null) => + status?.state === "starting" || status?.state === "running"; + +const isRecentRun = (status: HostDockerUpgradeStatus) => { + const finished = Date.parse(status.finishedAt ?? status.startedAt); + return Number.isFinite(finished) && Date.now() - finished < RECENT_RUN_MS; +}; + +const DockerUpgradeHeadline = ({ + status, + connectionLost, +}: { + status: HostDockerUpgradeStatus; + connectionLost: boolean; +}) => { + const versions = `${status.from ?? "unknown"} → ${status.to}`; + if (isDockerUpgradeActive(status)) { + return ( +

+ + + {connectionLost + ? "Docker is restarting, and this panel with it. Waiting for the panel to come back; this window reconnects by itself." + : `Upgrading Docker Engine ${versions} on the host. Sites and this panel stop for a minute or two.`} + +

+ ); + } + if (status.state === "succeeded") { + return ( +

+ + Docker Engine upgraded: {versions}. +

+ ); + } + if (status.state === "interrupted") { + return ( +

+ + + The Docker Engine upgrade stopped reporting before it finished. Check + the host with: journalctl -u dokploy-docker-engine-upgrade + +

+ ); + } + return ( +

+ + + The Docker Engine upgrade failed + {status.exitCode !== null ? ` (exit code ${status.exitCode})` : ""}. The + log below says whether Docker was rolled back to {status.from}. + +

+ ); +}; + +const DockerUpgradeProgress = ({ + status, + connectionLost, +}: { + status: HostDockerUpgradeStatus; + connectionLost: boolean; +}) => ( +
+ + {status.log && ( + 
+				{status.log}
+
+ )} +
+); + const StatusLine = ({ row }: { row: HostStackRow }) => { if (row.status === "outdated") { return ( @@ -87,11 +168,15 @@ const HostStackRowItem = ({ checked, onCheckedChange, onOpenWebUpdate, + dockerUpgradeBusy, + onUpgradeDocker, }: { row: HostStackRow; checked: boolean; onCheckedChange: (checked: boolean) => void; onOpenWebUpdate: () => void; + dockerUpgradeBusy: boolean; + onUpgradeDocker: () => void; }) => { const checkboxId = `host-stack-${row.name}`; return ( @@ -128,8 +213,21 @@ const HostStackRowItem = ({ )} + {row.action === "docker" && row.outdated && ( + + )} {row.reason && - (row.action === "manual" || row.status === "unknown") && ( + (row.action === "manual" || + row.status === "unknown" || + (row.action === "docker" && row.outdated)) && (

{row.reason}

)} {row.action === "manual" && row.command && ( @@ -137,6 +235,18 @@ const HostStackRowItem = ({ {row.command} )} + {row.action === "docker" && + row.command && + (row.outdated || row.status === "unknown") && ( +
+ + Or run it on the host yourself + + + {row.command} + +
+ )} ); @@ -159,6 +269,51 @@ const PanelHostCard = ({ const [runComponents, setRunComponents] = useState([]); const [logs, setLogs] = useState(""); const logBuffer = useRef(""); + const [isDockerConfirmOpen, setIsDockerConfirmOpen] = useState(false); + const [isWatchingDocker, setIsWatchingDocker] = useState(false); + const utils = api.useUtils(); + + // The panel itself goes down while Docker restarts, so polling keeps going + // through failed requests until the host reports the run finished. + const dockerUpgrade = api.settings.hostDockerUpgrade.useQuery(undefined, { + refetchOnWindowFocus: false, + retry: false, + refetchInterval: (query) => + isWatchingDocker || isDockerUpgradeActive(query.state.data ?? null) + ? DOCKER_POLL_MS + : false, + }); + const dockerStatus = dockerUpgrade.data ?? null; + const dockerActive = isDockerUpgradeActive(dockerStatus); + const dockerRow = rows.find((row) => row.name === "docker"); + + const upgradeDocker = api.settings.upgradeHostDocker.useMutation({ + onSuccess(status) { + utils.settings.hostDockerUpgrade.setData(undefined, status); + setIsWatchingDocker(true); + }, + onError(err) { + toast.error(err.message); + void dockerUpgrade.refetch(); + }, + }); + + useEffect(() => { + if (dockerActive) { + setIsWatchingDocker(true); + return; + } + if (!isWatchingDocker || !dockerStatus) { + return; + } + setIsWatchingDocker(false); + if (dockerStatus.state === "succeeded") { + toast.success(`Docker Engine upgraded to ${dockerStatus.to}`); + } else { + toast.error("Docker Engine upgrade did not finish, see the log"); + } + void refetch(); + }, [dockerActive, dockerStatus, isWatchingDocker, refetch]); const outdatedCount = rows.filter((row) => row.outdated).length; const unknownCount = rows.filter((row) => row.status === "unknown").length; @@ -277,10 +432,26 @@ const PanelHostCard = ({ setIsOpen(false); setIsWebUpdateOpen(true); }} + dockerUpgradeBusy={ + dockerActive || + isWatchingDocker || + upgradeDocker.isPending + } + onUpgradeDocker={() => setIsDockerConfirmOpen(true)} /> ))} + {dockerStatus && + (dockerActive || + isWatchingDocker || + isRecentRun(dockerStatus)) && ( + + )} + {selected.map((name) => WARNINGS[name] ? (

@@ -297,7 +468,7 @@ const PanelHostCard = ({ {isUpdating && } {selected.length > 0 ? `Update ${selected.map((name) => LABELS[name]).join(" and ")}` - : "Nothing to update here"} + : "Traefik and Redis: nothing to update"} {logs && ( @@ -338,6 +509,45 @@ const PanelHostCard = ({ + + + + + + Upgrade Docker Engine to {dockerRow?.target} on the panel host? + + +

+ + Docker restarts, and every container on this server restarts + with it: all sites and this panel are unavailable for a minute + or two. + + + The upgrade runs on the host, outside the panel. If the + services do not come back healthy, it rolls Docker back to{" "} + {dockerRow?.installed} by itself. Keep this window open; it + reconnects when the panel is back. + +
+ + + + Cancel + { + setIsDockerConfirmOpen(false); + upgradeDocker.mutate(); + }} + > + Upgrade Docker + + + + ); }; diff --git a/apps/dokploy/server/api/routers/settings.ts b/apps/dokploy/server/api/routers/settings.ts index 0113b1263cd..3be6c914e86 100644 --- a/apps/dokploy/server/api/routers/settings.ts +++ b/apps/dokploy/server/api/routers/settings.ts @@ -18,6 +18,7 @@ import { DOKPLOY_KEEP_IMAGES_MIN, diskHistoryInputSchema, dockerDiskUsageDetailLimitSchema, + dockerRow, filterProtectedTraefikEntries, findServerById, getAgentHitsUpdateCommand, @@ -40,6 +41,7 @@ import { HOST_STACK_COMPONENTS, hostStackRefusals, IS_CLOUD, + isHostDockerUpgradeActive, isHostStackUiComponent, isHostStackUpdateRunning, isServerUpdateRunning, @@ -53,6 +55,8 @@ import { readDirectory, readEnvironmentVariableList, readEnvironmentVariables, + readHostDockerUpgradeStatus, + readHostDockerVersion, readHostStackRows, readMainConfig, readMonitoringConfig, @@ -65,6 +69,7 @@ import { resolveDockerDiskUsageDetailLimit, sendDockerCleanupNotifications, setupGPUSupport, + startHostDockerUpgrade, startLogCleanup, startServerUpdate, stopLogCleanup, @@ -258,6 +263,13 @@ export const settingsRouter = createTRPCRouter({ message: refusals.join(" "), }); } + if (isHostDockerUpgradeActive(await readHostDockerUpgradeStatus())) { + throw new TRPCError({ + code: "CONFLICT", + message: + "Docker Engine is being upgraded on this host. Try again when it finishes.", + }); + } const releaseHostStackLock = isServerUpdateRunning() ? null : acquireHostStackUpdateLock(); @@ -295,6 +307,77 @@ export const settingsRouter = createTRPCRouter({ }); }); }), + hostDockerUpgrade: adminProcedure.query(async ({ ctx }) => { + await assertLocalHostAccess(ctx); + if (IS_CLOUD) { + return null; + } + return await readHostDockerUpgradeStatus(); + }), + upgradeHostDocker: adminProcedure + .meta({ + openapi: { + path: "/settings/upgrade-host-docker", + method: "POST", + override: true, + enabled: false, + }, + }) + .mutation(async ({ ctx }) => { + await assertLocalHostAccess(ctx); + if (IS_CLOUD) { + throw new TRPCError({ + code: "FORBIDDEN", + message: "The panel host components are not managed here.", + }); + } + const docker = dockerRow(await readHostDockerVersion()); + if (!docker.outdated) { + throw new TRPCError({ + code: "BAD_REQUEST", + message: + docker.installed === null + ? "The installed Docker Engine version could not be read on the panel host." + : `Docker Engine already runs ${docker.installed}.`, + }); + } + const releaseHostStackLock = isServerUpdateRunning() + ? null + : acquireHostStackUpdateLock(); + if (!releaseHostStackLock) { + throw new TRPCError({ + code: "CONFLICT", + message: + "Another update of this panel host is running. Try again when it finishes.", + }); + } + try { + if (isHostDockerUpgradeActive(await readHostDockerUpgradeStatus())) { + throw new TRPCError({ + code: "CONFLICT", + message: "Docker Engine is already being upgraded on this host.", + }); + } + await audit(ctx, { + action: "update", + resourceType: "settings", + resourceName: "panel-host", + metadata: { components: ["docker"] }, + }); + await startHostDockerUpgrade(docker.installed); + } catch (error) { + if (error instanceof TRPCError) { + throw error; + } + throw new TRPCError({ + code: "INTERNAL_SERVER_ERROR", + message: error instanceof Error ? error.message : String(error), + }); + } finally { + releaseHostStackLock(); + } + return await readHostDockerUpgradeStatus(); + }), toggleDashboard: adminProcedure .input(apiEnableDashboard) .mutation(async ({ input, ctx }) => { @@ -793,7 +876,10 @@ export const settingsRouter = createTRPCRouter({ const keepImages = input?.keepImages; const data = await getUpdateData(packageInfo.version); // After the await: a host-stack update can start while the update check runs. - if (isHostStackUpdateRunning()) { + if ( + isHostStackUpdateRunning() || + isHostDockerUpgradeActive(await readHostDockerUpgradeStatus()) + ) { throw new TRPCError({ code: "CONFLICT", message: diff --git a/apps/dokploy/server/api/utils/procedure-access.ts b/apps/dokploy/server/api/utils/procedure-access.ts index 5eea0c3bc22..4a7ff84ffb7 100644 --- a/apps/dokploy/server/api/utils/procedure-access.ts +++ b/apps/dokploy/server/api/utils/procedure-access.ts @@ -626,6 +626,7 @@ export const PROCEDURE_ACCESS: Record = { "settings.haveActivateRequests": "read", "settings.haveTraefikDashboardPortEnabled": "read", "settings.health": "read", + "settings.hostDockerUpgrade": "read", "settings.hostStack": "read", "settings.isCloud": "read", "settings.isUserSubscribed": "read", @@ -657,6 +658,7 @@ export const PROCEDURE_ACCESS: Record = { "settings.updateTraefik": "dangerous", "settings.updateTraefikPorts": "dangerous", "settings.updateWebServerTraefikConfig": "dangerous", + "settings.upgradeHostDocker": "dangerous", "settings.writeTraefikEnv": "dangerous", "sshKey.all": "read", "sshKey.allForApps": "read", diff --git a/docs/server-maintenance.md b/docs/server-maintenance.md index 40743d3f9a5..cb72e17979f 100644 --- a/docs/server-maintenance.md +++ b/docs/server-maintenance.md @@ -7,13 +7,29 @@ В карточке **Panel host (this server)** есть два способа обновления: - **Components** обновляет **Traefik** (веб-сервер, через который идёт весь HTTPS-трафик) и **Redis** (кэш панели). Во время замены Traefik на несколько секунд недоступны все сайты и админка. Во время замены Redis он на несколько секунд недоступен. +- В том же окне **Components** у строки **Docker** есть своя кнопка **Upgrade to <версия>**. Она обновляет Docker Engine; подробности ниже. - **Web Server Update** обновляет саму **панель**. +### Обновление Docker из панели + +При обновлении Docker перезапускается его демон, а вместе с ним все контейнеры сервера: сайты и сама панель недоступны минуту-две. Поэтому панель не обновляет Docker сама изнутри своего контейнера. Она записывает скрипт в `/etc/dokploy/host-updates/` и запускает его на сервере как отдельную службу systemd `dokploy-docker-engine-upgrade`. Эта служба работает вне Docker и переживает его перезапуск. + +Что делает скрипт: + +1. Скачивает новые пакеты Docker и сохраняет текущие для отката в `/var/backups/dokploy`. Если скачать не удалось, сервер не меняется. +2. Ставит новые пакеты. Docker перезапускается. +3. До пяти минут ждёт, пока снова заработают Traefik, панель, Postgres и Redis. +4. Если они не поднялись, сам возвращает прежнюю версию Docker из сохранённых пакетов. + +Окно с кнопкой можно не закрывать: пока панель перезапускается, оно ждёт и потом само показывает журнал и итог. Журнал лежит в `/etc/dokploy/host-updates/docker-engine-upgrade.log`, на сервере его видно и через `journalctl -u dokploy-docker-engine-upgrade`. + +Кнопка работает на серверах с systemd и apt (Ubuntu, Debian, машина OrbStack на Mac). На других серверах панель покажет причину отказа, и тогда Docker обновляют командой из раздела 2. + ## 2. Что обновляется только в плановое окно -Docker и Postgres кнопкой не обновляются. Карточка показывает для них команду, которую запускают в заранее выбранное время. +Postgres кнопкой не обновляется. Для Docker кнопка есть (см. выше), но простой при обновлении такой же, как при ручном запуске. Поэтому его тоже лучше запускать в тихое время и заранее предупреждать пользователей. -**Docker Engine.** При обновлении перезапускается демон Docker. Вместе с ним останавливаются все контейнеры сервера, включая панель. Поэтому это делают один раз, в объявленное окно, и предупреждают пользователей о простое. Команда: `DOCKER_ENGINE_UPGRADE=1 bash install-agenthits.sh update`. +**Docker Engine вручную.** Если кнопка недоступна, Docker обновляют командой на сервере: `DOCKER_ENGINE_UPGRADE=1 bash install-agenthits.sh update`. Эта же команда видна в карточке под строкой Docker. **Postgres (база панели).** Смена major-версии (например, 17 → 18) требует переноса данных. Это отдельная работа, а не обновление. Поэтому сервер сохраняет текущую major-версию: установщик её не меняет. Сейчас с Postgres ничего делать не нужно. Обновление внутри одной major-версии (например, 18.5 → 18.6) выполняет та же команда обновления. Перед заменой она сохраняет резервную копию базы. diff --git a/packages/server/src/index.ts b/packages/server/src/index.ts index 27b07df27f5..279fc1575df 100644 --- a/packages/server/src/index.ts +++ b/packages/server/src/index.ts @@ -73,6 +73,7 @@ export * from "./services/web-server-settings"; export * from "./services/web-server-update"; export * from "./setup/config-paths"; export * from "./setup/forward-auth-setup"; +export * from "./setup/host-docker-upgrade"; export * from "./setup/host-stack"; export * from "./setup/host-stack-lock"; export * from "./setup/host-stack-rows"; diff --git a/packages/server/src/setup/host-docker-upgrade.ts b/packages/server/src/setup/host-docker-upgrade.ts new file mode 100644 index 00000000000..b4edace60f4 --- /dev/null +++ b/packages/server/src/setup/host-docker-upgrade.ts @@ -0,0 +1,321 @@ +import { + appendFile, + chmod, + mkdir, + readFile, + rename, + stat, + writeFile, +} from "node:fs/promises"; +import path from "node:path"; +import { spawnAsync } from "../utils/process/spawnAsync"; +import { PINNED_VERSIONS } from "./component-versions"; +import { buildDockerEngineUpgradeSteps } from "./server-components"; + +// The panel and the host share /etc/dokploy, so the panel writes the script and +// reads its log and status here while the script itself runs on the host. +export const HOST_UPDATES_DIR = "/etc/dokploy/host-updates"; +export const DOCKER_UPGRADE_UNIT = "dokploy-docker-engine-upgrade"; + +const SCRIPT_NAME = "docker-engine-upgrade.sh"; +const LOG_NAME = "docker-engine-upgrade.log"; +const STATUS_NAME = "docker-engine-upgrade.json"; +const LOG_TAIL_BYTES = 64 * 1024; +// The longest silent stretch of a run is the health wait after the restart and +// the same wait after a rollback; a run quiet for longer than this has died. +const STALE_AFTER_MS = 30 * 60 * 1000; + +export type HostDockerUpgradeState = + | "starting" + | "running" + | "succeeded" + | "failed" + | "interrupted"; + +export type HostDockerUpgradeStatus = { + state: HostDockerUpgradeState; + from: string | null; + to: string; + startedAt: string; + finishedAt: string | null; + exitCode: number | null; + log: string; +}; + +type StoredStatus = Omit; + +const STATES: readonly HostDockerUpgradeState[] = [ + "starting", + "running", + "succeeded", + "failed", + "interrupted", +]; + +export const isHostDockerUpgradeActive = ( + status: Pick | null, +) => status?.state === "starting" || status?.state === "running"; + +const filesIn = (dir: string) => ({ + script: path.join(dir, SCRIPT_NAME), + log: path.join(dir, LOG_NAME), + status: path.join(dir, STATUS_NAME), +}); + +const shellQuote = (value: string) => `'${value.replaceAll("'", "'\\''")}'`; + +const HOST_READY_CHECK = ` + [ "$(docker info --format '{{.Swarm.LocalNodeState}}' 2>/dev/null)" = "active" ] || return 1 + local service="" container="" + for service in dokploy dokploy-postgres dokploy-redis; do + container="$(docker ps -q --no-trunc --filter "label=com.docker.swarm.service.name=$service" --filter status=running | head -n1)" + [ -n "$container" ] || return 1 + case "$service" in + dokploy) + case "$(docker inspect --format '{{if .State.Health}}{{.State.Health.Status}}{{else}}none{{end}}' "$container" 2>/dev/null)" in + healthy | none) ;; + *) return 1 ;; + esac + ;; + dokploy-postgres) + docker exec "$container" pg_isready -U dokploy -d dokploy >/dev/null 2>&1 || return 1 + ;; + dokploy-redis) + [ "$(docker exec "$container" redis-cli ping 2>/dev/null)" = "PONG" ] || return 1 + ;; + esac + done`; + +export const buildHostDockerUpgradeScript = (dir = HOST_UPDATES_DIR) => { + const files = filesIn(dir); + return `#!/usr/bin/env bash +# Runs on the panel host as a transient systemd unit, outside Docker, so the +# daemon restart below does not stop it the way it stops the panel. +STATUS_FILE=${shellQuote(files.status)} +LOG_FILE=${shellQuote(files.log)} +TARGET_VERSION=${shellQuote(PINNED_VERSIONS.docker)} +exec >>"$LOG_FILE" 2>&1 +export DEBIAN_FRONTEND=noninteractive +# needrestart would otherwise restart unrelated host services after apt-get. +export NEEDRESTART_MODE=l +DOKPLOY_DOCKER_VERIFY_TIMEOUT="\${DOKPLOY_DOCKER_VERIFY_TIMEOUT:-300}" +started_at="$(date -u +%Y-%m-%dT%H:%M:%SZ)" +previous_version="$(docker version --format '{{.Server.Version}}' 2>/dev/null | tr -cd '0-9A-Za-z.+~-' || true)" +write_status() { + local finished="null" from="null" + if [ -n "$3" ]; then + finished="\\"$3\\"" + fi + if [ -n "$previous_version" ]; then + from="\\"$previous_version\\"" + fi + printf '{"state":"%s","from":%s,"to":"%s","startedAt":"%s","finishedAt":%s,"exitCode":%s}\\n' "$1" "$from" "$TARGET_VERSION" "$started_at" "$finished" "$2" >"$STATUS_FILE.tmp" && mv -f "$STATUS_FILE.tmp" "$STATUS_FILE" +} +finish_status() { + local code=$? + if [ "$code" = 0 ]; then + write_status succeeded 0 "$(date -u +%Y-%m-%dT%H:%M:%SZ)" + else + write_status failed "$code" "$(date -u +%Y-%m-%dT%H:%M:%SZ)" + fi +} +write_status running null "" +trap finish_status EXIT +echo "Docker Engine upgrade started on the host at $started_at: \${previous_version:-unknown} -> $TARGET_VERSION" +set -e +SUDO_CMD="" +${buildDockerEngineUpgradeSteps(HOST_READY_CHECK)} +echo "Docker Engine upgrade finished ✅" +`; +}; + +// Runs inside a one-off privileged container that enters the host's +// namespaces, so systemd-run starts the script as a host unit that outlives +// both that container and the Docker daemon restart. +export const buildDockerUpgradeLauncherArgs = ( + image: string, + dir = HOST_UPDATES_DIR, +) => { + const script = filesIn(dir).script; + const hostCommand = [ + `[ -f ${shellQuote(script)} ] || { echo "The upgrade script is not on the host at ${script}: the panel's /etc/dokploy is not mounted from the host." >&2; exit 4; }`, + `[ -d /run/systemd/system ] && command -v systemd-run >/dev/null 2>&1 || { echo "This host does not run systemd, so the upgrade cannot run outside Docker from the panel." >&2; exit 3; }`, + `exec systemd-run --quiet --collect --unit=${DOCKER_UPGRADE_UNIT} --description="Dokploy Docker Engine upgrade" /bin/bash ${shellQuote(script)}`, + ].join("\n"); + return [ + "run", + "--rm", + "--pull=never", + "--privileged", + "--pid=host", + "--network=host", + "--user=0:0", + "--label=com.dokploy.host-update=docker-engine", + "--entrypoint=nsenter", + image, + "--target=1", + "--mount", + "--uts", + "--ipc", + "--net", + "--pid", + "--", + "/bin/sh", + "-c", + hostCommand, + ]; +}; + +const parseStoredStatus = (raw: string): StoredStatus | null => { + let value: unknown; + try { + value = JSON.parse(raw); + } catch { + return null; + } + if (typeof value !== "object" || value === null) { + return null; + } + const record = value as Record; + const state = STATES.find((candidate) => candidate === record.state); + if (!state || typeof record.to !== "string") { + return null; + } + return { + state, + from: typeof record.from === "string" ? record.from : null, + to: record.to, + startedAt: typeof record.startedAt === "string" ? record.startedAt : "", + finishedAt: + typeof record.finishedAt === "string" ? record.finishedAt : null, + exitCode: typeof record.exitCode === "number" ? record.exitCode : null, + }; +}; + +const readIfExists = async (file: string) => { + try { + return await readFile(file, "utf8"); + } catch (error) { + if ((error as NodeJS.ErrnoException).code === "ENOENT") { + return null; + } + throw error; + } +}; + +const lastModified = async (file: string) => { + try { + return (await stat(file)).mtimeMs; + } catch { + return 0; + } +}; + +const writeStoredStatus = async (dir: string, status: StoredStatus) => { + const file = filesIn(dir).status; + await writeFile(`${file}.tmp`, `${JSON.stringify(status)}\n`, { + mode: 0o600, + }); + await rename(`${file}.tmp`, file); +}; + +export const readHostDockerUpgradeStatus = async ( + dir = HOST_UPDATES_DIR, + now = Date.now(), +): Promise => { + const files = filesIn(dir); + const raw = await readIfExists(files.status); + const stored = raw === null ? null : parseStoredStatus(raw); + if (!stored) { + return null; + } + const log = (await readIfExists(files.log)) ?? ""; + const status: HostDockerUpgradeStatus = { + ...stored, + log: log.length > LOG_TAIL_BYTES ? log.slice(-LOG_TAIL_BYTES) : log, + }; + if (isHostDockerUpgradeActive(status)) { + const touched = Math.max( + await lastModified(files.status), + await lastModified(files.log), + ); + if (now - touched > STALE_AFTER_MS) { + return { ...status, state: "interrupted" }; + } + } + return status; +}; + +const readPanelImage = async () => { + const output = await spawnAsync("docker", [ + "service", + "inspect", + "dokploy", + "--format", + "{{.Spec.TaskTemplate.ContainerSpec.Image}}", + ]); + const image = output.toString().trim(); + if (!image) { + throw new Error("The image of the dokploy service could not be read."); + } + return image; +}; + +const errorText = (error: unknown) => { + const stderr = (error as { stderr?: { toString(): string } }).stderr; + const text = stderr?.toString().trim(); + if (text) { + return text; + } + return error instanceof Error ? error.message : String(error); +}; + +// Callers check that no run is active and hold the host-stack lock until this +// returns; the run itself goes on in the host unit after that. +export const startHostDockerUpgrade = async ( + from: string | null, + { + dir = HOST_UPDATES_DIR, + readImage = readPanelImage, + launch = (args: string[]) => spawnAsync("docker", args), + }: { + dir?: string; + readImage?: () => Promise; + launch?: (args: string[]) => Promise; + } = {}, +) => { + const files = filesIn(dir); + await mkdir(dir, { recursive: true, mode: 0o700 }); + await writeFile(files.script, buildHostDockerUpgradeScript(dir), { + mode: 0o700, + }); + await chmod(files.script, 0o700); + const startedAt = new Date().toISOString(); + await writeFile( + files.log, + `Starting the Docker Engine upgrade to ${PINNED_VERSIONS.docker} from the panel at ${startedAt}\n`, + { mode: 0o600 }, + ); + const starting: StoredStatus = { + state: "starting", + from, + to: PINNED_VERSIONS.docker, + startedAt, + finishedAt: null, + exitCode: null, + }; + await writeStoredStatus(dir, starting); + try { + const image = await readImage(); + await launch(buildDockerUpgradeLauncherArgs(image, dir)); + } catch (error) { + const message = `The upgrade could not start on the host: ${errorText(error)}`; + await appendFile(files.log, `Error: ${message}\nNothing was changed.\n`); + await writeStoredStatus(dir, { + ...starting, + state: "failed", + finishedAt: new Date().toISOString(), + }); + throw new Error(message); + } +}; diff --git a/packages/server/src/setup/host-stack-rows.ts b/packages/server/src/setup/host-stack-rows.ts index 14ecc22eb23..12640d3d433 100644 --- a/packages/server/src/setup/host-stack-rows.ts +++ b/packages/server/src/setup/host-stack-rows.ts @@ -15,7 +15,7 @@ export const HOST_STACK_UI_COMPONENTS = ["traefik", "redis"] as const; export type HostStackUiComponent = (typeof HOST_STACK_UI_COMPONENTS)[number]; -export type HostStackAction = "panel" | "ui" | "manual"; +export type HostStackAction = "panel" | "ui" | "docker" | "manual"; export type HostStackStatus = "current" | "outdated" | "unknown"; @@ -54,7 +54,7 @@ const UNKNOWN_REASON = const PANEL_UNKNOWN_REASON = "The update check did not return a version for this panel."; const DOCKER_REASON = - "Docker Engine upgrades restart the daemon, which stops this panel. Run them only in the maintenance window."; + "Upgrading restarts Docker: every site on this server and this panel stop for a minute or two. The upgrade runs on the host and rolls Docker back by itself if the services do not come back healthy."; const POSTGRES_REASON = "Postgres changes only in the maintenance window, and keeps its major version."; @@ -63,7 +63,8 @@ const HOST_STACK_REFUSALS: Record< string > = { panel: "The panel is updated from the Web Server Update dialog.", - docker: `Docker Engine is not upgraded from the panel because the daemon restart stops it. In the maintenance window run: ${DOCKER_UPGRADE_COMMAND}`, + docker: + "Docker Engine is upgraded with its own button in the Panel host components list, because the daemon restart stops this panel.", postgres: "Postgres is not updated from the panel, and its major version never changes here. In the maintenance window run the host update with POSTGRES_IMAGE set to the image Postgres runs now (see the Panel host components list).", }; @@ -145,12 +146,12 @@ const panelRow = ({ }; }; -const dockerRow = (readout: string | null): HostStackRow => { +export const dockerRow = (readout: string | null): HostStackRow => { const target = PINNED_VERSIONS.docker; const installed = versionOf(readout); if (installed === null) { return { - ...unknownRow("docker", target, "manual"), + ...unknownRow("docker", target, "docker"), command: DOCKER_UPGRADE_COMMAND, }; } @@ -161,7 +162,7 @@ const dockerRow = (readout: string | null): HostStackRow => { target, status: outdated ? "outdated" : "current", outdated, - action: "manual", + action: "docker", reason: DOCKER_REASON, command: DOCKER_UPGRADE_COMMAND, }; diff --git a/packages/server/src/setup/host-stack.ts b/packages/server/src/setup/host-stack.ts index 0b40b4f46b1..0895bb531ae 100644 --- a/packages/server/src/setup/host-stack.ts +++ b/packages/server/src/setup/host-stack.ts @@ -41,12 +41,15 @@ const checkPanelUpdate = async (currentVersion: string) => { } }; +export const readHostDockerVersion = () => + readCommand("docker version --format '{{.Server.Version}}'"); + export const readHostStackRows = async ( currentVersion: string, ): Promise => { const [docker, traefikImage, postgresImage, redisImage, update] = await Promise.all([ - readCommand("docker version --format '{{.Server.Version}}'"), + readHostDockerVersion(), readCommand("docker inspect -f '{{.Config.Image}}' dokploy-traefik"), readCommand( "docker service inspect dokploy-postgres --format '{{.Spec.TaskTemplate.ContainerSpec.Image}}'", diff --git a/packages/server/src/setup/server-components.ts b/packages/server/src/setup/server-components.ts index 9dca2c9b643..02436b4defa 100644 --- a/packages/server/src/setup/server-components.ts +++ b/packages/server/src/setup/server-components.ts @@ -323,18 +323,24 @@ docker_prepare_apt() { return 0 } +# The panel host replaces this with a check of its own swarm services. +docker_host_ready() { + return 0 +} + docker_ready_on() { [ "$($SUDO_CMD docker version --format '{{.Server.Version}}' 2>/dev/null)" = "$1" ] || return 1 if [ "$2" = 1 ]; then [ "$($SUDO_CMD docker inspect -f '{{.State.Running}}' dokploy-traefik 2>/dev/null)" = "true" ] || return 1 fi + docker_host_ready } docker_wait_ready() { local attempt=0 until docker_ready_on "$1" "$2"; do attempt=$((attempt + 1)) - if [ "$attempt" -ge 30 ]; then + if [ "$attempt" -ge "\${DOKPLOY_DOCKER_VERIFY_TIMEOUT:-30}" ]; then return 1 fi sleep 1 @@ -607,6 +613,23 @@ ${predownloadGate(components)} ${components.map(updateStepFor).join("\n")} `; +// On the panel host a skipped Docker upgrade is the whole job failing, and the +// host is ready only when its own services are back, so both differ from the +// remote-server step above. +export const buildDockerEngineUpgradeSteps = (hostReadyCheck: string) => ` +${predownloadGate(["docker"])} +if [ "$DOCKER_SKIPPED" = 1 ]; then + echo "Error: Docker not upgraded: $DOCKER_SKIP_REASON. Nothing was changed." >&2 + exit 1 +fi +docker_host_ready() { +${hostReadyCheck} +} +echo "Upgrading Docker Engine to ${PINNED_VERSIONS.docker}. The daemon restarts, and every container on this server restarts with it." +docker_update_from_apt +echo "Docker version ${PINNED_VERSIONS.docker} installed ✅" +`; + export const wrapComponentUpdateSteps = (steps: string) => ` set -e if [ "$(id -u)" -eq 0 ]; then SUDO_CMD=""; else SUDO_CMD="sudo"; fi From ada0cb585d22a0a9d3f83d00e0de74c707ec1af8 Mon Sep 17 00:00:00 2001 From: merge-check <140916359+agentHits@users.noreply.github.com> Date: Thu, 29 Oct 2026 07:42:37 -0400 Subject: [PATCH 2/2] =?UTF-8?q?fix(panel-host):=20=D0=B6=D0=B4=D0=B0=D1=82?= =?UTF-8?q?=D1=8C=20Docker=20=D0=BF=D0=BE=20=D1=87=D0=B0=D1=81=D0=B0=D0=BC?= =?UTF-8?q?=20=D0=B8=20=D0=BF=D0=B8=D1=81=D0=B0=D1=82=D1=8C=20=D0=B2=20?= =?UTF-8?q?=D0=B6=D1=83=D1=80=D0=BD=D0=B0=D0=BB,=20=D1=87=D0=B5=D0=B3?= =?UTF-8?q?=D0=BE=20=D0=B6=D0=B4=D1=91=D0=BC?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Первый реальный запуск на сервере (Ubuntu 24.04 в OrbStack) откатил Docker 29.9.2: за отведённое время сервисы не признали готовыми. Ожидание считало попытки, а не секунды. Одна попытка с проверками swarm, панели, Postgres и Redis на нагруженном сервере шла около двух секунд, поэтому «300 секунд» растянулись на 9 минут. По журналу при этом нельзя было понять, какая проверка не проходила. Теперь docker_wait_ready ждёт до срока по date +%s. Раз в 30 секунд он пишет в журнал, чего ждёт: версию Docker, Traefik, задачу swarm, здоровье панели, Postgres или Redis. При истечении срока та же причина попадает в текст ошибки. Это касается и обновления Docker на удалённых серверах. Co-Authored-By: Claude Opus 5.5 --- .../__test__/server/host-docker-upgrade.test.ts | 3 +++ .../__test__/server/server-components.test.ts | 5 ++++- .../server/src/setup/host-docker-upgrade.ts | 13 ++++++++++--- packages/server/src/setup/server-components.ts | 17 ++++++++++++++--- 4 files changed, 31 insertions(+), 7 deletions(-) diff --git a/apps/dokploy/__test__/server/host-docker-upgrade.test.ts b/apps/dokploy/__test__/server/host-docker-upgrade.test.ts index 069c1034c6f..3b53e1f2132 100644 --- a/apps/dokploy/__test__/server/host-docker-upgrade.test.ts +++ b/apps/dokploy/__test__/server/host-docker-upgrade.test.ts @@ -145,6 +145,9 @@ describe("buildHostDockerUpgradeScript", { timeout: 60_000 }, () => { expect(run.log).toContain( `Error: Docker ${PINNED_VERSIONS.docker} did not come up healthy after the install.`, ); + expect(run.log).toContain( + `Not ready on Docker ${PINNED_VERSIONS.docker} after 3 seconds: the panel container is starting`, + ); expect(run.log).toContain(`Rolling Docker back to ${PREVIOUS_ENGINE}`); expect(run.stored).toMatchObject({ state: "failed", diff --git a/apps/dokploy/__test__/server/server-components.test.ts b/apps/dokploy/__test__/server/server-components.test.ts index 2f7c2e5a202..e19d359bd19 100644 --- a/apps/dokploy/__test__/server/server-components.test.ts +++ b/apps/dokploy/__test__/server/server-components.test.ts @@ -374,8 +374,9 @@ describe("buildComponentUpdateScript", () => { ); expect(wait).toContain( - 'if [ "$attempt" -ge "${DOKPLOY_DOCKER_VERIFY_TIMEOUT:-30}" ]; then', + 'local timeout="${DOKPLOY_DOCKER_VERIFY_TIMEOUT:-30}"', ); + expect(wait).toContain('if [ "$(date +%s)" -ge "$deadline" ]; then'); expect(wait).toContain("sleep 1"); expect(wait).not.toContain("sleep 2"); }); @@ -590,6 +591,8 @@ const runComponentScript = ( FAKE_OLD_IMAGE: "traefik:v3.6.25", TMPDIR: dir, TRAEFIK_SETTLE_SECONDS: "0", + // The fake sleep returns at once, so a failed wait spins until this deadline. + DOKPLOY_DOCKER_VERIFY_TIMEOUT: "2", DOKPLOY_BACKUP_DIR: backups, DOKPLOY_OS_RELEASE: osRelease, DOKPLOY_APT_SOURCES_DIR: sources, diff --git a/packages/server/src/setup/host-docker-upgrade.ts b/packages/server/src/setup/host-docker-upgrade.ts index b4edace60f4..7cd4c0b18b9 100644 --- a/packages/server/src/setup/host-docker-upgrade.ts +++ b/packages/server/src/setup/host-docker-upgrade.ts @@ -65,26 +65,33 @@ const filesIn = (dir: string) => ({ const shellQuote = (value: string) => `'${value.replaceAll("'", "'\\''")}'`; const HOST_READY_CHECK = ` + DOCKER_NOT_READY="this node is not an active swarm node" [ "$(docker info --format '{{.Swarm.LocalNodeState}}' 2>/dev/null)" = "active" ] || return 1 - local service="" container="" + local service="" container="" health="" for service in dokploy dokploy-postgres dokploy-redis; do + DOCKER_NOT_READY="$service has no running task" container="$(docker ps -q --no-trunc --filter "label=com.docker.swarm.service.name=$service" --filter status=running | head -n1)" [ -n "$container" ] || return 1 case "$service" in dokploy) - case "$(docker inspect --format '{{if .State.Health}}{{.State.Health.Status}}{{else}}none{{end}}' "$container" 2>/dev/null)" in + health="$(docker inspect --format '{{if .State.Health}}{{.State.Health.Status}}{{else}}none{{end}}' "$container" 2>/dev/null)" + DOCKER_NOT_READY="the panel container is \${health:-not inspectable}" + case "$health" in healthy | none) ;; *) return 1 ;; esac ;; dokploy-postgres) + DOCKER_NOT_READY="Postgres does not accept connections" docker exec "$container" pg_isready -U dokploy -d dokploy >/dev/null 2>&1 || return 1 ;; dokploy-redis) + DOCKER_NOT_READY="Redis does not answer PING" [ "$(docker exec "$container" redis-cli ping 2>/dev/null)" = "PONG" ] || return 1 ;; esac - done`; + done + DOCKER_NOT_READY=""`; export const buildHostDockerUpgradeScript = (dir = HOST_UPDATES_DIR) => { const files = filesIn(dir); diff --git a/packages/server/src/setup/server-components.ts b/packages/server/src/setup/server-components.ts index 02436b4defa..fefa188d5fe 100644 --- a/packages/server/src/setup/server-components.ts +++ b/packages/server/src/setup/server-components.ts @@ -329,20 +329,31 @@ docker_host_ready() { } docker_ready_on() { + DOCKER_NOT_READY="Docker Engine does not report $1 yet" [ "$($SUDO_CMD docker version --format '{{.Server.Version}}' 2>/dev/null)" = "$1" ] || return 1 if [ "$2" = 1 ]; then + DOCKER_NOT_READY="dokploy-traefik is not running" [ "$($SUDO_CMD docker inspect -f '{{.State.Running}}' dokploy-traefik 2>/dev/null)" = "true" ] || return 1 fi + DOCKER_NOT_READY="" docker_host_ready } +# A deadline, not a count of attempts: on a loaded host one round of checks +# takes seconds, so counted attempts stretched the wait far past its limit. docker_wait_ready() { - local attempt=0 + local timeout="\${DOKPLOY_DOCKER_VERIFY_TIMEOUT:-30}" + local deadline=$(($(date +%s) + timeout)) + local next_report=$(($(date +%s) + 30)) until docker_ready_on "$1" "$2"; do - attempt=$((attempt + 1)) - if [ "$attempt" -ge "\${DOKPLOY_DOCKER_VERIFY_TIMEOUT:-30}" ]; then + if [ "$(date +%s)" -ge "$deadline" ]; then + echo "Not ready on Docker $1 after $timeout seconds: \${DOCKER_NOT_READY:-the services are not back}" >&2 return 1 fi + if [ "$(date +%s)" -ge "$next_report" ]; then + echo "Waiting for the services on Docker $1: \${DOCKER_NOT_READY:-the services are not back}" + next_report=$(($(date +%s) + 30)) + fi sleep 1 done }