diff --git a/apps/dokploy/__test__/settings/dokploy-image-cleanup.test.ts b/apps/dokploy/__test__/settings/dokploy-image-cleanup.test.ts index 13e2aacdbe9..9529f0e4a3e 100644 --- a/apps/dokploy/__test__/settings/dokploy-image-cleanup.test.ts +++ b/apps/dokploy/__test__/settings/dokploy-image-cleanup.test.ts @@ -25,6 +25,7 @@ const { cleanupOldDokployImages, getAgentHitsReloadCommand, getAgentHitsUpdateCommand, + getAgentHitsUpdateData, getDokployImageCleanupCommand, getDokployImageKeepCount, getDokployImageRepositories, @@ -346,7 +347,7 @@ describe("Dokploy update commands", () => { expectPullBeforeUpdate(command, "docker pull dokploy/dokploy\\:v0.30.7"); expect(command).toContain( - "--image dokploy/dokploy\\:v0.30.7 --env-add DOKPLOY_KEEP_OLD_IMAGES\\=3 ", + '--image "$pinned_image" --env-add DOKPLOY_KEEP_OLD_IMAGES\\=3 ', ); expect(getOfficialUpdateCommand("v0.30.7")).not.toContain( "DOKPLOY_KEEP_OLD_IMAGES", @@ -467,3 +468,200 @@ esac } }); }); + +describe("Dokploy update command", () => { + const AGENTHITS_IMAGE = "ghcr.io/agenthits/dokploy:agenthits-dev"; + const PINNED = `sha256:${"c".repeat(64)}`; + const FORK_VERSION = "agenthits-v1.0.9-3-g0123456789ab"; + + beforeEach(() => { + vi.stubEnv("DOKPLOY_AGENTHITS_UPDATE_IMAGE", undefined); + vi.stubEnv("DOKPLOY_AGENTHITS_UPDATE_TAG", undefined); + }); + + afterEach(() => { + vi.unstubAllEnvs(); + vi.unstubAllGlobals(); + }); + + // The fake image lists a digest of another repository first, as a host that + // also pulled the build from a mirror would. + const runUpdate = ( + command: string, + repository: string, + env: Record = {}, + ) => { + const dir = mkdtempSync(join(tmpdir(), "dokploy-update-")); + const log = join(dir, "log"); + try { + writeFileSync( + join(dir, "docker"), + `#!/bin/sh +echo "$*" >> "${log}" +case "$*" in + "pull "*) [ -z "$PULL_FAILS" ] || exit 1 ;; + "ps -q"*) echo task1 ;; + "inspect --format {{.Image}} task1") echo ${PREVIOUS} ;; + "image inspect "*) + [ -z "$NO_IMAGE" ] || exit 1 + echo "mirror.example/dokploy@sha256:${"d".repeat(64)}" + [ -n "$NO_DIGEST" ] || echo "${repository}@${PINNED}" + echo sha256:new ;; +esac +`, + ); + chmodSync(join(dir, "docker"), 0o755); + writeFileSync(log, ""); + let status = 0; + let stdout = ""; + try { + stdout = execFileSync("sh", ["-c", command], { + env: { + ...process.env, + ...env, + PATH: `${dir}:${process.env.PATH}`, + }, + encoding: "utf8", + stdio: "pipe", + }); + } catch (error) { + status = (error as { status: number }).status; + stdout = String((error as { stdout?: string }).stdout ?? ""); + } + const calls = readFileSync(log, "utf8").trim().split("\n"); + return { + status, + stdout, + calls, + update: calls.find((call) => call.startsWith("service update")), + }; + } finally { + rmSync(dir, { recursive: true, force: true }); + } + }; + + const runAgentHitsUpdate = (env?: Record) => + runUpdate( + getAgentHitsUpdateCommand("v0.30.6", FORK_VERSION, "v0.30.7", 2), + "ghcr.io/agenthits/dokploy", + env, + ); + + it( + "pins the AgentHits build it pulled by digest", + () => { + const { status, stdout, calls, update } = runAgentHitsUpdate(); + expect(status).toBe(0); + expect(calls.filter((call) => call.startsWith("pull"))).toEqual([ + `pull ${AGENTHITS_IMAGE}`, + ]); + expect(stdout).toContain(UPDATE_IMAGE_PULLED_MARKER); + expect(update).toBe( + `service update --force --no-resolve-image --image ${AGENTHITS_IMAGE}@${PINNED} --env-add RELEASE_TAG=agenthits-dev --env-add DOKPLOY_OFFICIAL_VERSION=v0.30.7 --env-add DOKPLOY_FORK_VERSION=${FORK_VERSION} --env-add DOKPLOY_KEEP_OLD_IMAGES=2 --env-add DOKPLOY_PREVIOUS_IMAGE=${PREVIOUS} --update-order stop-first --update-failure-action rollback dokploy`, + ); + expect(calls.at(-1)).toBe(update); + }, + SPAWN_TEST_TIMEOUT_MS, + ); + + it( + "pins a moving official tag by digest", + () => { + const { status, update } = runUpdate( + getOfficialUpdateCommand("canary"), + "dokploy/dokploy", + ); + expect(status).toBe(0); + expect(update).toBe( + `service update --force --no-resolve-image --image dokploy/dokploy:canary@${PINNED} --env-add DOKPLOY_PREVIOUS_IMAGE=${PREVIOUS} --update-order stop-first --update-failure-action rollback dokploy`, + ); + }, + SPAWN_TEST_TIMEOUT_MS, + ); + + it( + "falls back to the image id when the pull left no digest for the repository", + () => { + const { status, update } = runAgentHitsUpdate({ NO_DIGEST: "1" }); + expect(status).toBe(0); + expect(update).toContain("--no-resolve-image --image sha256:new "); + }, + SPAWN_TEST_TIMEOUT_MS, + ); + + it( + "leaves the service alone when the pull fails", + () => { + const { status, stdout, calls } = runAgentHitsUpdate({ + PULL_FAILS: "1", + }); + expect(status).toBe(1); + expect(stdout).not.toContain(UPDATE_IMAGE_PULLED_MARKER); + expect(calls).toEqual([`pull ${AGENTHITS_IMAGE}`]); + }, + SPAWN_TEST_TIMEOUT_MS, + ); + + it( + "does not fall back to the moving tag when the pulled image is missing", + () => { + const { status, update } = runAgentHitsUpdate({ NO_IMAGE: "1" }); + expect(status).toBe(1); + expect(update).toBeUndefined(); + }, + SPAWN_TEST_TIMEOUT_MS, + ); + + it( + "does not offer the pinned build as an update again", + async () => { + vi.stubEnv("DOKPLOY_FORK_VERSION", FORK_VERSION); + vi.stubEnv("DOKPLOY_OFFICIAL_VERSION", "v0.30.7"); + const pinnedImage = + runAgentHitsUpdate().update?.match(/--image (\S+)/)?.[1]; + expect(pinnedImage).toBe(`${AGENTHITS_IMAGE}@${PINNED}`); + + const checkAgainst = (latestDigest: string) => { + execAsync.mockReset(); + execAsync.mockResolvedValue({ stdout: `${pinnedImage}\n`, stderr: "" }); + vi.stubGlobal( + "fetch", + vi.fn(async (input: RequestInfo | URL) => { + const url = input.toString(); + if (url.includes("/token")) { + return Response.json({ token: "token" }); + } + if (url.endsWith("/manifests/agenthits-dev")) { + return Response.json( + { config: { digest: "sha256:config" } }, + { headers: { "docker-content-digest": latestDigest } }, + ); + } + if (url.endsWith("/blobs/sha256:config")) { + return Response.json({ + config: { + Env: [ + "DOKPLOY_OFFICIAL_VERSION=v0.30.7", + `DOKPLOY_FORK_VERSION=${FORK_VERSION}`, + ], + }, + }); + } + throw new Error(`Unexpected URL: ${url}`); + }), + ); + return getAgentHitsUpdateData("v0.30.7"); + }; + + expect(await checkAgainst(PINNED)).toMatchObject({ + updateAvailable: false, + currentDigest: PINNED, + }); + expect(await checkAgainst(`sha256:${"e".repeat(64)}`)).toMatchObject({ + updateAvailable: true, + currentDigest: PINNED, + }); + }, + SPAWN_TEST_TIMEOUT_MS, + ); +}); diff --git a/apps/dokploy/__test__/settings/fork-version.test.ts b/apps/dokploy/__test__/settings/fork-version.test.ts index 1c38624b14d..e1091535100 100644 --- a/apps/dokploy/__test__/settings/fork-version.test.ts +++ b/apps/dokploy/__test__/settings/fork-version.test.ts @@ -490,8 +490,9 @@ describe("AgentHits fork version metadata", () => { ); expect(command).toContain( - "--image ghcr.io/agenthits/dokploy\\:agenthits-dev", + "docker pull ghcr.io/agenthits/dokploy\\:agenthits-dev", ); + expect(command).toContain('--image "$pinned_image"'); expect(command).toContain("--env-add RELEASE_TAG\\=agenthits-dev"); expect(command).toContain("--env-add DOKPLOY_OFFICIAL_VERSION\\=v0.30.0"); expect(command).toContain( diff --git a/packages/server/src/services/settings.ts b/packages/server/src/services/settings.ts index f49a486a4b4..201fe811b50 100644 --- a/packages/server/src/services/settings.ts +++ b/packages/server/src/services/settings.ts @@ -626,12 +626,33 @@ export const getUpdateDiskSpace = async (): Promise => { }; }; +// Swarm pulls a tag again whenever it starts a task, so a service that names a +// moving tag such as `agenthits-dev` or `canary` can come up on a newer build +// after a crash, a node reboot or a Docker restart. The update names the build +// it just pulled instead: by digest, which swarm does not pull while the image +// is present, or by image id when the pull left no digest for this repository. +const getPinPulledImageCommand = (image: string) => { + const reference = image.split("@")[0] ?? image; + return `pinned_image="" +for pulled_ref in $(docker image inspect --format '{{range .RepoDigests}}{{println .}}{{end}}{{.Id}}' ${quoteShellArg(image)} 2>/dev/null); do + case "$pulled_ref" in + ${quoteShellArg(getImageRepository(image))}@sha256:*) pinned_image=${quoteShellArg(reference)}"@\${pulled_ref#*@}"; break ;; + sha256:*) pinned_image="$pulled_ref" ;; + esac +done +if [ -z "$pinned_image" ]; then + echo "The pulled image could not be found, so the panel was not updated." >&2 + exit 1 +fi`; +}; + export const getAgentHitsUpdateCommand = ( currentVersion: string, forkVersion?: string | null, officialVersion?: string | null, keepImages?: number | null, ) => { + const image = getAgentHitsUpdateImage(); const forkVersionArg = forkVersion?.trim() ? `--env-add ${quoteShellArg(`DOKPLOY_FORK_VERSION=${forkVersion.trim()}`)}` : "$fork_version_env_arg"; @@ -645,15 +666,16 @@ export const getAgentHitsUpdateCommand = ( // stop-first, so a pull inside it would happen with Dokploy down. Stop-first // also keeps two panel versions from running migrations at the same time. return ` -docker pull ${quoteShellArg(getAgentHitsUpdateImage())} || exit 1 +docker pull ${quoteShellArg(image)} || exit 1 echo ${quoteShellArg(UPDATE_IMAGE_PULLED_MARKER)} ${RECORD_PREVIOUS_IMAGE} +${getPinPulledImageCommand(image)} fork_version_env_arg="" if docker service inspect dokploy --format '{{range .Spec.TaskTemplate.ContainerSpec.Env}}{{println .}}{{end}}' 2>/dev/null | grep -q '^DOKPLOY_FORK_VERSION='; then fork_version_env_arg="--env-rm DOKPLOY_FORK_VERSION" fi -docker service update --force \\ - --image ${quoteShellArg(getAgentHitsUpdateImage())} \\ +docker service update --force --no-resolve-image \\ + --image "$pinned_image" \\ --env-add ${quoteShellArg(`RELEASE_TAG=${getAgentHitsUpdateTag()}`)} \\ --env-add ${quoteShellArg(`DOKPLOY_OFFICIAL_VERSION=${officialVersionArg}`)} \\ ${forkVersionArg} \\ @@ -665,11 +687,10 @@ docker service update --force \\ `; }; -// The service names a moving tag such as `agenthits-dev`, and swarm pulls a -// tag again whenever it starts a task, so a plain forced restart can come up on -// a newer build. Reload pins the image the panel runs now: by digest, which -// swarm does not pull when the image is present, or by image id for a build -// that never came from a registry. +// A service last updated by an older panel still names a moving tag, so a plain +// forced restart can come up on a newer build (see getPinPulledImageCommand). +// Reload pins the image the panel runs now: by digest, or by image id for a +// build that never came from a registry. export const getAgentHitsReloadCommand = () => ` running_tasks=$(${DOKPLOY_RUNNING_TASKS}) running_image="" @@ -695,15 +716,16 @@ export const getOfficialUpdateCommand = ( version: string, keepImages?: number | null, ) => { - const image = quoteShellArg(getOfficialUpdateImage(version)); + const image = getOfficialUpdateImage(version); const keepImagesArg = keepImages === undefined ? "" : getDokployKeepImagesEnvArg(keepImages); return ` -docker pull ${image} || exit 1 +docker pull ${quoteShellArg(image)} || exit 1 echo ${quoteShellArg(UPDATE_IMAGE_PULLED_MARKER)} ${RECORD_PREVIOUS_IMAGE} -docker service update --force --image ${image} ${keepImagesArg} ${PREVIOUS_IMAGE_ENV_ARG} --update-order stop-first --update-failure-action rollback dokploy +${getPinPulledImageCommand(image)} +docker service update --force --no-resolve-image --image "$pinned_image" ${keepImagesArg} ${PREVIOUS_IMAGE_ENV_ARG} --update-order stop-first --update-failure-action rollback dokploy `; };