From 344a57fff036d0ec2898f21d821d6dcd17efc758 Mon Sep 17 00:00:00 2001 From: Grant Emsley Date: Sat, 3 Oct 2026 14:33:02 -0400 Subject: [PATCH 001/109] feat(git): add a guard that keeps git stacks from writing to RAM or the flash On Unraid, /mnt itself is RAM. A path that is not on a mounted disk, pool or share (a typo such as /mnt/usr/appdata, or /mnt/user while the array is stopped) silently lands in RAM, and a stray file left in /mnt/user can stop the user shares from mounting at the next array start. GitPathGuard checks, immediately before any write, that the array is started, that the path is on a live mount below /mnt, and that no symlink leads out of it. createDirectory() creates missing levels one at a time and never creates a mount point or a share. - A tmpfs, ramfs or rootfs mount under /mnt does not count as a disk: Unassigned Devices can put a tmpfs over /mnt/disks, and a folder left there for an unplugged disk must not pass. - An exclusive share (Unraid 6.12+), where /mnt/user/ is a symlink to /mnt//, is followed to the pool when the pool is mounted. Any other symlink at the share level is refused. Nothing uses it yet; the git stack code in the following commits does. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_011iUWDG8wTp8MhsFDqNC4NL --- source/compose.manager/include/Defines.php | 6 + .../compose.manager/include/GitPathGuard.php | 341 +++++++++++++++ tests/bootstrap.php | 4 + tests/unit/GitPathGuardTest.php | 401 ++++++++++++++++++ 4 files changed, 752 insertions(+) create mode 100644 source/compose.manager/include/GitPathGuard.php create mode 100644 tests/unit/GitPathGuardTest.php diff --git a/source/compose.manager/include/Defines.php b/source/compose.manager/include/Defines.php index 9870a29c..4bb80f18 100644 --- a/source/compose.manager/include/Defines.php +++ b/source/compose.manager/include/Defines.php @@ -36,6 +36,12 @@ function locate_compose_root($name) defined('COMPOSE_CREDENTIAL_KEY_FILE') || define('COMPOSE_CREDENTIAL_KEY_FILE', '/boot/config/plugins/compose.manager/credentials.key'); defined('COMPOSE_DOCKER_CONFIG_DIR') || define('COMPOSE_DOCKER_CONFIG_DIR', '/var/tmp/compose-manager-docker-config'); defined('COMPOSE_GITHUB_DEVICE_DIR') || define('COMPOSE_GITHUB_DEVICE_DIR', '/var/tmp/compose-manager-github-device'); +// Git-backed stacks: where Unraid reports the array state and the live mounts, +// and the folder under which disks, pools and shares are mounted. +defined('COMPOSE_UNRAID_VAR_INI') || define('COMPOSE_UNRAID_VAR_INI', '/var/local/emhttp/var.ini'); +defined('COMPOSE_MOUNTS_FILE') || define('COMPOSE_MOUNTS_FILE', '/proc/self/mounts'); +defined('COMPOSE_GIT_MNT_DIR') || define('COMPOSE_GIT_MNT_DIR', '/mnt'); +defined('COMPOSE_GIT_DEFAULT_CLONES_ROOT') || define('COMPOSE_GIT_DEFAULT_CLONES_ROOT', '/mnt/user/appdata/compose.manager/git'); /** * Reserved filename at the compose root level used by the plugin installer diff --git a/source/compose.manager/include/GitPathGuard.php b/source/compose.manager/include/GitPathGuard.php new file mode 100644 index 00000000..b4556860 --- /dev/null +++ b/source/compose.manager/include/GitPathGuard.php @@ -0,0 +1,341 @@ + chr((int) octdec($match[1])), + $fields[1] + ); + } + return $mountPoints; + } + + /** + * The mount under /mnt that holds a path, or null when the path is not on one. + * + * Picks the longest listed mount point that contains the path. /mnt itself + * and / never count: on Unraid both are RAM. + */ + public static function findMountFor(string $path): ?string + { + $mntDir = rtrim(COMPOSE_GIT_MNT_DIR, '/'); + $best = null; + foreach (self::listMountPoints() as $mountPoint) { + $mountPoint = rtrim($mountPoint, '/'); + if (!str_starts_with($mountPoint, $mntDir . '/')) { + continue; + } + if ($path !== $mountPoint && !str_starts_with($path, $mountPoint . '/')) { + continue; + } + if ($best === null || strlen($mountPoint) > strlen($best)) { + $best = $mountPoint; + } + } + return $best; + } + + /** + * Check the shape of an absolute path without touching the filesystem. + * + * @throws InvalidArgumentException naming the problem + */ + public static function assertCleanAbsolutePath(string $path, string $what): void + { + if ($path === '') { + throw new InvalidArgumentException("$what is empty."); + } + if (preg_match('/[\x00-\x1f\x7f]/', $path) === 1) { + throw new InvalidArgumentException("$what contains a control character."); + } + if (!str_starts_with($path, '/')) { + throw new InvalidArgumentException("$what must be an absolute path: $path"); + } + if (str_contains($path, '\\')) { + throw new InvalidArgumentException("$what must not contain a backslash: $path"); + } + foreach (explode('/', substr($path, 1)) as $part) { + if ($part === '' || $part === '.' || $part === '..') { + throw new InvalidArgumentException("$what must not contain empty, '.' or '..' parts: $path"); + } + } + } + + /** + * Check that a path is a usable place to keep git clones. + * + * The path must sit on a mount below /mnt (a user share, a disk, a pool or + * an Unassigned Devices mount), and its share folder (see shareFolder()) + * must already exist. Deeper directories may be created later. This keeps + * a typo from creating a new share, or a directory in RAM. + * + * Does not check the array state; assertSafeToWrite() does that at the + * moment of writing. + * + * @throws InvalidArgumentException naming the problem + */ + public static function assertValidClonesRoot(string $path): void + { + self::assertCleanAbsolutePath($path, 'The git clones folder'); + $path = self::followExclusiveShare($path); + + $mount = self::findMountFor($path); + if ($mount === null) { + throw new InvalidArgumentException( + "The git clones folder $path is not on a disk, pool or share under " + . COMPOSE_GIT_MNT_DIR . ". Anything written there would be kept in RAM and lost at reboot. " + . "Check the path for a typo, and that the array is started." + ); + } + $share = self::shareFolder($path); + if ($share === null) { + throw new InvalidArgumentException( + "The git clones folder must be a folder inside $mount, not $mount itself." + ); + } + if (self::isSymlink($share)) { + throw new InvalidArgumentException("$share is a symlink, so it was not used. Check the path for a typo."); + } + if (!is_dir($share)) { + throw new InvalidArgumentException( + "$share does not exist. Create it first (for a user share, add the share in Unraid), " + . "or check the path for a typo." + ); + } + + self::assertExistingPartStaysInside($path, $mount); + } + + /** + * The share folder of a path below /mnt: its first two folders, such as + * /mnt/user/appdata, /mnt/cache/appdata or /mnt/disk1/appdata. Null for a + * path that is not at least that deep. + * + * This is measured from /mnt, not from the mount that holds the path: on a + * ZFS pool every share is a dataset with its own mount, so the mount for + * /mnt/cache/appdata/compose.manager is /mnt/cache/appdata itself. Call + * after followExclusiveShare(). + */ + public static function shareFolder(string $path): ?string + { + $mntDir = rtrim(COMPOSE_GIT_MNT_DIR, '/'); + if (!str_starts_with($path, $mntDir . '/')) { + return null; + } + $parts = explode('/', substr($path, strlen($mntDir) + 1)); + if (count($parts) < 2) { + return null; + } + return $mntDir . '/' . $parts[0] . '/' . $parts[1]; + } + + /** + * Throw unless it is safe to create or change something at this path right now. + * + * Checks, in order: the array is started, the path is on a live mount below + * /mnt, and the part of the path that exists does not lead out of that mount + * through a symlink. Call this immediately before each write, not once at + * the start of a long operation: the array can stop in between. + * + * @throws RuntimeException naming the problem + */ + public static function assertSafeToWrite(string $path): void + { + try { + self::assertCleanAbsolutePath($path, 'The path'); + } catch (InvalidArgumentException $error) { + throw new RuntimeException($error->getMessage(), 0, $error); + } + + if (!self::isArrayStarted()) { + throw new RuntimeException("The array is not started, so nothing was written to $path."); + } + + $path = self::followExclusiveShare($path); + $mount = self::findMountFor($path); + if ($mount === null) { + throw new RuntimeException( + "$path is not on a mounted disk, pool or share, so nothing was written there." + ); + } + + try { + self::assertExistingPartStaysInside($path, $mount); + } catch (InvalidArgumentException $error) { + throw new RuntimeException($error->getMessage(), 0, $error); + } + } + + /** + * Create a directory and any missing parents, checking each level first. + * + * Unlike mkdir($path, 0755, true), this never creates the share folder or + * anything above it: those must exist (see assertValidClonesRoot). + * + * @throws RuntimeException if a level cannot be created safely + */ + public static function createDirectory(string $path, int $mode = 0755): void + { + self::assertSafeToWrite($path); + $path = self::followExclusiveShare($path); + + $current = self::shareFolder($path); + if ($current === null || !is_dir($current) || self::isSymlink($current)) { + throw new RuntimeException(($current ?? $path) . " does not exist, so $path was not created."); + } + + $levelsToCreate = []; + if ($path !== $current) { + $levelsToCreate = explode('/', substr($path, strlen($current) + 1)); + } + foreach ($levelsToCreate as $part) { + $current .= '/' . $part; + if (self::isSymlink($current)) { + throw new RuntimeException("$current is a symlink, so $path was not created."); + } + if (is_dir($current)) { + continue; + } + if (file_exists($current)) { + throw new RuntimeException("$current exists and is not a folder, so $path was not created."); + } + self::assertSafeToWrite($current); + if (!@mkdir($current, $mode) && !is_dir($current)) { + throw new RuntimeException("Could not create $current."); + } + } + } + + /** + * The same path on the pool, when its share is an exclusive share. + * + * Since Unraid 6.12, a share that lives on one pool only can be "exclusive": + * /mnt/user/ is then a symlink to /mnt//, made by + * Unraid. Only that exact shape is followed, and only to a live mount: any + * other symlink at the share level is left alone, so the checks still + * refuse it. + */ + public static function followExclusiveShare(string $path): string + { + $userDir = rtrim(COMPOSE_GIT_MNT_DIR, '/') . '/user'; + if (!str_starts_with($path, $userDir . '/')) { + return $path; + } + $parts = explode('/', substr($path, strlen($userDir) + 1), 2); + $share = $parts[0]; + $target = @readlink($userDir . '/' . $share); + if ($target === false) { + return $path; + } + + $pattern = '#^' . preg_quote(rtrim(COMPOSE_GIT_MNT_DIR, '/'), '#') . '/([^/]+)/' . preg_quote($share, '#') . '$#'; + if (preg_match($pattern, $target, $match) !== 1 || $match[1] === 'user' || $match[1] === 'user0') { + return $path; + } + $poolMount = rtrim(COMPOSE_GIT_MNT_DIR, '/') . '/' . $match[1]; + if (self::findMountFor($poolMount) !== $poolMount) { + return $path; + } + return $target . (isset($parts[1]) ? '/' . $parts[1] : ''); + } + + /** + * Throw if the deepest existing part of a path resolves outside its mount. + * + * @throws InvalidArgumentException + */ + private static function assertExistingPartStaysInside(string $path, string $mount): void + { + $existing = $path; + while (!file_exists($existing) && !self::isSymlink($existing)) { + $existing = dirname($existing); + } + + $resolved = realpath($existing); + $resolvedMount = realpath($mount); + if ($resolved === false || $resolvedMount === false) { + throw new InvalidArgumentException("Could not resolve $existing."); + } + if ($resolved !== $resolvedMount && !str_starts_with($resolved, $resolvedMount . '/')) { + throw new InvalidArgumentException( + "$existing leads outside $mount (it resolves to $resolved), so it was not used." + ); + } + } + + /** + * Whether a path is a symlink (dangling or not). + * + * Uses readlink() rather than is_link(): PHP caches is_link() answers + * during a run, and a stream wrapper registered for file:// (the test + * framework has one) can change them. readlink() is never cached and + * always asks the filesystem. + */ + private static function isSymlink(string $path): bool + { + return @readlink($path) !== false; + } +} diff --git a/tests/bootstrap.php b/tests/bootstrap.php index d1d6b9b7..30d60a2c 100644 --- a/tests/bootstrap.php +++ b/tests/bootstrap.php @@ -47,6 +47,10 @@ function composeLogger($message, $data = null, $type = 'daemon', $level = 'info' define('COMPOSE_CREDENTIAL_KEY_FILE', sys_get_temp_dir() . '/compose_manager_credentials.key'); define('COMPOSE_DOCKER_CONFIG_DIR', sys_get_temp_dir() . '/compose_manager_docker_configs'); define('COMPOSE_GITHUB_DEVICE_DIR', sys_get_temp_dir() . '/compose_manager_github_device'); +// Git-backed stack tests fake Unraid's /mnt, array state and mount table in temp files. +define('COMPOSE_GIT_MNT_DIR', sys_get_temp_dir() . '/compose_git_mnt'); +define('COMPOSE_UNRAID_VAR_INI', sys_get_temp_dir() . '/compose_git_var.ini'); +define('COMPOSE_MOUNTS_FILE', sys_get_temp_dir() . '/compose_git_mounts'); // Point to the dev-env resvg binary when present; plugin path used on real Unraid define('COMPOSE_RESVG_BIN', is_executable('/tmp/resvg') ? '/tmp/resvg' : '/usr/local/emhttp/plugins/compose.manager/bin/resvg'); diff --git a/tests/unit/GitPathGuardTest.php b/tests/unit/GitPathGuardTest.php new file mode 100644 index 00000000..5cdd6f93 --- /dev/null +++ b/tests/unit/GitPathGuardTest.php @@ -0,0 +1,401 @@ +mnt = COMPOSE_GIT_MNT_DIR; + $this->outside = sys_get_temp_dir() . '/compose_git_outside'; + $this->removeTree($this->mnt); + $this->removeTree($this->outside); + mkdir($this->outside); + mkdir($this->mnt . '/user/appdata', 0755, true); + mkdir($this->mnt . '/disk1/appdata', 0755, true); + mkdir($this->mnt . '/disks/usb stick/stuff', 0755, true); + $this->setArrayState('STARTED'); + $this->setMounts(['user', 'disk1']); + } + + protected function tearDown(): void + { + $this->removeTree($this->mnt); + $this->removeTree($this->outside); + @unlink(COMPOSE_UNRAID_VAR_INI); + @unlink(COMPOSE_MOUNTS_FILE); + parent::tearDown(); + } + + // ----- array state ----- + + public function testArrayStartedIsReadFromVarIni(): void + { + $this->assertTrue(GitPathGuard::isArrayStarted()); + } + + public function testArrayStoppedOrStartingCountsAsNotStarted(): void + { + foreach (['STOPPED', 'STARTING', 'STOPPING', ''] as $state) { + $this->setArrayState($state); + $this->assertFalse(GitPathGuard::isArrayStarted(), "mdState=$state"); + } + } + + public function testMissingVarIniCountsAsNotStarted(): void + { + unlink(COMPOSE_UNRAID_VAR_INI); + $this->assertFalse(GitPathGuard::isArrayStarted()); + } + + // ----- mount table ----- + + public function testMountPointsWithEscapedSpacesAreDecoded(): void + { + file_put_contents(COMPOSE_MOUNTS_FILE, "/dev/sdx1 {$this->mnt}/disks/usb\\040stick xfs rw 0 0\n"); + $this->assertSame([$this->mnt . '/disks/usb stick'], GitPathGuard::listMountPoints()); + } + + public function testFindMountPicksTheLongestMatchingMount(): void + { + file_put_contents( + COMPOSE_MOUNTS_FILE, + "rootfs / rootfs rw 0 0\n" + . "rootfs {$this->mnt} rootfs rw 0 0\n" + . "shfs {$this->mnt}/user fuse.shfs rw 0 0\n" + . "/dev/sdx1 {$this->mnt}/user/appdata/inner xfs rw 0 0\n" + ); + $this->assertSame($this->mnt . '/user/appdata/inner', GitPathGuard::findMountFor($this->mnt . '/user/appdata/inner/x')); + $this->assertSame($this->mnt . '/user', GitPathGuard::findMountFor($this->mnt . '/user/appdata/x')); + } + + public function testMntItselfAndRootNeverCountAsAMount(): void + { + file_put_contents(COMPOSE_MOUNTS_FILE, "rootfs / rootfs rw 0 0\nrootfs {$this->mnt} rootfs rw 0 0\n"); + $this->assertNull(GitPathGuard::findMountFor($this->mnt . '/user/appdata/git')); + } + + public function testMountNameThatOnlySharesAPrefixDoesNotMatch(): void + { + // /mnt/user0 must not be taken as being on /mnt/user. + $this->assertNull(GitPathGuard::findMountFor($this->mnt . '/user0/appdata')); + } + + // ----- clones root ----- + + public function testUsualClonesRootsAreAccepted(): void + { + GitPathGuard::assertValidClonesRoot($this->mnt . '/user/appdata/compose.manager/git'); + GitPathGuard::assertValidClonesRoot($this->mnt . '/disk1/appdata/git'); + GitPathGuard::assertValidClonesRoot($this->mnt . '/user/appdata'); + $this->addToAssertionCount(3); + } + + public function testUnassignedDevicesMountIsAccepted(): void + { + $this->setMounts(['user', 'disks/usb stick']); + GitPathGuard::assertValidClonesRoot($this->mnt . '/disks/usb stick/stuff/git'); + $this->addToAssertionCount(1); + } + + /** @return array */ + public static function badRootShapes(): array + { + return [ + 'empty' => [''], + 'relative' => ['user/appdata/git'], + 'dot dot' => ['/x/../etc'], + 'dot' => ['/x/./y'], + 'double slash' => ['/x//y'], + 'trailing slash' => ['/x/y/'], + 'newline' => ["/x/y\nz"], + 'nul' => ["/x/y\0z"], + 'backslash' => ['/x\\y'], + ]; + } + + #[\PHPUnit\Framework\Attributes\DataProvider('badRootShapes')] + public function testMalformedClonesRootIsRefused(string $root): void + { + $this->expectException(InvalidArgumentException::class); + GitPathGuard::assertValidClonesRoot($root); + } + + public function testRootFilesystemAndFlashAreRefused(): void + { + foreach (['/', '/boot/config/plugins/compose.manager/git', '/tmp/git', '/root/git'] as $root) { + try { + GitPathGuard::assertValidClonesRoot($root); + $this->fail("Accepted $root"); + } catch (InvalidArgumentException) { + $this->addToAssertionCount(1); + } + } + } + + public function testMntItselfAndAMountItselfAreRefused(): void + { + foreach ([$this->mnt, $this->mnt . '/user', $this->mnt . '/disk1'] as $root) { + try { + GitPathGuard::assertValidClonesRoot($root); + $this->fail("Accepted $root"); + } catch (InvalidArgumentException) { + $this->addToAssertionCount(1); + } + } + } + + public function testTypoInMountNameIsRefusedAsRam(): void + { + // /mnt/usr is not a mount: on Unraid this would be written to RAM. + mkdir($this->mnt . '/usr/appdata', 0755, true); + $this->expectException(InvalidArgumentException::class); + $this->expectExceptionMessage('RAM'); + GitPathGuard::assertValidClonesRoot($this->mnt . '/usr/appdata/git'); + } + + public function testUserShareNotMountedIsRefused(): void + { + // The Homelab 2026-09-22 state: /mnt/user exists as a plain directory, shfs is not mounted. + $this->setMounts(['disk1']); + $this->expectException(InvalidArgumentException::class); + GitPathGuard::assertValidClonesRoot($this->mnt . '/user/appdata/git'); + } + + public function testMissingShareIsRefusedRatherThanCreated(): void + { + // A typo in the share name would otherwise create a new user share. + $this->expectException(InvalidArgumentException::class); + $this->expectExceptionMessage('does not exist'); + GitPathGuard::assertValidClonesRoot($this->mnt . '/user/apdata/git'); + } + + public function testShareThatIsASymlinkIsRefused(): void + { + symlink($this->outside, $this->mnt . '/user/etc'); + $this->expectException(InvalidArgumentException::class); + GitPathGuard::assertValidClonesRoot($this->mnt . '/user/etc/git'); + } + + public function testExclusiveShareOnAPoolIsAcceptedAndWrittenOnThePool(): void + { + // Unraid 6.12+: an exclusive share is a symlink /mnt/user/ -> /mnt//. + mkdir($this->mnt . '/cache/media', 0755, true); + symlink($this->mnt . '/cache/media', $this->mnt . '/user/media'); + $this->setMounts(['user', 'disk1', 'cache']); + + GitPathGuard::assertValidClonesRoot($this->mnt . '/user/media/compose.manager/git'); + GitPathGuard::createDirectory($this->mnt . '/user/media/compose.manager/git'); + + $this->assertDirectoryExists($this->mnt . '/cache/media/compose.manager/git'); + } + + public function testShareThatIsAZfsDatasetIsAcceptedAndItsFoldersCreated(): void + { + // On a ZFS pool each share is a dataset, mounted on its own. + mkdir($this->mnt . '/cache/appdata', 0755, true); + $this->setMounts(['user', 'cache', 'cache/appdata']); + + GitPathGuard::assertValidClonesRoot($this->mnt . '/cache/appdata/compose.manager/git'); + GitPathGuard::createDirectory($this->mnt . '/cache/appdata/compose.manager/git'); + + $this->assertDirectoryExists($this->mnt . '/cache/appdata/compose.manager/git'); + } + + public function testExclusiveShareThatIsAZfsDatasetIsAccepted(): void + { + // The default clones root, with appdata exclusive to a ZFS cache pool. + $this->removeTree($this->mnt . '/user/appdata'); + mkdir($this->mnt . '/cache/appdata', 0755, true); + symlink($this->mnt . '/cache/appdata', $this->mnt . '/user/appdata'); + $this->setMounts(['user', 'cache', 'cache/appdata']); + + GitPathGuard::assertValidClonesRoot($this->mnt . '/user/appdata/compose.manager/git'); + GitPathGuard::createDirectory($this->mnt . '/user/appdata/compose.manager/git'); + + $this->assertDirectoryExists($this->mnt . '/cache/appdata/compose.manager/git'); + } + + public function testMissingShareOnAZfsPoolIsRefused(): void + { + $this->setMounts(['user', 'cache']); + mkdir($this->mnt . '/cache'); + $this->expectException(InvalidArgumentException::class); + $this->expectExceptionMessage('does not exist'); + GitPathGuard::assertValidClonesRoot($this->mnt . '/cache/apdata/git'); + } + + public function testSymlinkedShareIsNamedAsASymlink(): void + { + symlink($this->outside, $this->mnt . '/user/etc'); + $this->expectException(InvalidArgumentException::class); + $this->expectExceptionMessage('is a symlink'); + GitPathGuard::assertValidClonesRoot($this->mnt . '/user/etc/git'); + } + + public function testExclusiveShareWhosePoolIsNotMountedIsRefused(): void + { + mkdir($this->mnt . '/cache/media', 0755, true); + symlink($this->mnt . '/cache/media', $this->mnt . '/user/media'); + + $this->expectException(InvalidArgumentException::class); + GitPathGuard::assertValidClonesRoot($this->mnt . '/user/media/git'); + } + + public function testShareSymlinkToADifferentlyNamedFolderIsRefused(): void + { + mkdir($this->mnt . '/cache/other', 0755, true); + symlink($this->mnt . '/cache/other', $this->mnt . '/user/media'); + $this->setMounts(['user', 'disk1', 'cache']); + + $this->expectException(InvalidArgumentException::class); + GitPathGuard::assertValidClonesRoot($this->mnt . '/user/media/git'); + } + + public function testFolderLeftOnARamMountForAnUnpluggedDiskIsRefused(): void + { + // Unassigned Devices can put a tmpfs over /mnt/disks; the disk's own mount is gone. + file_put_contents( + COMPOSE_MOUNTS_FILE, + "rootfs {$this->mnt} rootfs rw 0 0\ntmpfs {$this->mnt}/disks tmpfs rw 0 0\nshfs {$this->mnt}/user fuse.shfs rw 0 0\n" + ); + $this->assertNull(GitPathGuard::findMountFor($this->mnt . '/disks/usb stick/stuff')); + + $this->expectException(InvalidArgumentException::class); + GitPathGuard::assertValidClonesRoot($this->mnt . '/disks/usb stick/git'); + } + + public function testSymlinkDeeperInThePathLeadingOutsideIsRefused(): void + { + symlink($this->outside, $this->mnt . '/user/appdata/escape'); + $this->expectException(InvalidArgumentException::class); + $this->expectExceptionMessage('leads outside'); + GitPathGuard::assertValidClonesRoot($this->mnt . '/user/appdata/escape/git'); + } + + public function testSymlinkThatStaysInsideTheMountIsAccepted(): void + { + mkdir($this->mnt . '/user/appdata/real'); + symlink($this->mnt . '/user/appdata/real', $this->mnt . '/user/appdata/link'); + GitPathGuard::assertValidClonesRoot($this->mnt . '/user/appdata/link/git'); + $this->addToAssertionCount(1); + } + + // ----- writing ----- + + public function testWriteIsRefusedWhileTheArrayIsStopped(): void + { + $this->setArrayState('STOPPED'); + $this->expectException(RuntimeException::class); + $this->expectExceptionMessage('array is not started'); + GitPathGuard::assertSafeToWrite($this->mnt . '/user/appdata/git'); + } + + public function testWriteIsRefusedWhenTheMountHasGone(): void + { + $this->setMounts(['disk1']); + $this->expectException(RuntimeException::class); + GitPathGuard::assertSafeToWrite($this->mnt . '/user/appdata/git'); + } + + public function testCreateDirectoryMakesOnlyTheMissingLevels(): void + { + $target = $this->mnt . '/user/appdata/compose.manager/git'; + GitPathGuard::createDirectory($target); + $this->assertDirectoryExists($target); + } + + public function testCreateDirectoryNeverCreatesTheShare(): void + { + try { + GitPathGuard::createDirectory($this->mnt . '/user/newshare/git'); + $this->fail('Created a new share'); + } catch (RuntimeException) { + $this->assertDirectoryDoesNotExist($this->mnt . '/user/newshare'); + } + } + + public function testCreateDirectoryCreatesNothingWhileTheArrayIsStopped(): void + { + $this->setArrayState('STOPPED'); + try { + GitPathGuard::createDirectory($this->mnt . '/user/appdata/compose.manager/git'); + $this->fail('Created a folder on a stopped array'); + } catch (RuntimeException) { + $this->assertDirectoryDoesNotExist($this->mnt . '/user/appdata/compose.manager'); + } + } + + public function testCreateDirectoryStopsAtAFileInTheWay(): void + { + file_put_contents($this->mnt . '/user/appdata/compose.manager', 'not a folder'); + $this->expectException(RuntimeException::class); + $this->expectExceptionMessage('not a folder'); + GitPathGuard::createDirectory($this->mnt . '/user/appdata/compose.manager/git'); + } + + public function testCreateDirectoryStopsAtASymlinkInTheWay(): void + { + mkdir($this->mnt . '/user/appdata/real'); + symlink($this->mnt . '/user/appdata/real', $this->mnt . '/user/appdata/link'); + $this->expectException(RuntimeException::class); + $this->expectExceptionMessage('symlink'); + GitPathGuard::createDirectory($this->mnt . '/user/appdata/link/git'); + } + + // ----- helpers ----- + + private function setArrayState(string $state): void + { + file_put_contents(COMPOSE_UNRAID_VAR_INI, "version=\"7.3.2\"\nmdState=\"$state\"\n"); + } + + /** @param string[] $names mounts below the fake /mnt */ + private function setMounts(array $names): void + { + $lines = ["rootfs / rootfs rw 0 0", "rootfs {$this->mnt} rootfs rw 0 0"]; + foreach ($names as $name) { + $escaped = str_replace(' ', '\\040', $this->mnt . '/' . $name); + $lines[] = "shfs $escaped fuse.shfs rw 0 0"; + } + file_put_contents(COMPOSE_MOUNTS_FILE, implode("\n", $lines) . "\n"); + } + + private function removeTree(string $path): void + { + // readlink(), not is_link(): the test stream wrapper hides symlinks from is_link(), + // and following a symlink here would delete whatever it points at. + if (@readlink($path) !== false || is_file($path)) { + unlink($path); + return; + } + if (!is_dir($path)) { + return; + } + foreach (scandir($path) ?: [] as $entry) { + if ($entry !== '.' && $entry !== '..') { + $this->removeTree($path . '/' . $entry); + } + } + rmdir($path); + } +} From 116433367b9ed02c1b0f9fcfa1383de695f899c6 Mon Sep 17 00:00:00 2001 From: Grant Emsley Date: Sat, 3 Oct 2026 14:33:03 -0400 Subject: [PATCH 002/109] feat(git): add a locked-down git runner for git stacks Every git call for git-backed stacks goes through GitCommand::run(). It ignores the system and global git config, never prompts, runs no repository hooks, allows only the https and local-path transports, starts from an empty environment so stray GIT_DIR or GIT_WORK_TREE values cannot redirect it, passes arguments without a shell, and is stopped by GNU timeout after a time limit. git refuses a repository owned by another user ("dubious ownership"). A clone can end up owned by nobody:users after Unraid's New Permissions tool, and a repository on this server is often owned by the container that keeps it. Each run trusts the folder it works in and any repository the caller names, and nothing else, through a global config file that holds only those safe.directory entries (git 2.47 ignores safe.directory given with -c when it reads from a local repository). ProcessRunner reports a command killed by a signal as 128 plus the signal number, and counts it as timed out only when the time limit was reached. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_011iUWDG8wTp8MhsFDqNC4NL --- source/compose.manager/include/Defines.php | 1 + source/compose.manager/include/GitCommand.php | 155 ++++++++++++ .../compose.manager/include/ProcessRunner.php | 128 ++++++++++ tests/bootstrap.php | 1 + tests/unit/GitCommandTest.php | 235 ++++++++++++++++++ 5 files changed, 520 insertions(+) create mode 100644 source/compose.manager/include/GitCommand.php create mode 100644 source/compose.manager/include/ProcessRunner.php create mode 100644 tests/unit/GitCommandTest.php diff --git a/source/compose.manager/include/Defines.php b/source/compose.manager/include/Defines.php index 4bb80f18..a9697c6f 100644 --- a/source/compose.manager/include/Defines.php +++ b/source/compose.manager/include/Defines.php @@ -41,6 +41,7 @@ function locate_compose_root($name) defined('COMPOSE_UNRAID_VAR_INI') || define('COMPOSE_UNRAID_VAR_INI', '/var/local/emhttp/var.ini'); defined('COMPOSE_MOUNTS_FILE') || define('COMPOSE_MOUNTS_FILE', '/proc/self/mounts'); defined('COMPOSE_GIT_MNT_DIR') || define('COMPOSE_GIT_MNT_DIR', '/mnt'); +defined('COMPOSE_GIT_HOME_DIR') || define('COMPOSE_GIT_HOME_DIR', '/var/tmp/compose-manager-git-home'); defined('COMPOSE_GIT_DEFAULT_CLONES_ROOT') || define('COMPOSE_GIT_DEFAULT_CLONES_ROOT', '/mnt/user/appdata/compose.manager/git'); /** diff --git a/source/compose.manager/include/GitCommand.php b/source/compose.manager/include/GitCommand.php new file mode 100644 index 00000000..5ea5bddb --- /dev/null +++ b/source/compose.manager/include/GitCommand.php @@ -0,0 +1,155 @@ + $extraEnvironment Extra environment variables for this run + * @param string[] $trustedRepositories Repositories git may use even when another user owns them + */ + public static function run( + array $args, + ?string $workingDirectory = null, + int $timeoutSeconds = self::DEFAULT_TIMEOUT_SECONDS, + array $extraEnvironment = [], + array $trustedRepositories = [] + ): ProcessResult { + $homeDirectory = self::prepareHomeDirectory(); + + $command = ['git']; + foreach (self::FORCED_CONFIG as $setting) { + $command[] = '-c'; + $command[] = $setting; + } + foreach ($args as $arg) { + $command[] = $arg; + } + + $environment = [ + 'PATH' => '/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin', + 'HOME' => $homeDirectory, + 'XDG_CONFIG_HOME' => $homeDirectory, + 'GIT_CONFIG_NOSYSTEM' => '1', + 'GIT_CONFIG_GLOBAL' => self::trustFile($homeDirectory, $workingDirectory, $trustedRepositories), + 'GIT_TERMINAL_PROMPT' => '0', + 'GIT_ASKPASS' => '/bin/false', + // Paths are always literal names, never patterns such as "*" or ":(top)". + 'GIT_LITERAL_PATHSPECS' => '1', + // Plain English messages, so errors read the same everywhere. + 'LC_ALL' => 'C', + ]; + foreach ($extraEnvironment as $name => $value) { + $environment[$name] = $value; + } + + return ProcessRunner::run($command, $workingDirectory ?? $homeDirectory, $environment, $timeoutSeconds); + } + + /** + * The global config file for a run: /dev/null, or a file that only lists + * the repositories git may use although another user owns them. + * + * git refuses such a repository ("dubious ownership"). A clone can end up + * owned by nobody:users after Unraid's New Permissions tool, and a + * repository on this server is often owned by the container that keeps it. + * Only the folder git is asked to work in and the stack's own repository + * are listed. It has to be the global config: git 2.47 ignores + * safe.directory given with -c when it reads from a local repository. + * One small file is kept per distinct list, in the home folder under + * /var/tmp, which is in RAM and emptied at reboot. + * + * @param string[] $trustedRepositories + */ + private static function trustFile(string $homeDirectory, ?string $workingDirectory, array $trustedRepositories): string + { + $paths = []; + foreach (array_merge($workingDirectory !== null ? [$workingDirectory] : [], $trustedRepositories) as $path) { + $paths[] = $path; + $real = realpath($path); + if ($real !== false) { + $paths[] = $real; + } + } + $paths = array_values(array_unique($paths)); + if ($paths === []) { + return '/dev/null'; + } + + $content = "[safe]\n"; + foreach ($paths as $path) { + // Quoted, with \ and " escaped, as git's config format needs. + $content .= "\tdirectory = \"" . addcslashes($path, "\\\"") . "\"\n"; + } + $file = $homeDirectory . '/trust-' . hash('sha256', $content) . '.gitconfig'; + if (!is_file($file)) { + $tmpFile = $file . '.tmp-' . bin2hex(random_bytes(4)); + // A short write (a full /var/tmp) must not leave a cut-off file under the + // content's name: it is never rewritten, and git would refuse to read it. + if (file_put_contents($tmpFile, $content) !== strlen($content) || !rename($tmpFile, $file)) { + @unlink($tmpFile); + return '/dev/null'; + } + } + return $file; + } + + /** + * The empty folder git runs with as HOME, so no personal config is read. + */ + private static function prepareHomeDirectory(): string + { + $home = COMPOSE_GIT_HOME_DIR; + if (!is_dir($home)) { + @mkdir($home, 0700, true); + } + return $home; + } +} diff --git a/source/compose.manager/include/ProcessRunner.php b/source/compose.manager/include/ProcessRunner.php new file mode 100644 index 00000000..75026eef --- /dev/null +++ b/source/compose.manager/include/ProcessRunner.php @@ -0,0 +1,128 @@ +exitCode === 0 && !$this->timedOut; + } + + /** + * A one-line description of a failure, for messages and logs. + */ + public function errorSummary(): string + { + if ($this->timedOut) { + return 'The command did not finish in time and was stopped.'; + } + $lines = array_values(array_filter( + array_map('trim', explode("\n", $this->stderr)), + static fn(string $line): bool => $line !== '' + )); + if ($lines === []) { + return "The command exited with code $this->exitCode."; + } + // git and docker put the useful line after "fatal:" or "error:"; hints may follow it. + foreach (array_reverse($lines) as $line) { + if (str_starts_with($line, 'fatal:') || str_starts_with($line, 'error:')) { + return $line; + } + } + return $lines[count($lines) - 1]; + } +} + +/** + * Runs a command without a shell, with no input, a clean environment and a time limit. + */ +final class ProcessRunner +{ + /** + * @param string[] $command The program and its arguments, passed as-is (no shell) + * @param array $environment The complete environment for the command + */ + public static function run(array $command, string $workingDirectory, array $environment, int $timeoutSeconds): ProcessResult + { + // GNU timeout stops the command and every helper it started, since it + // signals the whole process group. + $fullCommand = array_merge(['timeout', '--kill-after=10', (string) $timeoutSeconds], $command); + + $descriptors = [ + 0 => ['pipe', 'r'], + 1 => ['pipe', 'w'], + 2 => ['pipe', 'w'], + ]; + $startedAt = microtime(true); + $process = proc_open($fullCommand, $descriptors, $pipes, $workingDirectory, $environment); + if (!is_resource($process)) { + return new ProcessResult(127, '', 'Could not start ' . $command[0] . '.', false); + } + + // The command gets no input: its stdin is closed straight away. + fclose($pipes[0]); + + // Read both pipes as output arrives, so a full pipe can never stall the command. + stream_set_blocking($pipes[1], false); + stream_set_blocking($pipes[2], false); + $stdout = ''; + $stderr = ''; + while (true) { + // A pipe at end-of-file always counts as readable, so it is left + // out of the wait; otherwise the loop would spin while the command + // runs on with that output closed. + $read = []; + foreach ([1, 2] as $index) { + if (!feof($pipes[$index])) { + $read[] = $pipes[$index]; + } + } + if ($read === []) { + // Usually the command is exiting: check again shortly. A command + // that closes both outputs and keeps running is checked every 20 ms. + usleep(20000); + } else { + $write = null; + $except = null; + @stream_select($read, $write, $except, 1); + } + $stdout .= (string) stream_get_contents($pipes[1]); + $stderr .= (string) stream_get_contents($pipes[2]); + $status = proc_get_status($process); + if (!$status['running']) { + break; + } + } + $stdout .= (string) stream_get_contents($pipes[1]); + $stderr .= (string) stream_get_contents($pipes[2]); + fclose($pipes[1]); + fclose($pipes[2]); + proc_close($process); + + $exitCode = (int) $status['exitcode']; + if ($status['signaled']) { + // Killed by a signal (timeout passes a child's signal on): report it + // the way a shell does, as 128 plus the signal number. + $exitCode = 128 + (int) $status['termsig']; + } + // timeout exits 124 when it stopped the command, or 137 when it had to + // kill it. 137 also means the command was killed by something else (such + // as the out-of-memory killer), so the time spent decides. + $ranOutOfTime = microtime(true) - $startedAt >= $timeoutSeconds; + $timedOut = ($exitCode === 124 || $exitCode === 137) && $ranOutOfTime; + + return new ProcessResult($exitCode, $stdout, $stderr, $timedOut); + } +} diff --git a/tests/bootstrap.php b/tests/bootstrap.php index 30d60a2c..133343be 100644 --- a/tests/bootstrap.php +++ b/tests/bootstrap.php @@ -51,6 +51,7 @@ function composeLogger($message, $data = null, $type = 'daemon', $level = 'info' define('COMPOSE_GIT_MNT_DIR', sys_get_temp_dir() . '/compose_git_mnt'); define('COMPOSE_UNRAID_VAR_INI', sys_get_temp_dir() . '/compose_git_var.ini'); define('COMPOSE_MOUNTS_FILE', sys_get_temp_dir() . '/compose_git_mounts'); +define('COMPOSE_GIT_HOME_DIR', sys_get_temp_dir() . '/compose_git_home'); // Point to the dev-env resvg binary when present; plugin path used on real Unraid define('COMPOSE_RESVG_BIN', is_executable('/tmp/resvg') ? '/tmp/resvg' : '/usr/local/emhttp/plugins/compose.manager/bin/resvg'); diff --git a/tests/unit/GitCommandTest.php b/tests/unit/GitCommandTest.php new file mode 100644 index 00000000..a70af62d --- /dev/null +++ b/tests/unit/GitCommandTest.php @@ -0,0 +1,235 @@ +workDir = sys_get_temp_dir() . '/compose_git_command_test_' . getmypid(); + $this->removeTree($this->workDir); + mkdir($this->workDir, 0755, true); + } + + protected function tearDown(): void + { + putenv('GIT_DIR'); + putenv('GIT_WORK_TREE'); + $this->removeTree($this->workDir); + parent::tearDown(); + } + + public function testRunsGitAndCapturesOutput(): void + { + $result = GitCommand::run(['--version']); + $this->assertTrue($result->succeeded()); + $this->assertStringStartsWith('git version', $result->stdout); + } + + public function testFailureReportsTheLastErrorLine(): void + { + $result = GitCommand::run(['rev-parse', '--verify', 'no-such-ref'], $this->makeRepo()); + $this->assertFalse($result->succeeded()); + $this->assertStringContainsString('fatal', $result->errorSummary()); + } + + public function testStrayGitEnvironmentVariablesAreIgnored(): void + { + $repo = $this->makeRepo(); + putenv('GIT_DIR=' . $this->workDir . '/elsewhere'); + putenv('GIT_WORK_TREE=' . $this->workDir . '/elsewhere'); + + $result = GitCommand::run(['rev-parse', '--show-toplevel'], $repo); + + $this->assertTrue($result->succeeded(), $result->stderr); + $this->assertSame(realpath($repo), trim($result->stdout)); + } + + public function testRepositoryHooksNeverRun(): void + { + $repo = $this->makeRepo(); + $marker = $this->workDir . '/hook-ran'; + $hook = $repo . '/.git/hooks/post-checkout'; + file_put_contents($hook, "#!/bin/sh\ntouch '$marker'\n"); + chmod($hook, 0755); + + $result = GitCommand::run(['checkout', '--detach', 'HEAD'], $repo); + + $this->assertTrue($result->succeeded(), $result->stderr); + $this->assertFileDoesNotExist($marker); + } + + public function testHooksPathSetInTheRepositoryConfigIsOverridden(): void + { + $repo = $this->makeRepo(); + $marker = $this->workDir . '/hook-ran'; + mkdir($this->workDir . '/evil-hooks'); + file_put_contents($this->workDir . '/evil-hooks/post-checkout', "#!/bin/sh\ntouch '$marker'\n"); + chmod($this->workDir . '/evil-hooks/post-checkout', 0755); + $this->git(['config', 'core.hooksPath', $this->workDir . '/evil-hooks'], $repo); + + GitCommand::run(['checkout', '--detach', 'HEAD'], $repo); + + $this->assertFileDoesNotExist($marker); + } + + public function testExtTransportIsRefused(): void + { + $marker = $this->workDir . '/ext-ran'; + $result = GitCommand::run(['ls-remote', "ext::sh -c touch% $marker"]); + + $this->assertFalse($result->succeeded()); + $this->assertFileDoesNotExist($marker); + } + + public function testPlainHttpTransportIsRefused(): void + { + $result = GitCommand::run(['ls-remote', 'http://127.0.0.1:9/repo.git'], null, 10); + $this->assertFalse($result->succeeded()); + $this->assertStringContainsString('transport', $result->stderr); + } + + /** + * git allows ssh and git by default (unlike ext), so only the runner's + * protocol.allow=never refuses them. + * + * @return array + */ + public static function transportsGitAllowsByDefault(): array + { + return [ + 'ssh' => ['ssh://127.0.0.1:9/repo.git'], + 'git' => ['git://127.0.0.1:9/repo.git'], + ]; + } + + #[\PHPUnit\Framework\Attributes\DataProvider('transportsGitAllowsByDefault')] + public function testTransportsGitAllowsByDefaultAreRefused(string $url): void + { + $result = GitCommand::run(['ls-remote', '--', $url], null, 10); + $this->assertFalse($result->succeeded()); + $this->assertStringContainsString('not allowed', $result->stderr); + } + + public function testLocalPathTransportIsAllowed(): void + { + $repo = $this->makeRepo(); + $result = GitCommand::run(['ls-remote', '--', $repo]); + $this->assertTrue($result->succeeded(), $result->stderr); + $this->assertStringContainsString('refs/heads/main', $result->stdout); + } + + public function testArgumentsArePassedLiterallyWithoutAShell(): void + { + $repo = $this->makeRepo(); + $name = '-rf $(touch pwned); `id`'; + file_put_contents($repo . '/' . $name, 'x'); + + $result = GitCommand::run(['add', '--', $name], $repo); + + $this->assertTrue($result->succeeded(), $result->stderr); + $this->assertFileDoesNotExist($repo . '/pwned'); + $status = GitCommand::run(['status', '--porcelain', '-z'], $repo); + $this->assertStringContainsString($name, $status->stdout); + } + + public function testRunIsStoppedAfterTheTimeLimit(): void + { + $started = microtime(true); + $result = GitCommand::run(['-c', 'alias.wait=!sleep 30', 'wait'], $this->makeRepo(), 1); + + $this->assertTrue($result->timedOut); + $this->assertFalse($result->succeeded()); + $this->assertLessThan(15, microtime(true) - $started); + } + + public function testCommandKilledBySomethingElseIsNotCalledATimeout(): void + { + $result = \ProcessRunner::run(['sh', '-c', 'kill -9 $$'], '/', ['PATH' => '/usr/bin:/bin'], 30); + + $this->assertSame(137, $result->exitCode); + $this->assertFalse($result->timedOut); + } + + public function testCommandThatClosesItsOutputEarlyIsStillReadToTheEnd(): void + { + $started = microtime(true); + $result = \ProcessRunner::run( + ['sh', '-c', 'exec 1>&-; sleep 1; echo finished >&2'], + '/', + ['PATH' => '/usr/bin:/bin'], + 30 + ); + + $this->assertSame(0, $result->exitCode); + $this->assertSame("finished\n", $result->stderr); + $this->assertLessThan(10, microtime(true) - $started); + } + + public function testRepositoryOwnedByAnotherUserIsUsedWhenItIsTheWorkingFolder(): void + { + if (posix_geteuid() !== 0) { + $this->markTestSkipped('Needs root to change file owners.'); + } + $repo = $this->makeRepo(); + $chown = \ProcessRunner::run(['chown', '-R', '65534', $repo], '/', ['PATH' => '/usr/bin:/bin'], 30); + $this->assertTrue($chown->succeeded(), $chown->stderr); + + $this->assertTrue(GitCommand::run(['status', '--porcelain'], $repo)->succeeded()); + // Only the folder git works in is trusted, not every repository. + $this->assertFalse(GitCommand::run(['-C', $repo, 'status', '--porcelain'])->succeeded()); + } + + // ----- helpers ----- + + private function makeRepo(): string + { + $repo = $this->workDir . '/repo'; + if (!is_dir($repo . '/.git')) { + mkdir($repo, 0755, true); + $this->git(['init', '-q', '-b', 'main'], $repo); + file_put_contents($repo . '/compose.yaml', "services: {}\n"); + $this->git(['add', 'compose.yaml'], $repo); + $this->git(['-c', 'user.name=t', '-c', 'user.email=t@example.invalid', 'commit', '-q', '-m', 'first'], $repo); + } + return $repo; + } + + /** @param string[] $args */ + private function git(array $args, string $dir): void + { + $result = GitCommand::run($args, $dir); + $this->assertTrue($result->succeeded(), 'git ' . implode(' ', $args) . ': ' . $result->stderr); + } + + private function removeTree(string $path): void + { + // readlink(), not is_link(): the test stream wrapper hides symlinks from is_link(). + if (@readlink($path) !== false || is_file($path)) { + unlink($path); + return; + } + if (!is_dir($path)) { + return; + } + foreach (scandir($path) ?: [] as $entry) { + if ($entry !== '.' && $entry !== '..') { + $this->removeTree($path . '/' . $entry); + } + } + rmdir($path); + } +} From 7cb47cb500fc8b35b687bb372eeb634ee7664114 Mon Sep 17 00:00:00 2001 From: Grant Emsley Date: Sat, 3 Oct 2026 14:33:03 -0400 Subject: [PATCH 003/109] feat(git): store and validate a git stack's settings A git stack keeps its repository address, branch, compose file path and clone folder in git.json in the stack folder. Every value is checked when the settings are created and again on every load, since the file can be edited by hand or restored from a backup: - the address must be an https URL with no user name or password in it, or a repository path under /mnt; - the branch must pass git check-ref-format and cannot start with "-"; - the compose path must be a relative .yml/.yaml path that stays inside the repository and outside .git; - the clone folder must be under /mnt and carry the stack's clone id, so two stacks can never share one. A file with missing, extra or wrongly typed settings is reported, never partly used. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_011iUWDG8wTp8MhsFDqNC4NL --- .../include/GitStackSettings.php | 349 +++++++++++++++ tests/unit/GitStackSettingsTest.php | 399 ++++++++++++++++++ .../fixtures/git-stack-settings/format-1.json | 8 + 3 files changed, 756 insertions(+) create mode 100644 source/compose.manager/include/GitStackSettings.php create mode 100644 tests/unit/GitStackSettingsTest.php create mode 100644 tests/unit/fixtures/git-stack-settings/format-1.json diff --git a/source/compose.manager/include/GitStackSettings.php b/source/compose.manager/include/GitStackSettings.php new file mode 100644 index 00000000..80414390 --- /dev/null +++ b/source/compose.manager/include/GitStackSettings.php @@ -0,0 +1,349 @@ +getMessage(), 0, $error); + } + } + + /** + * Write git.json to the stack folder, replacing it in one step. + * + * @throws RuntimeException if it cannot be written safely + */ + public function save(string $stackDir): void + { + $stackDir = rtrim($stackDir, '/'); + if (str_starts_with($stackDir, rtrim(COMPOSE_GIT_MNT_DIR, '/') . '/')) { + // A projects folder kept on the array: never write there while it is not mounted. + GitPathGuard::assertSafeToWrite($stackDir); + } + if (!is_dir($stackDir)) { + throw new RuntimeException("Stack folder $stackDir does not exist."); + } + + $json = json_encode([ + 'version' => self::FORMAT_VERSION, + 'url' => $this->url, + 'branch' => $this->branch, + 'composePath' => $this->composePath, + 'cloneId' => $this->cloneId, + 'cloneDir' => $this->cloneDir, + ], JSON_PRETTY_PRINT | JSON_UNESCAPED_SLASHES); + if ($json === false) { + throw new RuntimeException('Could not encode the git settings.'); + } + + $file = $stackDir . '/' . self::FILE_NAME; + $tmpFile = $file . '.tmp-' . bin2hex(random_bytes(4)); + // A short write (a full flash) counts as a failure, so a cut-off file never replaces the good one. + $content = $json . "\n"; + if (file_put_contents($tmpFile, $content) !== strlen($content)) { + @unlink($tmpFile); + throw new RuntimeException("Could not write $tmpFile."); + } + if (!rename($tmpFile, $file)) { + @unlink($tmpFile); + throw new RuntimeException("Could not replace $file."); + } + } + + /** + * Full path of the compose file inside the clone. + */ + public function composeFileInClone(): string + { + return $this->cloneDir . '/' . $this->composePath; + } + + /** + * The clone folder name for a stack: a readable part from the stack folder + * name, plus part of the clone id so two stacks can never share a folder. + */ + public static function cloneFolderName(string $stackFolderName, string $cloneId): string + { + $readable = strtolower($stackFolderName); + $readable = (string) preg_replace('/[^a-z0-9._-]+/', '-', $readable); + $readable = trim($readable, '.-_'); + $readable = substr($readable, 0, 40); + if ($readable === '') { + $readable = 'stack'; + } + return $readable . '-' . substr($cloneId, 0, 8); + } + + /** + * Check a repository address. + * + * Accepted: an https URL without a user name or password in it, or the + * absolute path of a repository under /mnt (for a bare repository on this + * server that you push to). + * + * @throws InvalidArgumentException naming the problem + */ + public static function validateUrl(string $url): void + { + if ($url === '') { + throw new InvalidArgumentException('The repository address is empty.'); + } + if (strlen($url) > 2048) { + throw new InvalidArgumentException('The repository address is too long.'); + } + if (preg_match('/[\x00-\x20\x7f]/', $url) === 1) { + throw new InvalidArgumentException('The repository address must not contain spaces or control characters.'); + } + + if (str_starts_with($url, '/')) { + GitPathGuard::assertCleanAbsolutePath($url, 'The repository path'); + if (!str_starts_with($url, rtrim(COMPOSE_GIT_MNT_DIR, '/') . '/')) { + throw new InvalidArgumentException( + 'A repository on this server must be under ' . COMPOSE_GIT_MNT_DIR . ": $url" + ); + } + return; + } + + // Lower case only: git matches the transport name exactly, and refuses "HTTPS". + if (!str_starts_with($url, 'https://')) { + throw new InvalidArgumentException( + "The repository address must start with https://, or be a path under " . COMPOSE_GIT_MNT_DIR + . " for a repository on this server: $url" + ); + } + + $parts = parse_url($url); + if ($parts === false || !isset($parts['host']) || $parts['host'] === '') { + throw new InvalidArgumentException("The repository address is not a valid URL: $url"); + } + if (isset($parts['user']) || isset($parts['pass'])) { + throw new InvalidArgumentException( + 'The repository address must not contain a user name, password or token ' + . '(it would be stored in plain text). Private repositories are not supported yet.' + ); + } + if (isset($parts['query']) || isset($parts['fragment'])) { + throw new InvalidArgumentException("The repository address must not contain ? or #: $url"); + } + if (preg_match('/^(?:[a-z0-9](?:[a-z0-9-]*[a-z0-9])?\.)*[a-z0-9](?:[a-z0-9-]*[a-z0-9])?$/i', $parts['host']) !== 1) { + throw new InvalidArgumentException("The repository address has an invalid host name: {$parts['host']}"); + } + $path = $parts['path'] ?? ''; + if ($path === '' || $path === '/') { + throw new InvalidArgumentException("The repository address has no repository path: $url"); + } + } + + /** + * Check a branch name, using git's own rules. + * + * @throws InvalidArgumentException naming the problem + */ + public static function validateBranch(string $branch): void + { + if ($branch === '') { + throw new InvalidArgumentException('The branch is empty.'); + } + if (strlen($branch) > 255) { + throw new InvalidArgumentException('The branch name is too long.'); + } + if (str_starts_with($branch, '-')) { + throw new InvalidArgumentException("The branch name must not start with '-': $branch"); + } + if ($branch === 'HEAD') { + throw new InvalidArgumentException("'HEAD' is not a branch name. Enter the branch to deploy, such as main."); + } + $result = GitCommand::run(['check-ref-format', 'refs/heads/' . $branch], null, 10); + if (!$result->succeeded()) { + throw new InvalidArgumentException("'$branch' is not a valid branch name."); + } + } + + /** + * Check the path of the compose file inside the repository. + * + * It must be a relative path to a .yml or .yaml file that stays inside the + * repository. + * + * @throws InvalidArgumentException naming the problem + */ + public static function validateComposePath(string $composePath): void + { + if ($composePath === '') { + throw new InvalidArgumentException('The compose file path is empty.'); + } + if (strlen($composePath) > 1024) { + throw new InvalidArgumentException('The compose file path is too long.'); + } + if (preg_match('/[\x00-\x1f\x7f]/', $composePath) === 1) { + throw new InvalidArgumentException('The compose file path contains a control character.'); + } + if (str_starts_with($composePath, '/')) { + throw new InvalidArgumentException( + "The compose file path must be relative to the top of the repository, without a leading /: $composePath" + ); + } + if (str_contains($composePath, '\\')) { + throw new InvalidArgumentException("Use / in the compose file path, not \\: $composePath"); + } + if (str_starts_with($composePath, '-')) { + throw new InvalidArgumentException("The compose file path must not start with '-': $composePath"); + } + foreach (explode('/', $composePath) as $part) { + if ($part === '' || $part === '.' || $part === '..') { + throw new InvalidArgumentException("The compose file path must not contain empty, '.' or '..' parts: $composePath"); + } + if (strtolower($part) === '.git') { + throw new InvalidArgumentException("The compose file path must not be inside .git: $composePath"); + } + } + if (preg_match('/\.ya?ml$/i', $composePath) !== 1) { + throw new InvalidArgumentException("The compose file path must end in .yml or .yaml: $composePath"); + } + } + + /** + * Validate every value and build the settings. + * + * @throws InvalidArgumentException naming the first problem found + */ + private static function fromValues( + string $url, + string $branch, + string $composePath, + string $cloneId, + string $cloneDir + ): self { + self::validateUrl($url); + self::validateBranch($branch); + self::validateComposePath($composePath); + + if (preg_match('/^[0-9a-f]{16}$/', $cloneId) !== 1) { + throw new InvalidArgumentException("The clone id is not valid: $cloneId"); + } + GitPathGuard::assertCleanAbsolutePath($cloneDir, 'The clone folder'); + if (!str_starts_with($cloneDir, rtrim(COMPOSE_GIT_MNT_DIR, '/') . '/')) { + throw new InvalidArgumentException('The clone folder must be under ' . COMPOSE_GIT_MNT_DIR . ": $cloneDir"); + } + if (!str_ends_with(basename($cloneDir), '-' . substr($cloneId, 0, 8))) { + throw new InvalidArgumentException("The clone folder $cloneDir does not belong to clone id $cloneId."); + } + + return new self($url, $branch, $composePath, $cloneId, $cloneDir); + } +} diff --git a/tests/unit/GitStackSettingsTest.php b/tests/unit/GitStackSettingsTest.php new file mode 100644 index 00000000..b028c6af --- /dev/null +++ b/tests/unit/GitStackSettingsTest.php @@ -0,0 +1,399 @@ +mnt = COMPOSE_GIT_MNT_DIR; + $this->removeTree($this->mnt); + mkdir($this->mnt . '/user/appdata', 0755, true); + file_put_contents(COMPOSE_UNRAID_VAR_INI, "mdState=\"STARTED\"\n"); + file_put_contents(COMPOSE_MOUNTS_FILE, "rootfs {$this->mnt} rootfs rw 0 0\nshfs {$this->mnt}/user fuse.shfs rw 0 0\n"); + + $this->stackDir = sys_get_temp_dir() . '/compose_git_settings_stack'; + $this->removeTree($this->stackDir); + mkdir($this->stackDir); + } + + protected function tearDown(): void + { + $this->removeTree($this->mnt); + $this->removeTree($this->stackDir); + @unlink(COMPOSE_UNRAID_VAR_INI); + @unlink(COMPOSE_MOUNTS_FILE); + parent::tearDown(); + } + + // ----- repository address ----- + + /** @return array */ + public static function goodUrls(): array + { + return [ + 'github' => ['https://github.com/owner/repo.git'], + 'no .git suffix' => ['https://github.com/owner/repo'], + 'port' => ['https://git.example.com:3000/owner/repo.git'], + 'nested path' => ['https://gitlab.example.com/group/sub/repo.git'], + 'local bare repo' => [COMPOSE_GIT_MNT_DIR . '/user/appdata/repos/stacks.git'], + ]; + } + + #[DataProvider('goodUrls')] + public function testGoodRepositoryAddressesAreAccepted(string $url): void + { + GitStackSettings::validateUrl($url); + $this->addToAssertionCount(1); + } + + /** @return array */ + public static function badUrls(): array + { + return [ + 'empty' => ['', 'empty'], + 'plain http' => ['http://github.com/owner/repo.git', 'https://'], + 'upper case scheme, which git refuses' => ['HTTPS://github.com/owner/repo.git', 'https://'], + 'ssh' => ['ssh://git@github.com/owner/repo.git', 'https://'], + 'scp style' => ['git@github.com:owner/repo.git', 'https://'], + 'file url' => ['file:///mnt/user/repo.git', 'https://'], + 'ext transport' => ['ext::sh -c touch% /tmp/x', 'spaces'], + 'ext transport without spaces' => ['ext::sh', 'https://'], + 'leading dash' => ['--upload-pack=touch /tmp/x', 'spaces'], + 'leading dash no space' => ['-uhttps://x/y', 'https://'], + 'token in url' => ['https://user:ghp_secret@github.com/owner/repo.git', 'password'], + 'user in url' => ['https://user@github.com/owner/repo.git', 'user name'], + 'query' => ['https://github.com/owner/repo.git?x=1', '?'], + 'no path' => ['https://github.com', 'no repository path'], + 'bad host' => ['https://-evil.com/repo.git', 'host'], + 'space' => ['https://github.com/owner/my repo.git', 'spaces'], + 'newline' => ["https://github.com/owner/repo.git\n", 'spaces'], + 'relative path' => ['repos/stacks.git', 'https://'], + 'local outside mnt' => ['/boot/repos/stacks.git', 'must be under'], + 'local traversal' => [COMPOSE_GIT_MNT_DIR . '/user/../../etc', "'..'"], + ]; + } + + #[DataProvider('badUrls')] + public function testBadRepositoryAddressesAreRefused(string $url, string $messagePart): void + { + $this->expectException(InvalidArgumentException::class); + $this->expectExceptionMessage($messagePart); + GitStackSettings::validateUrl($url); + } + + // ----- branch ----- + + /** @return array */ + public static function goodBranches(): array + { + return [['main'], ['master'], ['release/2026.10'], ['feature-x_y']]; + } + + #[DataProvider('goodBranches')] + public function testGoodBranchNamesAreAccepted(string $branch): void + { + GitStackSettings::validateBranch($branch); + $this->addToAssertionCount(1); + } + + /** @return array */ + public static function badBranches(): array + { + return [ + 'empty' => [''], + 'leading dash' => ['-main'], + 'option' => ['--upload-pack=x'], + 'double dot' => ['a..b'], + 'reflog syntax' => ['main@{1}'], + 'space' => ['my branch'], + 'tilde' => ['main~1'], + 'caret' => ['main^'], + 'colon' => ['a:b'], + 'trailing lock' => ['main.lock'], + 'trailing slash' => ['main/'], + 'HEAD' => ['HEAD'], + 'newline' => ["main\n"], + ]; + } + + #[DataProvider('badBranches')] + public function testBadBranchNamesAreRefused(string $branch): void + { + $this->expectException(InvalidArgumentException::class); + GitStackSettings::validateBranch($branch); + } + + // ----- compose path ----- + + /** @return array */ + public static function goodComposePaths(): array + { + return [['compose.yaml'], ['docker-compose.yml'], ['whoami/compose.yaml'], ['stacks/media/Plex.YML'], ['a b/c.yaml']]; + } + + #[DataProvider('goodComposePaths')] + public function testGoodComposePathsAreAccepted(string $path): void + { + GitStackSettings::validateComposePath($path); + $this->addToAssertionCount(1); + } + + /** @return array */ + public static function badComposePaths(): array + { + return [ + 'empty' => [''], + 'absolute' => ['/etc/compose.yaml'], + 'traversal' => ['../compose.yaml'], + 'inner traversal' => ['a/../../compose.yaml'], + 'dot part' => ['./compose.yaml'], + 'double slash' => ['a//compose.yaml'], + 'trailing slash' => ['a/'], + 'backslash' => ['a\\compose.yaml'], + 'inside .git' => ['.git/compose.yaml'], + 'inside .GIT' => ['.GIT/compose.yaml'], + 'not yaml' => ['compose.json'], + 'directory' => ['whoami'], + 'leading dash' => ['-compose.yaml'], + 'newline' => ["compose.yaml\n"], + ]; + } + + #[DataProvider('badComposePaths')] + public function testBadComposePathsAreRefused(string $path): void + { + $this->expectException(InvalidArgumentException::class); + GitStackSettings::validateComposePath($path); + } + + // ----- clone folder ----- + + public function testCloneFolderNameIsReadableAndUnique(): void + { + $this->assertSame('my-stack-0123abcd', GitStackSettings::cloneFolderName('My Stack', '0123abcd89ef4567')); + $this->assertSame('stack-0123abcd', GitStackSettings::cloneFolderName('...', '0123abcd89ef4567')); + $this->assertSame('caf-0123abcd', GitStackSettings::cloneFolderName("Caf\u{e9}", '0123abcd89ef4567')); + } + + public function testTwoStacksWithNamesThatLookAlikeGetDifferentCloneFolders(): void + { + $root = $this->mnt . '/user/appdata/git'; + $first = GitStackSettings::createNew('https://github.com/o/r.git', 'main', 'compose.yaml', $root, 'My Stack'); + $second = GitStackSettings::createNew('https://github.com/o/r.git', 'main', 'compose.yaml', $root, 'my-stack'); + $this->assertNotSame($first->cloneDir, $second->cloneDir); + } + + public function testCreateNewRefusesAClonesRootInRam(): void + { + $this->expectException(InvalidArgumentException::class); + GitStackSettings::createNew('https://github.com/o/r.git', 'main', 'compose.yaml', $this->mnt . '/usr/appdata/git', 'x'); + } + + // ----- load and save ----- + + public function testSaveThenLoadGivesTheSameSettings(): void + { + $settings = $this->makeSettings(); + $settings->save($this->stackDir); + $loaded = GitStackSettings::load($this->stackDir); + + $this->assertNotNull($loaded); + $this->assertSame($settings->url, $loaded->url); + $this->assertSame($settings->branch, $loaded->branch); + $this->assertSame($settings->composePath, $loaded->composePath); + $this->assertSame($settings->cloneId, $loaded->cloneId); + $this->assertSame($settings->cloneDir, $loaded->cloneDir); + $this->assertSame([], glob($this->stackDir . '/git.json.tmp-*') ?: []); + } + + public function testStackWithoutGitJsonIsNotAGitStack(): void + { + $this->assertNull(GitStackSettings::load($this->stackDir)); + } + + public function testFileWithWindowsLineEndingsAndByteOrderMarkLoads(): void + { + $this->makeSettings()->save($this->stackDir); + $file = $this->stackDir . '/git.json'; + file_put_contents($file, "\xEF\xBB\xBF" . str_replace("\n", "\r\n", (string) file_get_contents($file))); + $this->assertNotNull(GitStackSettings::load($this->stackDir)); + } + + /** @return array): mixed}> */ + public static function brokenFiles(): array + { + return [ + 'not json' => [static fn(array $data): string => '{not json'], + 'list instead of object' => [static fn(array $data): string => '[1, 2]'], + 'missing key' => [static function (array $data): string { + unset($data['branch']); + return (string) json_encode($data); + }], + 'extra key' => [static function (array $data): string { + $data['deployNow'] = true; + return (string) json_encode($data); + }], + 'newer format' => [static function (array $data): string { + $data['version'] = 2; + return (string) json_encode($data); + }], + 'number instead of string' => [static function (array $data): string { + $data['branch'] = 5; + return (string) json_encode($data); + }], + 'hand-edited bad branch' => [static function (array $data): string { + $data['branch'] = '--upload-pack=touch /tmp/x'; + return (string) json_encode($data); + }], + 'hand-edited compose path escaping the clone' => [static function (array $data): string { + $data['composePath'] = '../../etc/compose.yaml'; + return (string) json_encode($data); + }], + 'clone folder moved to the flash' => [static function (array $data): string { + $data['cloneDir'] = '/boot/config/x-' . substr((string) $data['cloneId'], 0, 8); + return (string) json_encode($data); + }], + 'clone folder of another stack' => [static function (array $data): string { + $data['cloneDir'] = dirname((string) $data['cloneDir']) . '/other-ffffffff'; + return (string) json_encode($data); + }], + 'clone id not hex' => [static function (array $data): string { + $data['cloneId'] = '../../../../etc'; + return (string) json_encode($data); + }], + ]; + } + + #[DataProvider('brokenFiles')] + public function testBrokenSettingsFileIsReportedNotUsed(callable $breakIt): void + { + $this->makeSettings()->save($this->stackDir); + $file = $this->stackDir . '/git.json'; + $data = json_decode((string) file_get_contents($file), true); + file_put_contents($file, $breakIt($data)); + + $this->expectException(RuntimeException::class); + GitStackSettings::load($this->stackDir); + } + + // ----- files written by released versions ----- + + /** @return array */ + public static function releasedFormats(): array + { + $formats = []; + foreach (glob(__DIR__ . '/fixtures/git-stack-settings/*.json') ?: [] as $fixture) { + $formats[basename($fixture, '.json')] = [$fixture]; + } + return $formats; + } + + #[DataProvider('releasedFormats')] + public function testFileWrittenByAReleasedVersionStillLoads(string $fixture): void + { + // Every git.json a released version wrote must keep loading (see the class comment). + $content = str_replace('@MNT@', $this->mnt, (string) file_get_contents($fixture)); + file_put_contents($this->stackDir . '/git.json', $content); + + $settings = GitStackSettings::load($this->stackDir); + + $this->assertNotNull($settings); + $this->assertSame('main', $settings->branch); + } + + public function testFormatOneLoadsWithTheBehaviourItWasWrittenWith(): void + { + $content = str_replace('@MNT@', $this->mnt, (string) file_get_contents(__DIR__ . '/fixtures/git-stack-settings/format-1.json')); + file_put_contents($this->stackDir . '/git.json', $content); + + $settings = GitStackSettings::load($this->stackDir); + + $this->assertSame('https://github.com/owner/repo.git', $settings->url); + $this->assertSame('whoami/compose.yaml', $settings->composePath); + } + + public function testSettingFromANewerVersionIsRefusedByName(): void + { + $this->makeSettings()->save($this->stackDir); + $file = $this->stackDir . '/git.json'; + $data = json_decode((string) file_get_contents($file), true); + $data['autoDeploy'] = 'off'; + file_put_contents($file, (string) json_encode($data)); + + $this->expectException(RuntimeException::class); + $this->expectExceptionMessage('does not know: autoDeploy'); + GitStackSettings::load($this->stackDir); + } + + public function testJsonSyntaxErrorIsNamed(): void + { + $this->makeSettings()->save($this->stackDir); + $file = $this->stackDir . '/git.json'; + // A trailing comma, the usual slip when editing by hand. + file_put_contents($file, str_replace("\n}", ",\n}", (string) file_get_contents($file))); + + $this->expectException(RuntimeException::class); + $this->expectExceptionMessage('is not valid JSON: Syntax error'); + GitStackSettings::load($this->stackDir); + } + + public function testSaveIntoAProjectsFolderOnAStoppedArrayIsRefused(): void + { + $stackOnArray = $this->mnt . '/user/appdata/projects/mystack'; + mkdir($stackOnArray, 0755, true); + file_put_contents(COMPOSE_UNRAID_VAR_INI, "mdState=\"STOPPED\"\n"); + + try { + $this->makeSettings()->save($stackOnArray); + $this->fail('Saved while the array was stopped'); + } catch (RuntimeException) { + $this->assertFileDoesNotExist($stackOnArray . '/git.json'); + } + } + + // ----- helpers ----- + + private function makeSettings(): GitStackSettings + { + return GitStackSettings::createNew( + 'https://github.com/owner/repo.git', + 'main', + 'whoami/compose.yaml', + $this->mnt . '/user/appdata/git', + 'whoami' + ); + } + + private function removeTree(string $path): void + { + // readlink(), not is_link(): the test stream wrapper hides symlinks from is_link(). + if (@readlink($path) !== false || is_file($path)) { + unlink($path); + return; + } + if (!is_dir($path)) { + return; + } + foreach (scandir($path) ?: [] as $entry) { + if ($entry !== '.' && $entry !== '..') { + $this->removeTree($path . '/' . $entry); + } + } + rmdir($path); + } +} diff --git a/tests/unit/fixtures/git-stack-settings/format-1.json b/tests/unit/fixtures/git-stack-settings/format-1.json new file mode 100644 index 00000000..56fca42b --- /dev/null +++ b/tests/unit/fixtures/git-stack-settings/format-1.json @@ -0,0 +1,8 @@ +{ + "version": 1, + "url": "https://github.com/owner/repo.git", + "branch": "main", + "composePath": "whoami/compose.yaml", + "cloneId": "0123456789abcdef", + "cloneDir": "@MNT@/user/appdata/compose.manager/git/whoami-01234567" +} From 02b0c60cc6cc36af730b6ca59c5425b1c5462019 Mon Sep 17 00:00:00 2001 From: Grant Emsley Date: Sat, 3 Oct 2026 14:33:03 -0400 Subject: [PATCH 004/109] feat(git): clone, fetch and check out a git stack's repository safely GitClone manages the clone behind one git stack. It clones only into a folder that does not exist yet (and removes only that folder if the clone fails), marks the clone with the stack's id, and before changing anything checks that the folder is still that marked clone of that repository. The clone is checked out detached at a commit and never committed to. Untracked files, such as data a container writes through a relative bind mount, are never removed or overwritten: there is no git clean and no forced checkout, and a checkout that would put a file where an untracked one is stops and changes nothing. A plain checkout refuses that by itself, so the whole repository is checked out (a sparse checkout overwrites untracked files silently). Ignored files need one more flag: git treats them as expendable and overwrites them by default, so every checkout passes --no-overwrite-ignore. A data folder listed in .gitignore is the usual layout, so this case matters most. Local edits to tracked files are listed, and a checkout is refused while there are any. Saving and discarding them is the next commit. A repository on this server is trusted by git even when another user owns it, so a stack can deploy from one a container keeps under appdata. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_011iUWDG8wTp8MhsFDqNC4NL --- source/compose.manager/include/GitClone.php | 486 ++++++++++++++ tests/unit/GitCloneTest.php | 705 ++++++++++++++++++++ 2 files changed, 1191 insertions(+) create mode 100644 source/compose.manager/include/GitClone.php create mode 100644 tests/unit/GitCloneTest.php diff --git a/source/compose.manager/include/GitClone.php b/source/compose.manager/include/GitClone.php new file mode 100644 index 00000000..32546e52 --- /dev/null +++ b/source/compose.manager/include/GitClone.php @@ -0,0 +1,486 @@ +settings; + } + + /** + * Whether something exists at the clone folder path (a folder, file or symlink). + */ + public function exists(): bool + { + return file_exists($this->settings->cloneDir) || @readlink($this->settings->cloneDir) !== false; + } + + /** + * Ask the remote which commit the branch points at, without changing anything. + * + * @throws RuntimeException if the remote cannot be reached or the branch does not exist + */ + public function remoteBranchCommit(): string + { + $ref = 'refs/heads/' . $this->settings->branch; + $result = $this->git( + ['ls-remote', '--heads', '--', $this->settings->url, $ref], + null, + self::REMOTE_TIMEOUT_SECONDS + ); + if (!$result->succeeded()) { + throw new RuntimeException('Could not reach the repository: ' . $result->errorSummary()); + } + foreach (explode("\n", trim($result->stdout)) as $line) { + $fields = preg_split('/\s+/', trim($line)); + if (is_array($fields) && count($fields) === 2 && $fields[1] === $ref && self::isCommitId($fields[0])) { + return $fields[0]; + } + } + throw new RuntimeException("The branch '{$this->settings->branch}' does not exist in the repository."); + } + + /** + * Clone the repository into the stack's clone folder, checked out at the + * branch's current commit. + * + * The folder must not exist yet. If anything fails, the folder this call + * created is removed again, and nothing else is touched. + * + * @return string the commit checked out + * @throws RuntimeException naming the problem + */ + public function create(): string + { + $cloneDir = $this->settings->cloneDir; + $clonesRoot = dirname($cloneDir); + GitPathGuard::assertValidClonesRoot($clonesRoot); + GitPathGuard::createDirectory($clonesRoot); + + GitPathGuard::assertSafeToWrite($cloneDir); + if ($this->exists()) { + throw new RuntimeException("$cloneDir already exists, so nothing was cloned into it."); + } + + try { + $args = ['clone', '--no-checkout', '--single-branch', '--no-tags', '--branch', $this->settings->branch]; + if ($this->isLocalRepository()) { + // --no-local makes git copy objects the normal way instead of + // hard-linking them, and lets the blob filter below apply. + $args[] = '--no-local'; + } + $args[] = '--filter=blob:none'; + $args[] = '--'; + $args[] = $this->settings->url; + $args[] = $cloneDir; + + $result = $this->git($args, null, self::CLONE_TIMEOUT_SECONDS); + if (!$result->succeeded()) { + throw new RuntimeException('Could not clone the repository: ' . $result->errorSummary()); + } + if (!is_dir($cloneDir . '/.git')) { + throw new RuntimeException("git reported success but $cloneDir/.git is missing."); + } + + GitPathGuard::assertSafeToWrite($cloneDir . '/.git'); + if (file_put_contents($cloneDir . '/.git/' . self::MARKER_FILE, $this->settings->cloneId . "\n") === false) { + throw new RuntimeException("Could not write the clone marker in $cloneDir."); + } + + $commit = $this->resolveCommit('refs/remotes/origin/' . $this->settings->branch); + $this->runOrThrow(['checkout', '--detach', $commit], 'Could not check out the files'); + return $commit; + } catch (Throwable $error) { + $this->removeFolderCreatedByThisRun(); + throw $error instanceof RuntimeException ? $error : new RuntimeException($error->getMessage(), 0, $error); + } + } + + /** + * Throw unless the clone folder is provably this stack's plugin-made clone. + * + * Checks that it is a real folder (not a symlink), that it is the top of a + * git work tree with its own .git folder, that the marker names this + * stack's clone id, and that its remote is still this stack's repository. + * + * @throws GitCloneNotOwnedException naming the mismatch + */ + public function assertOwned(): void + { + $cloneDir = $this->settings->cloneDir; + if (@readlink($cloneDir) !== false) { + throw new GitCloneNotOwnedException("$cloneDir is a symlink, so it was not used."); + } + if (!is_dir($cloneDir)) { + throw new GitCloneNotOwnedException("The clone folder $cloneDir is missing. (Is the array started?)"); + } + $gitDir = $cloneDir . '/.git'; + if (@readlink($gitDir) !== false || !is_dir($gitDir)) { + throw new GitCloneNotOwnedException("$cloneDir is not a git clone made by this plugin (no .git folder)."); + } + + $marker = @file_get_contents($gitDir . '/' . self::MARKER_FILE); + if ($marker === false || trim($marker) !== $this->settings->cloneId) { + throw new GitCloneNotOwnedException( + "$cloneDir was not made by this plugin for this stack (its marker is missing or names another stack), so it was not changed." + ); + } + + $topLevel = $this->git(['rev-parse', '--show-toplevel'], $cloneDir, 30); + if (!$topLevel->succeeded() || trim($topLevel->stdout) !== realpath($cloneDir)) { + throw new GitCloneNotOwnedException("$cloneDir is not the top of its own git clone."); + } + + $remote = $this->git(['config', '--get', 'remote.origin.url'], $cloneDir, 30); + if (!$remote->succeeded() || trim($remote->stdout) !== $this->settings->url) { + throw new GitCloneNotOwnedException( + "$cloneDir is a clone of a different repository than this stack's (" + . trim($remote->stdout) . '). Clone it again to change the repository.' + ); + } + } + + /** + * Fetch the stack's branch from the remote. Changes only .git, never the files. + * + * @return string the commit the branch now points at + * @throws RuntimeException naming the problem + */ + public function fetch(): string + { + $this->assertOwned(); + GitPathGuard::assertSafeToWrite($this->settings->cloneDir . '/.git'); + + $branch = $this->settings->branch; + $result = $this->git( + ['fetch', '--no-tags', '--prune', 'origin', '--', "+refs/heads/$branch:refs/remotes/origin/$branch"], + $this->settings->cloneDir, + self::FETCH_TIMEOUT_SECONDS + ); + if (!$result->succeeded()) { + throw new RuntimeException('Could not fetch from the repository: ' . $result->errorSummary()); + } + return $this->resolveCommit('refs/remotes/origin/' . $branch); + } + + /** + * The commit the files are checked out at. + * + * @throws RuntimeException if it cannot be read + */ + public function checkedOutCommit(): string + { + $this->assertOwned(); + return $this->resolveCommit('HEAD'); + } + + /** + * Tracked files that differ from the checked-out commit, as paths relative to the clone. + * + * Untracked files (including data a container wrote into the clone) are + * not local changes. + * + * @return string[] + * @throws RuntimeException if git status fails + */ + public function locallyChangedFiles(): array + { + $this->assertOwned(); + $result = $this->git( + ['status', '--porcelain=v1', '-z', '--untracked-files=no'], + $this->settings->cloneDir, + 60 + ); + if (!$result->succeeded()) { + throw new RuntimeException('Could not read the clone status: ' . $result->errorSummary()); + } + + $paths = []; + $entries = explode("\0", $result->stdout); + for ($i = 0; $i < count($entries); $i++) { + $entry = $entries[$i]; + if (strlen($entry) < 4) { + continue; + } + $paths[] = substr($entry, 3); + // A rename or copy is followed by its original path as a separate entry. + if ($entry[0] === 'R' || $entry[0] === 'C') { + $i++; + } + } + return $paths; + } + + /** + * Whether the stack's files would change between two commits: the folder + * holding the compose file, or anything at all when it is at the top level. + * + * @throws RuntimeException if either commit is unknown + */ + public function stackFilesChanged(string $fromCommit, string $toCommit): bool + { + $this->assertOwned(); + $args = ['diff', '--name-only', '-z', $this->commitArgument($fromCommit), $this->commitArgument($toCommit), '--']; + $stackFolder = $this->stackFolderInRepo(); + if ($stackFolder !== '') { + $args[] = $stackFolder; + } + $result = $this->git($args, $this->settings->cloneDir, 60); + if (!$result->succeeded()) { + throw new RuntimeException('Could not compare commits: ' . $result->errorSummary()); + } + return trim($result->stdout, "\0") !== ''; + } + + /** + * Whether the compose file exists at a commit. + * + * @throws RuntimeException if the commit is unknown + */ + public function composeFileExistsAt(string $commit): bool + { + $this->assertOwned(); + $result = $this->git( + ['ls-tree', '--name-only', '-z', $this->commitArgument($commit), '--', $this->settings->composePath], + $this->settings->cloneDir, + 60 + ); + if (!$result->succeeded()) { + throw new RuntimeException('Could not read the commit: ' . $result->errorSummary()); + } + return trim($result->stdout, "\0") === $this->settings->composePath; + } + + /** + * Check out a commit. Refuses when tracked files have local changes, and + * stops (changing nothing) if it would overwrite an untracked file, + * ignored or not. + * + * @throws RuntimeException naming the problem + */ + public function checkOut(string $commit): void + { + $this->assertOwned(); + $changed = $this->locallyChangedFiles(); + if ($changed !== []) { + throw new RuntimeException( + 'The clone has local changes (' . implode(', ', array_slice($changed, 0, 5)) + . (count($changed) > 5 ? ', ...' : '') . '). Save or discard them first.' + ); + } + GitPathGuard::assertSafeToWrite($this->settings->cloneDir); + + // A plain checkout (never sparse, never forced) refuses by itself to + // overwrite an untracked file, to replace one with a folder, or to + // write through an untracked symlink, and then changes nothing. + // Without --no-overwrite-ignore it silently overwrites ignored files + // (such as a data/ folder listed in .gitignore), so that is always given. + $result = $this->git( + ['checkout', '--detach', '--no-overwrite-ignore', $this->commitArgument($commit)], + $this->settings->cloneDir + ); + if ($result->succeeded()) { + return; + } + $inTheWay = self::untrackedPathsGitRefusedToOverwrite($result->stderr); + if ($inTheWay !== []) { + throw new RuntimeException( + 'The new commit adds files where untracked local files already are, so nothing was changed: ' + . implode(', ', array_slice($inTheWay, 0, 5)) . (count($inTheWay) > 5 ? ', ...' : '') + . '. Move them out of the way, then deploy again.' + ); + } + throw new RuntimeException("Could not check out $commit: " . $result->errorSummary()); + } + + /** + * The paths git lists when it refuses a checkout over untracked files. + * + * git prints "error: The following untracked working tree files would be + * overwritten by checkout:" (or "removed"), or, for a folder the new commit + * replaces with a file, "error: Updating the following directories would + * lose untracked files in them:", then one tab-indented path per line. + * + * @return string[] + */ + private static function untrackedPathsGitRefusedToOverwrite(string $stderr): array + { + $paths = []; + $inList = false; + foreach (explode("\n", $stderr) as $line) { + if (str_contains($line, 'untracked working tree files would be') + || str_contains($line, 'would lose untracked files in them')) { + $inList = true; + continue; + } + if ($inList && str_starts_with($line, "\t")) { + $paths[] = trim($line); + continue; + } + $inList = false; + } + return $paths; + } + + /** + * The folder in the repository that holds the compose file, or '' for the top level. + */ + public function stackFolderInRepo(): string + { + $folder = dirname($this->settings->composePath); + return $folder === '.' ? '' : $folder; + } + + /** + * Whether a string looks like a full commit id (SHA-1 or SHA-256). + */ + public static function isCommitId(string $value): bool + { + return preg_match('/^(?:[0-9a-f]{40}|[0-9a-f]{64})$/', $value) === 1; + } + + /** + * Resolve a reference to a full commit id. + * + * @throws RuntimeException if it does not name a commit + */ + private function resolveCommit(string $reference): string + { + $result = $this->git( + ['rev-parse', '--verify', '--quiet', '--end-of-options', $reference . '^{commit}'], + $this->settings->cloneDir, + 30 + ); + $commit = trim($result->stdout); + if (!$result->succeeded() || !self::isCommitId($commit)) { + throw new RuntimeException("Could not find commit $reference in the clone."); + } + return $commit; + } + + /** + * A commit id checked to be safe to pass to git as an argument. + * + * @throws InvalidArgumentException + */ + private function commitArgument(string $commit): string + { + if (!self::isCommitId($commit)) { + throw new InvalidArgumentException("Not a full commit id: $commit"); + } + return $commit; + } + + private function isLocalRepository(): bool + { + return str_starts_with($this->settings->url, '/'); + } + + /** + * Run git for this clone. + * + * A repository on this server (such as one a Gitea container keeps under + * appdata) is often owned by another user, and git refuses to read from + * it ("dubious ownership"). The stack's own repository path is trusted + * for the run, and nothing else (see GitCommand). + * + * @param string[] $args + */ + private function git(array $args, ?string $workingDirectory, int $timeoutSeconds = GitCommand::DEFAULT_TIMEOUT_SECONDS): ProcessResult + { + $trusted = $this->isLocalRepository() ? [$this->settings->url] : []; + return GitCommand::run($args, $workingDirectory, $timeoutSeconds, [], $trusted); + } + + /** + * @param string[] $args + * @throws RuntimeException + */ + private function runOrThrow(array $args, string $failureMessage): ProcessResult + { + $result = $this->git($args, $this->settings->cloneDir); + if (!$result->succeeded()) { + throw new RuntimeException("$failureMessage: " . $result->errorSummary()); + } + return $result; + } + + /** + * Remove the clone folder after a failed create(). + * + * Only called by create(), which checked the folder did not exist before + * it started. Deletes without following symlinks, and only a folder that + * is directly inside the clones root. + */ + private function removeFolderCreatedByThisRun(): void + { + $cloneDir = $this->settings->cloneDir; + if (!$this->exists()) { + return; + } + $parent = realpath(dirname($cloneDir)); + if ($parent === false || @readlink($cloneDir) !== false || dirname((string) realpath($cloneDir)) !== $parent) { + return; + } + self::removeTreeWithoutFollowingLinks($cloneDir); + } + + private static function removeTreeWithoutFollowingLinks(string $path): void + { + if (@readlink($path) !== false || !is_dir($path)) { + @unlink($path); + return; + } + foreach (scandir($path) ?: [] as $entry) { + if ($entry !== '.' && $entry !== '..') { + self::removeTreeWithoutFollowingLinks($path . '/' . $entry); + } + } + @rmdir($path); + } +} diff --git a/tests/unit/GitCloneTest.php b/tests/unit/GitCloneTest.php new file mode 100644 index 00000000..60c07f52 --- /dev/null +++ b/tests/unit/GitCloneTest.php @@ -0,0 +1,705 @@ +author is a + * normal clone of it used to make and push commits; clones made by the plugin + * go under $mnt/user/appdata/git. + */ +final class GitCloneTest extends TestCase +{ + private string $mnt; + private string $upstream; + private string $author; + private string $clonesRoot; + private string $stackDir; + + protected function setUp(): void + { + parent::setUp(); + $this->mnt = COMPOSE_GIT_MNT_DIR; + $this->removeTree($this->mnt); + mkdir($this->mnt . '/user/appdata', 0755, true); + mkdir($this->mnt . '/user/repos', 0755, true); + $this->setArrayState('STARTED'); + file_put_contents(COMPOSE_MOUNTS_FILE, "rootfs {$this->mnt} rootfs rw 0 0\nshfs {$this->mnt}/user fuse.shfs rw 0 0\n"); + + $this->upstream = $this->mnt . '/user/repos/upstream.git'; + $this->author = sys_get_temp_dir() . '/compose_git_clone_author'; + $this->clonesRoot = $this->mnt . '/user/appdata/git'; + $this->stackDir = sys_get_temp_dir() . '/compose_git_clone_stack'; + $this->removeTree($this->author); + $this->removeTree($this->stackDir); + mkdir($this->stackDir); + + $this->git(['init', '-q', '--bare', '-b', 'main', $this->upstream], null); + $this->git(['clone', '-q', $this->upstream, $this->author], null); + $this->git(['checkout', '-q', '-b', 'main'], $this->author); + $this->writeAndPush([ + 'whoami/compose.yaml' => "services:\n whoami:\n image: traefik/whoami\n", + 'other/compose.yaml' => "services:\n other:\n image: busybox\n", + 'README.md' => "stacks\n", + ], 'first'); + } + + protected function tearDown(): void + { + $this->removeTree($this->mnt); + $this->removeTree($this->author); + $this->removeTree($this->stackDir); + @unlink(COMPOSE_UNRAID_VAR_INI); + @unlink(COMPOSE_MOUNTS_FILE); + parent::tearDown(); + } + + // ----- creating a clone ----- + + public function testCreateChecksOutTheBranch(): void + { + $clone = $this->makeClone('whoami/compose.yaml'); + $commit = $clone->create(); + + $dir = $clone->settings()->cloneDir; + $this->assertSame($this->upstreamHead(), $commit); + $this->assertFileExists($dir . '/whoami/compose.yaml'); + $this->assertFileExists($dir . '/other/compose.yaml'); + $this->assertSame($clone->settings()->cloneId . "\n", file_get_contents($dir . '/.git/' . GitClone::MARKER_FILE)); + $clone->assertOwned(); + } + + public function testRepositoryAndCloneOwnedByAnotherUserStillWork(): void + { + if (posix_geteuid() !== 0) { + $this->markTestSkipped('Needs root to change file owners.'); + } + // A repository kept by a container, and New Permissions run over the clones. + $head = $this->upstreamHead(); + $this->chownTree($this->upstream, 65534); + $clone = $this->makeClone('whoami/compose.yaml'); + $this->assertSame($head, $clone->create()); + $this->chownTree($clone->settings()->cloneDir, 65534); + + $clone->assertOwned(); + $this->assertSame($head, $clone->fetch()); + $this->assertSame([], $clone->locallyChangedFiles()); + } + + public function testCreateRefusesAnExistingFolderAndLeavesItAlone(): void + { + $clone = $this->makeClone('whoami/compose.yaml'); + $dir = $clone->settings()->cloneDir; + mkdir($dir, 0755, true); + file_put_contents($dir . '/precious.txt', 'keep me'); + + try { + $clone->create(); + $this->fail('Cloned into an existing folder'); + } catch (RuntimeException $error) { + $this->assertStringContainsString('already exists', $error->getMessage()); + } + $this->assertSame('keep me', file_get_contents($dir . '/precious.txt')); + $this->assertSame(['.', '..', 'precious.txt'], scandir($dir)); + } + + public function testCreateRefusesAnExistingEmptyFolder(): void + { + $clone = $this->makeClone('whoami/compose.yaml'); + mkdir($clone->settings()->cloneDir, 0755, true); + $this->expectException(RuntimeException::class); + $clone->create(); + } + + public function testMissingBranchLeavesNothingBehind(): void + { + $clone = $this->makeClone('whoami/compose.yaml', 'mian'); + try { + $clone->create(); + $this->fail('Cloned a branch that does not exist'); + } catch (RuntimeException) { + $this->assertFalse($clone->exists()); + $this->assertDirectoryExists($this->clonesRoot); + } + } + + public function testMissingRepositoryLeavesNothingBehind(): void + { + $settings = GitStackSettings::createNew($this->mnt . '/user/repos/typo.git', 'main', 'whoami/compose.yaml', $this->clonesRoot, 'whoami'); + $clone = new GitClone($settings); + try { + $clone->create(); + $this->fail('Cloned a repository that does not exist'); + } catch (RuntimeException) { + $this->assertFalse($clone->exists()); + } + } + + public function testEmptyRepositoryIsRefused(): void + { + $empty = $this->mnt . '/user/repos/empty.git'; + $this->git(['init', '-q', '--bare', '-b', 'main', $empty], null); + $clone = new GitClone(GitStackSettings::createNew($empty, 'main', 'compose.yaml', $this->clonesRoot, 'x')); + + $this->expectException(RuntimeException::class); + $clone->create(); + } + + public function testNothingIsClonedWhileTheArrayIsStopped(): void + { + $clone = $this->makeClone('whoami/compose.yaml'); + $this->setArrayState('STOPPED'); + try { + $clone->create(); + $this->fail('Cloned on a stopped array'); + } catch (RuntimeException) { + $this->assertDirectoryDoesNotExist($this->clonesRoot); + } + } + + public function testSubmodulesAreNotFetched(): void + { + $sub = $this->mnt . '/user/repos/sub.git'; + $this->git(['init', '-q', '--bare', '-b', 'main', $sub], null); + $this->git( + ['-c', 'protocol.file.allow=always', 'submodule', 'add', '-q', $this->upstream, 'whoami/vendored'], + $this->author + ); + $this->commitAndPush('add submodule'); + + $clone = $this->makeClone('whoami/compose.yaml'); + $clone->create(); + + $this->assertSame(['.', '..'], scandir($clone->settings()->cloneDir . '/whoami/vendored')); + } + + // ----- ownership ----- + + public function testFolderWithoutMarkerIsNotOwned(): void + { + $clone = $this->createdClone(); + unlink($clone->settings()->cloneDir . '/.git/' . GitClone::MARKER_FILE); + $this->expectException(GitCloneNotOwnedException::class); + $clone->assertOwned(); + } + + public function testFolderMarkedForAnotherStackIsNotOwned(): void + { + $clone = $this->createdClone(); + file_put_contents($clone->settings()->cloneDir . '/.git/' . GitClone::MARKER_FILE, "ffffffffffffffff\n"); + $this->expectException(GitCloneNotOwnedException::class); + $clone->assertOwned(); + } + + public function testSymlinkInPlaceOfTheCloneIsNotOwned(): void + { + $clone = $this->createdClone(); + $dir = $clone->settings()->cloneDir; + rename($dir, $dir . '-real'); + symlink($dir . '-real', $dir); + $this->expectException(GitCloneNotOwnedException::class); + $clone->assertOwned(); + } + + public function testCloneOfADifferentRepositoryIsNotOwned(): void + { + $clone = $this->createdClone(); + $this->git(['remote', 'set-url', 'origin', $this->mnt . '/user/repos/other.git'], $clone->settings()->cloneDir); + $this->expectException(GitCloneNotOwnedException::class); + $this->expectExceptionMessage('different repository'); + $clone->assertOwned(); + } + + public function testGitFilePointingElsewhereIsNotOwned(): void + { + $clone = $this->createdClone(); + $dir = $clone->settings()->cloneDir; + rename($dir . '/.git', $this->mnt . '/user/appdata/moved-git'); + file_put_contents($dir . '/.git', 'gitdir: ' . $this->mnt . "/user/appdata/moved-git\n"); + $this->expectException(GitCloneNotOwnedException::class); + $clone->assertOwned(); + } + + public function testMissingCloneIsReportedAsMissing(): void + { + $clone = $this->makeClone('whoami/compose.yaml'); + $this->expectException(GitCloneNotOwnedException::class); + $this->expectExceptionMessage('missing'); + $clone->assertOwned(); + } + + // ----- fetching and checking out ----- + + public function testFetchThenCheckOutMovesToTheNewCommit(): void + { + $clone = $this->createdClone(); + $this->writeAndPush(['whoami/compose.yaml' => "services:\n whoami:\n image: traefik/whoami:v2\n"], 'bump'); + + $newCommit = $clone->fetch(); + $this->assertSame($this->upstreamHead(), $newCommit); + $clone->checkOut($newCommit); + + $this->assertSame($newCommit, $clone->checkedOutCommit()); + $this->assertStringContainsString('v2', (string) file_get_contents($clone->settings()->cloneDir . '/whoami/compose.yaml')); + } + + public function testRemoteBranchCommitDoesNotChangeTheClone(): void + { + $clone = $this->createdClone(); + $before = $clone->checkedOutCommit(); + $this->writeAndPush(['whoami/new.txt' => "x\n"], 'more'); + + $this->assertSame($this->upstreamHead(), $clone->remoteBranchCommit()); + $this->assertSame($before, $clone->checkedOutCommit()); + } + + public function testForcePushUpstreamIsJustANewTarget(): void + { + $clone = $this->createdClone(); + $this->git(['-c', 'user.name=t', '-c', 'user.email=t@example.invalid', 'commit', '-q', '--amend', '-m', 'rewritten'], $this->author); + $this->git(['push', '-q', '--force', 'origin', 'main'], $this->author); + + $commit = $clone->fetch(); + $clone->checkOut($commit); + $this->assertSame($this->upstreamHead(), $clone->checkedOutCommit()); + } + + public function testBranchDeletedUpstreamFailsAndLeavesFilesAlone(): void + { + $clone = $this->createdClone(); + $before = $clone->checkedOutCommit(); + $this->git(['push', '-q', 'origin', 'HEAD:refs/heads/keep'], $this->author); + $this->git(['--git-dir=' . $this->upstream, 'symbolic-ref', 'HEAD', 'refs/heads/keep'], null); + $this->git(['push', '-q', 'origin', '--delete', 'main'], $this->author); + + try { + $clone->fetch(); + $this->fail('Fetched a deleted branch'); + } catch (RuntimeException) { + $this->assertSame($before, $clone->checkedOutCommit()); + $this->assertFileExists($clone->settings()->cloneDir . '/whoami/compose.yaml'); + } + } + + public function testUntrackedFilesSurviveACheckout(): void + { + $clone = $this->createdClone(); + $data = $clone->settings()->cloneDir . '/whoami/data/app.db'; + mkdir(dirname($data)); + file_put_contents($data, 'container data'); + $this->writeAndPush(['whoami/compose.yaml' => "services: {}\n"], 'change'); + + $clone->checkOut($clone->fetch()); + + $this->assertSame('container data', file_get_contents($data)); + $this->assertSame([], $clone->locallyChangedFiles()); + } + + public function testCheckoutThatWouldOverwriteAnUntrackedFileChangesNothing(): void + { + $clone = $this->createdClone(); + $before = $clone->checkedOutCommit(); + $local = $clone->settings()->cloneDir . '/whoami/config.yml'; + file_put_contents($local, 'my local config'); + $this->writeAndPush(['whoami/config.yml' => "from upstream\n"], 'add config'); + + try { + $clone->checkOut($clone->fetch()); + $this->fail('Overwrote an untracked file'); + } catch (RuntimeException $error) { + // git refuses, and the message names the file. + $this->assertStringContainsString('whoami/config.yml', $error->getMessage()); + $this->assertStringContainsString('Move them out of the way', $error->getMessage()); + } + $this->assertSame('my local config', file_get_contents($local)); + $this->assertSame($before, $clone->checkedOutCommit()); + } + + public function testCheckoutThatWouldPutAFileWhereAnUntrackedFolderIsChangesNothing(): void + { + $clone = $this->createdClone(); + $before = $clone->checkedOutCommit(); + $folder = $clone->settings()->cloneDir . '/whoami/data'; + mkdir($folder); + file_put_contents($folder . '/app.db', 'container data'); + $this->writeAndPush(['whoami/data' => "now a file upstream\n"], 'file where data folder is'); + + try { + $clone->checkOut($clone->fetch()); + $this->fail('Replaced an untracked folder'); + } catch (RuntimeException) { + $this->assertSame('container data', file_get_contents($folder . '/app.db')); + $this->assertSame($before, $clone->checkedOutCommit()); + } + } + + public function testCheckoutThatNeedsAFolderWhereAnUntrackedFileIsChangesNothing(): void + { + $clone = $this->createdClone(); + $before = $clone->checkedOutCommit(); + $file = $clone->settings()->cloneDir . '/whoami/conf'; + file_put_contents($file, 'local conf file'); + $this->writeAndPush(['whoami/conf/app.yml' => "upstream\n"], 'folder where conf file is'); + + try { + $clone->checkOut($clone->fetch()); + $this->fail('Replaced an untracked file with a folder'); + } catch (RuntimeException) { + $this->assertSame('local conf file', file_get_contents($file)); + $this->assertSame($before, $clone->checkedOutCommit()); + } + } + + public function testCheckoutThatWouldOverwriteAnIgnoredFileChangesNothing(): void + { + $this->writeAndPush(['.gitignore' => "whoami/data/\n"], 'ignore data'); + $clone = $this->createdClone(); + $before = $clone->checkedOutCommit(); + $data = $clone->settings()->cloneDir . '/whoami/data/settings.ini'; + mkdir(dirname($data)); + file_put_contents($data, 'live data written by the container'); + // The repository starts tracking a file under the ignored folder. + mkdir($this->author . '/whoami/data'); + file_put_contents($this->author . '/whoami/data/settings.ini', "template\n"); + $this->git(['add', '-f', '--', 'whoami/data/settings.ini'], $this->author); + $this->commitAndPush('track a template under data'); + + try { + $clone->checkOut($clone->fetch()); + $this->fail('Overwrote an ignored file'); + } catch (RuntimeException $error) { + $this->assertStringContainsString('whoami/data/settings.ini', $error->getMessage()); + $this->assertStringContainsString('Move them out of the way', $error->getMessage()); + } + $this->assertSame('live data written by the container', file_get_contents($data)); + $this->assertSame($before, $clone->checkedOutCommit()); + } + + public function testCheckoutThatDropsTheIgnoreRuleAndTracksTheFileChangesNothing(): void + { + $this->writeAndPush(['.gitignore' => "whoami/data/\n"], 'ignore data'); + $clone = $this->createdClone(); + $before = $clone->checkedOutCommit(); + $data = $clone->settings()->cloneDir . '/whoami/data/settings.ini'; + mkdir(dirname($data)); + file_put_contents($data, 'live data written by the container'); + $this->writeAndPush(['.gitignore' => "", 'whoami/data/settings.ini' => "template\n"], 'stop ignoring data'); + + try { + $clone->checkOut($clone->fetch()); + $this->fail('Overwrote a file that was ignored before the checkout'); + } catch (RuntimeException $error) { + $this->assertStringContainsString('whoami/data/settings.ini', $error->getMessage()); + } + $this->assertSame('live data written by the container', file_get_contents($data)); + $this->assertSame($before, $clone->checkedOutCommit()); + } + + public function testCheckoutThatNeedsAFolderWhereAnIgnoredFileIsChangesNothing(): void + { + $this->writeAndPush(['.gitignore' => "whoami/cache\n"], 'ignore cache'); + $clone = $this->createdClone(); + $before = $clone->checkedOutCommit(); + $file = $clone->settings()->cloneDir . '/whoami/cache'; + file_put_contents($file, 'ignored cache file'); + mkdir($this->author . '/whoami/cache'); + file_put_contents($this->author . '/whoami/cache/x', "upstream\n"); + $this->git(['add', '-f', '--', 'whoami/cache/x'], $this->author); + $this->commitAndPush('cache is a folder'); + + try { + $clone->checkOut($clone->fetch()); + $this->fail('Replaced an ignored file with a folder'); + } catch (RuntimeException $error) { + $this->assertStringContainsString('whoami/cache', $error->getMessage()); + } + $this->assertSame('ignored cache file', file_get_contents($file)); + $this->assertSame($before, $clone->checkedOutCommit()); + } + + public function testCheckoutThatTurnsAFolderHoldingIgnoredFilesIntoAFileChangesNothing(): void + { + $this->writeAndPush(['.gitignore' => "*.log\n", 'whoami/logs/README' => "logs go here\n"], 'logs folder'); + $clone = $this->createdClone(); + $before = $clone->checkedOutCommit(); + $log = $clone->settings()->cloneDir . '/whoami/logs/app.log'; + file_put_contents($log, 'ignored log'); + $this->git(['rm', '-q', '-r', 'whoami/logs'], $this->author); + $this->writeAndPush(['whoami/logs' => "now a file\n"], 'logs is a file'); + + try { + $clone->checkOut($clone->fetch()); + $this->fail('Removed a folder holding an ignored file'); + } catch (RuntimeException $error) { + // git's message for this case lists the folder. + $this->assertStringContainsString('whoami/logs', $error->getMessage()); + $this->assertStringContainsString('Move them out of the way', $error->getMessage()); + } + $this->assertSame('ignored log', file_get_contents($log)); + $this->assertSame($before, $clone->checkedOutCommit()); + } + + public function testTrackedFileBecomingASymlinkIsNotMistakenForAnUntrackedFile(): void + { + $clone = $this->createdClone(); + $this->git(['rm', '-q', 'whoami/compose.yaml'], $this->author); + @mkdir($this->author . '/whoami'); + symlink('../other/compose.yaml', $this->author . '/whoami/compose.yaml'); + $this->git(['add', 'whoami/compose.yaml'], $this->author); + $this->commitAndPush('compose file is now a symlink'); + + $clone->checkOut($clone->fetch()); + + $this->assertNotFalse(readlink($clone->settings()->cloneDir . '/whoami/compose.yaml')); + } + + public function testTrackedFileReplacedByAFolderIsNotMistakenForAnUntrackedFile(): void + { + $this->writeAndPush(['whoami/conf' => "a file\n"], 'conf is a file'); + $clone = $this->createdClone(); + $this->git(['rm', '-q', 'whoami/conf'], $this->author); + $this->writeAndPush(['whoami/conf/app.yml' => "now a folder\n"], 'conf is a folder'); + + $clone->checkOut($clone->fetch()); + + $this->assertFileExists($clone->settings()->cloneDir . '/whoami/conf/app.yml'); + } + + public function testTrackedFolderReplacedByAFileIsNotMistakenForAnUntrackedFile(): void + { + $this->writeAndPush(['whoami/conf/app.yml' => "in a folder\n"], 'conf is a folder'); + $clone = $this->createdClone(); + $this->git(['rm', '-q', '-r', 'whoami/conf'], $this->author); + $this->writeAndPush(['whoami/conf' => "now a file\n"], 'conf is a file'); + + $clone->checkOut($clone->fetch()); + + $this->assertSame("now a file\n", file_get_contents($clone->settings()->cloneDir . '/whoami/conf')); + } + + public function testUntrackedSymlinkWhereTheCommitNeedsAFolderChangesNothing(): void + { + $clone = $this->createdClone(); + $before = $clone->checkedOutCommit(); + $elsewhere = $this->mnt . '/user/appdata/elsewhere'; + mkdir($elsewhere); + symlink($elsewhere, $clone->settings()->cloneDir . '/whoami/conf'); + $this->writeAndPush(['whoami/conf/app.yml' => "upstream\n"], 'add conf folder'); + + try { + $clone->checkOut($clone->fetch()); + $this->fail('Checked out through an untracked symlink'); + } catch (RuntimeException $error) { + $this->assertStringContainsString('whoami/conf', $error->getMessage()); + } + $this->assertFileDoesNotExist($elsewhere . '/app.yml'); + $this->assertSame($before, $clone->checkedOutCommit()); + } + + public function testUntrackedFileInAnotherStackFolderIsAlsoProtected(): void + { + $clone = $this->createdClone(); + $local = $clone->settings()->cloneDir . '/other/notes.txt'; + file_put_contents($local, 'mine'); + $this->writeAndPush(['other/notes.txt' => "theirs\n"], 'other stack adds notes'); + + try { + $clone->checkOut($clone->fetch()); + $this->fail('Overwrote an untracked file outside the stack folder'); + } catch (RuntimeException) { + $this->assertSame('mine', file_get_contents($local)); + } + } + + public function testCheckoutIsRefusedWithLocalChanges(): void + { + $clone = $this->createdClone(); + file_put_contents($clone->settings()->cloneDir . '/whoami/compose.yaml', "edited\n"); + $this->writeAndPush(['whoami/compose.yaml' => "upstream\n"], 'upstream edit'); + $commit = $clone->fetch(); + + try { + $clone->checkOut($commit); + $this->fail('Checked out over local changes'); + } catch (RuntimeException $error) { + $this->assertStringContainsString('local changes', $error->getMessage()); + } + $this->assertSame("edited\n", file_get_contents($clone->settings()->cloneDir . '/whoami/compose.yaml')); + } + + public function testStaleIndexLockIsReportedAndLeftInPlace(): void + { + $clone = $this->createdClone(); + $lock = $clone->settings()->cloneDir . '/.git/index.lock'; + file_put_contents($lock, ''); + $this->writeAndPush(['whoami/compose.yaml' => "services: {}\n"], 'change'); + + try { + $clone->checkOut($clone->fetch()); + $this->fail('Ignored a stale index.lock'); + } catch (RuntimeException $error) { + $this->assertStringContainsString('index.lock', $error->getMessage()); + } + $this->assertFileExists($lock); + } + + // ----- local changes ----- + + public function testLocalEditIsListedAndUntrackedFilesAreNot(): void + { + $clone = $this->createdClone(); + $dir = $clone->settings()->cloneDir; + file_put_contents($dir . '/whoami/compose.yaml', "edited\n"); + file_put_contents($dir . '/whoami/untracked.txt', "x\n"); + + $this->assertSame(['whoami/compose.yaml'], $clone->locallyChangedFiles()); + } + + public function testOddFileNamesAreListedExactly(): void + { + $names = ['whoami/-rf now.yaml', "whoami/new\nline.txt", 'whoami/caf' . "\u{e9}" . '.txt', 'whoami/*.yaml']; + $files = []; + foreach ($names as $name) { + $files[$name] = "original\n"; + } + $this->writeAndPush($files, 'odd names'); + $clone = $this->createdClone(); + foreach ($names as $name) { + file_put_contents($clone->settings()->cloneDir . '/' . $name, "edited\n"); + } + + $changed = $clone->locallyChangedFiles(); + sort($changed); + sort($names); + $this->assertSame($names, $changed); + } + + // ----- what changed ----- + + public function testChangeInAnotherStackFolderDoesNotCount(): void + { + $clone = $this->createdClone(); + $before = $clone->checkedOutCommit(); + $this->writeAndPush(['other/compose.yaml' => "services: {}\n", 'README.md' => "changed\n"], 'other stack'); + + $this->assertFalse($clone->stackFilesChanged($before, $clone->fetch())); + } + + public function testChangeInTheStackFolderCounts(): void + { + $clone = $this->createdClone(); + $before = $clone->checkedOutCommit(); + $this->writeAndPush(['whoami/extra.env' => "A=1\n"], 'this stack'); + + $this->assertTrue($clone->stackFilesChanged($before, $clone->fetch())); + } + + public function testComposeFileRemovedUpstreamIsDetected(): void + { + $clone = $this->createdClone(); + $this->git(['rm', '-q', 'whoami/compose.yaml'], $this->author); + $this->commitAndPush('remove whoami'); + + $this->assertFalse($clone->composeFileExistsAt($clone->fetch())); + $this->assertTrue($clone->composeFileExistsAt($clone->checkedOutCommit())); + } + + public function testWindowsLineEndingsFromGitattributesDoNotCountAsLocalChanges(): void + { + $this->writeAndPush([ + '.gitattributes' => "* text=auto eol=crlf\n", + 'whoami/compose.yaml' => "services:\n whoami:\n image: traefik/whoami\n", + ], 'crlf'); + $clone = $this->createdClone(); + + $this->assertSame([], $clone->locallyChangedFiles()); + $this->assertStringContainsString("\r\n", (string) file_get_contents($clone->settings()->cloneDir . '/whoami/compose.yaml')); + } + + // ----- helpers ----- + + private function makeClone(string $composePath, string $branch = 'main'): GitClone + { + return new GitClone(GitStackSettings::createNew($this->upstream, $branch, $composePath, $this->clonesRoot, 'whoami')); + } + + private function createdClone(): GitClone + { + $clone = $this->makeClone('whoami/compose.yaml'); + $clone->create(); + return $clone; + } + + /** @param array $files */ + private function writeAndPush(array $files, string $message): void + { + foreach ($files as $path => $content) { + $full = $this->author . '/' . $path; + if (!is_dir(dirname($full))) { + mkdir(dirname($full), 0755, true); + } + file_put_contents($full, $content); + $this->git(['add', '--', $path], $this->author); + } + $this->commitAndPush($message); + } + + private function commitAndPush(string $message): void + { + $this->git(['-c', 'user.name=t', '-c', 'user.email=t@example.invalid', 'commit', '-q', '-m', $message], $this->author); + $this->git(['push', '-q', 'origin', 'main'], $this->author); + } + + private function upstreamHead(): string + { + return trim(GitCommand::run(['--git-dir=' . $this->upstream, 'rev-parse', 'refs/heads/main'])->stdout); + } + + /** @param string[] $args */ + private function git(array $args, ?string $dir): void + { + $result = GitCommand::run($args, $dir); + $this->assertTrue($result->succeeded(), 'git ' . implode(' ', $args) . ': ' . $result->stderr); + } + + private function setArrayState(string $state): void + { + file_put_contents(COMPOSE_UNRAID_VAR_INI, "mdState=\"$state\"\n"); + } + + private function chownTree(string $path, int $uid): void + { + $result = \ProcessRunner::run(['chown', '-R', (string) $uid, $path], '/', ['PATH' => '/usr/bin:/bin'], 30); + $this->assertTrue($result->succeeded(), $result->stderr); + } + + private function removeTree(string $path): void + { + // readlink(), not is_link(): the test stream wrapper hides symlinks from is_link(). + if (@readlink($path) !== false || is_file($path)) { + unlink($path); + return; + } + if (!is_dir($path)) { + return; + } + foreach (scandir($path) ?: [] as $entry) { + if ($entry !== '.' && $entry !== '..') { + $this->removeTree($path . '/' . $entry); + } + } + rmdir($path); + } +} From df71b4753ae87ee15bb12819bdba23348b8d830f Mon Sep 17 00:00:00 2001 From: Grant Emsley Date: Sun, 4 Oct 2026 16:10:30 -0400 Subject: [PATCH 005/109] feat(git): save a git stack's local changes as a patch before discarding them This is the only code that discards the user's own changes, so it is a step of its own. Local changes in the clone (edits, and commits made by hand) can be saved as a patch in the stack folder, and are discarded only after that. The patch is the only copy once they are discarded, so a short write (a full flash) stops the discard. Discarding is refused, before anything is saved, when an untracked or ignored file sits at a path tracked at the checked-out commit (a tracked file replaced by a folder of data, or a file removed from the index and then edited): "git reset --hard" would delete or overwrite it, and the patch would not hold it. Staged changes are unstaged first, because a staged new file (for example after "git add -A" swept up a container's ./data folder) is not listed as untracked, and the reset would delete it. isOnBranch() and lastCommitSharedWithBranch() tell a commit made by hand in the clone apart from a newer commit of the branch, so the patch can be taken against the right base. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_011iUWDG8wTp8MhsFDqNC4NL --- source/compose.manager/include/GitClone.php | 204 +++++++++++++++++++- tests/unit/GitCloneTest.php | 131 +++++++++++++ 2 files changed, 334 insertions(+), 1 deletion(-) diff --git a/source/compose.manager/include/GitClone.php b/source/compose.manager/include/GitClone.php index 32546e52..1d2871c9 100644 --- a/source/compose.manager/include/GitClone.php +++ b/source/compose.manager/include/GitClone.php @@ -32,7 +32,7 @@ final class GitCloneNotOwnedException extends RuntimeException * - it never runs git clean and never forces a checkout, so untracked files, * ignored or not, such as data a container wrote through a relative bind * mount, are never removed or overwritten, - * - it refuses to check out while tracked files have local changes. + * - local changes are saved as a patch before they are ever discarded. */ final class GitClone { @@ -367,6 +367,160 @@ private static function untrackedPathsGitRefusedToOverwrite(string $stderr): arr return $paths; } + /** + * Save everything that differs from a commit (local edits, and any commits + * made by hand in the clone) as a patch file in the stack folder. + * + * @return string|null the patch file written, or null when there was nothing to save + * @throws RuntimeException if the patch cannot be written + */ + public function saveChangesAsPatch(string $againstCommit, string $stackDir): ?string + { + $this->assertOwned(); + $result = $this->git( + ['diff', '--binary', $this->commitArgument($againstCommit), '--'], + $this->settings->cloneDir, + 120 + ); + if (!$result->succeeded()) { + throw new RuntimeException('Could not collect the local changes: ' . $result->errorSummary()); + } + if ($result->stdout === '') { + return null; + } + + $patchDir = rtrim($stackDir, '/') . '/git-changes'; + if (str_starts_with($patchDir, rtrim(COMPOSE_GIT_MNT_DIR, '/') . '/')) { + GitPathGuard::assertSafeToWrite($patchDir); + } + if (!is_dir($patchDir) && !@mkdir($patchDir, 0755) && !is_dir($patchDir)) { + throw new RuntimeException("Could not create $patchDir."); + } + + $base = $patchDir . '/' . date('Y-m-d_His') . '-' . substr($againstCommit, 0, 12); + $patchFile = $base . '.patch'; + for ($n = 2; file_exists($patchFile); $n++) { + $patchFile = "$base-$n.patch"; + } + // The patch is the only copy of the changes once they are discarded, + // so a short write (a full flash) must stop the discard. + if (file_put_contents($patchFile, $result->stdout) !== strlen($result->stdout)) { + // This run created the file (the loop above picked an unused name). + @unlink($patchFile); + throw new RuntimeException("Could not write all of $patchFile, so the local changes were kept."); + } + return $patchFile; + } + + /** + * Take every staged change out of the index, leaving the files as they are. + * + * A new file that was staged (for example by "git add -A", which also + * sweeps up a container's ./data folder) is in the index, so git does not + * list it as untracked, and "git reset --hard" would delete it. Unstaged, + * it is an untracked file again: assertDiscardKeepsUntrackedFiles() + * protects it and the discard leaves it alone. Call this first, before + * the check and before saving the patch. + * + * @throws RuntimeException naming the problem + */ + public function unstageAll(): void + { + $this->assertOwned(); + GitPathGuard::assertSafeToWrite($this->settings->cloneDir . '/.git'); + $this->runOrThrow(['reset', '--quiet', 'HEAD'], 'Could not unstage the staged changes'); + } + + /** + * Put the tracked files back to the checked-out commit. Untracked files are + * left alone, and so are staged new files (see unstageAll()). Call only + * after saveChangesAsPatch() has saved the changes. + * + * @throws RuntimeException naming the problem + */ + public function discardLocalChanges(): void + { + $this->assertOwned(); + $this->unstageAll(); + $this->assertDiscardKeepsUntrackedFiles(); + GitPathGuard::assertSafeToWrite($this->settings->cloneDir); + $this->runOrThrow(['reset', '--hard', '--quiet', 'HEAD'], 'Could not discard the local changes'); + } + + /** + * Throw if discarding the local changes would delete or overwrite an untracked file. + * + * "git reset --hard HEAD" puts back every file tracked at HEAD, and it + * replaces whatever untracked file or folder is at that path: a tracked + * file "logs" that was deleted and replaced by a folder of logs, or the + * other way round, a tracked folder "logs" that was deleted and replaced + * by a file. The patch from saveChangesAsPatch() does not hold those + * untracked contents, so they would be lost. Call this before saving the + * patch, so a refusal changes nothing. + * + * @throws RuntimeException listing the files in the way + */ + public function assertDiscardKeepsUntrackedFiles(): void + { + $tracked = $this->git(['ls-tree', '-r', '--name-only', '-z', 'HEAD'], $this->settings->cloneDir, 60); + // No --exclude-standard: ignored files are listed too, and are just as much at risk. + $untracked = $this->git(['ls-files', '-z', '--others'], $this->settings->cloneDir, 120); + if (!$tracked->succeeded() || !$untracked->succeeded()) { + throw new RuntimeException( + 'Could not list the files in the clone: ' + . (!$tracked->succeeded() ? $tracked->errorSummary() : $untracked->errorSummary()) + ); + } + + $trackedFiles = []; + // Every folder that holds a tracked file, at any depth. + $trackedFolders = []; + foreach (explode("\0", $tracked->stdout) as $path) { + if ($path === '') { + continue; + } + $trackedFiles[$path] = true; + $folder = dirname($path); + while ($folder !== '.' && !isset($trackedFolders[$folder])) { + $trackedFolders[$folder] = true; + $folder = dirname($folder); + } + } + + $atRisk = []; + foreach (explode("\0", $untracked->stdout) as $path) { + if ($path === '') { + continue; + } + // At risk: an untracked file where a tracked folder is. The reset + // would delete the file to put the folder back. + if (isset($trackedFolders[$path])) { + $atRisk[$path] = true; + continue; + } + // At risk: an untracked file at a tracked path, or inside a folder + // that sits where a tracked file is. + $candidate = $path; + while ($candidate !== '.' && $candidate !== '') { + if (isset($trackedFiles[$candidate])) { + $atRisk[$candidate] = true; + break; + } + $candidate = dirname($candidate); + } + } + + if ($atRisk !== []) { + $paths = array_keys($atRisk); + throw new RuntimeException( + 'Discarding the local changes would delete or overwrite untracked files at ' + . implode(', ', array_slice($paths, 0, 5)) . (count($paths) > 5 ? ', ...' : '') + . ', because the checked-out commit has a tracked file or folder there. Nothing was changed. ' + . 'Move them out of the clone, then deploy again.' + ); + } + } + /** * The folder in the repository that holds the compose file, or '' for the top level. */ @@ -376,6 +530,54 @@ public function stackFolderInRepo(): string return $folder === '.' ? '' : $folder; } + /** + * Whether a commit is on the stack's branch as last fetched (it, or a later + * commit of the branch, is what the branch points at). + * + * @throws RuntimeException if git cannot tell + */ + public function isOnBranch(string $commit): bool + { + $this->assertOwned(); + $result = $this->git( + ['merge-base', '--is-ancestor', $this->commitArgument($commit), 'refs/remotes/origin/' . $this->settings->branch], + $this->settings->cloneDir, + 60 + ); + // Exit code 1 means "not an ancestor"; anything else non-zero is an error. + if ($result->exitCode === 1 && !$result->timedOut) { + return false; + } + if (!$result->succeeded()) { + throw new RuntimeException('Could not compare the commit with the branch: ' . $result->errorSummary()); + } + return true; + } + + /** + * The last commit a commit shares with the stack's branch as last fetched, + * or null when they share none. + * + * @throws RuntimeException if git fails + */ + public function lastCommitSharedWithBranch(string $commit): ?string + { + $this->assertOwned(); + $result = $this->git( + ['merge-base', $this->commitArgument($commit), 'refs/remotes/origin/' . $this->settings->branch], + $this->settings->cloneDir, + 60 + ); + if ($result->exitCode === 1 && !$result->timedOut) { + return null; + } + $shared = trim($result->stdout); + if (!$result->succeeded() || !self::isCommitId($shared)) { + throw new RuntimeException('Could not compare the commit with the branch: ' . $result->errorSummary()); + } + return $shared; + } + /** * Whether a string looks like a full commit id (SHA-1 or SHA-256). */ diff --git a/tests/unit/GitCloneTest.php b/tests/unit/GitCloneTest.php index 60c07f52..dc0fe2b8 100644 --- a/tests/unit/GitCloneTest.php +++ b/tests/unit/GitCloneTest.php @@ -586,6 +586,137 @@ public function testOddFileNamesAreListedExactly(): void $this->assertSame($names, $changed); } + public function testLocalChangesAreSavedAsAPatchThenDiscarded(): void + { + $clone = $this->createdClone(); + $dir = $clone->settings()->cloneDir; + $deployed = $clone->checkedOutCommit(); + file_put_contents($dir . '/whoami/compose.yaml', "hotfix\n"); + file_put_contents($dir . '/whoami/untracked.txt', "keep\n"); + + $patch = $clone->saveChangesAsPatch($deployed, $this->stackDir); + $clone->discardLocalChanges(); + + $this->assertNotNull($patch); + $this->assertStringContainsString('+hotfix', (string) file_get_contents($patch)); + $this->assertSame([], $clone->locallyChangedFiles()); + $this->assertSame("keep\n", file_get_contents($dir . '/whoami/untracked.txt')); + + // The patch puts the change back. + $this->git(['apply', $patch], $dir); + $this->assertSame("hotfix\n", file_get_contents($dir . '/whoami/compose.yaml')); + } + + public function testDiscardRefusesWhenAFolderOfDataReplacedATrackedFile(): void + { + $this->writeAndPush(['whoami/logs' => "placeholder\n"], 'placeholder'); + $clone = $this->createdClone(); + $dir = $clone->settings()->cloneDir; + unlink($dir . '/whoami/logs'); + mkdir($dir . '/whoami/logs'); + file_put_contents($dir . '/whoami/logs/app.log', "data\n"); + + try { + $clone->discardLocalChanges(); + $this->fail('Discarded over an untracked folder of data'); + } catch (RuntimeException $error) { + $this->assertStringContainsString('whoami/logs', $error->getMessage()); + } + $this->assertSame("data\n", file_get_contents($dir . '/whoami/logs/app.log')); + } + + public function testDiscardRefusesWhenAFileOfDataReplacedATrackedFolder(): void + { + $this->writeAndPush(['whoami/logs/app.log' => "placeholder\n"], 'placeholder'); + $clone = $this->createdClone(); + $dir = $clone->settings()->cloneDir; + unlink($dir . '/whoami/logs/app.log'); + rmdir($dir . '/whoami/logs'); + file_put_contents($dir . '/whoami/logs', "data\n"); + + try { + $clone->discardLocalChanges(); + $this->fail('Discarded over an untracked file of data'); + } catch (RuntimeException $error) { + $this->assertStringContainsString('whoami/logs', $error->getMessage()); + } + $this->assertSame("data\n", file_get_contents($dir . '/whoami/logs')); + } + + public function testEditToAFileRemovedFromTheIndexIsSavedInThePatch(): void + { + $clone = $this->createdClone(); + $dir = $clone->settings()->cloneDir; + $deployed = $clone->checkedOutCommit(); + $this->git(['rm', '-q', '--cached', 'whoami/compose.yaml'], $dir); + file_put_contents($dir . '/whoami/compose.yaml', "edited\n"); + + // Unstaging tracks the file again, so the edit is an ordinary local change. + $clone->unstageAll(); + $patch = $clone->saveChangesAsPatch($deployed, $this->stackDir); + $clone->discardLocalChanges(); + + $this->assertStringContainsString('+edited', (string) file_get_contents((string) $patch)); + $this->git(['apply', (string) $patch], $dir); + $this->assertSame("edited\n", file_get_contents($dir . '/whoami/compose.yaml')); + } + + public function testStagedNewFileSurvivesADiscard(): void + { + $clone = $this->createdClone(); + $dir = $clone->settings()->cloneDir; + $deployed = $clone->checkedOutCommit(); + mkdir($dir . '/whoami/data'); + file_put_contents($dir . '/whoami/data/app.db', 'container data'); + file_put_contents($dir . '/whoami/compose.yaml', "hotfix\n"); + // "git add -A" after a hotfix also stages the container's data. + $this->git(['add', '-A'], $dir); + + $clone->unstageAll(); + $patch = $clone->saveChangesAsPatch($deployed, $this->stackDir); + $clone->discardLocalChanges(); + + $this->assertSame('container data', file_get_contents($dir . '/whoami/data/app.db')); + $this->assertSame([], $clone->locallyChangedFiles()); + // The data is not in the patch: it stayed where it was. + $this->assertStringNotContainsString('app.db', (string) file_get_contents((string) $patch)); + $this->assertStringContainsString('+hotfix', (string) file_get_contents((string) $patch)); + } + + public function testCommitMadeByHandInTheCloneIsInThePatch(): void + { + $clone = $this->createdClone(); + $dir = $clone->settings()->cloneDir; + $deployed = $clone->checkedOutCommit(); + file_put_contents($dir . '/whoami/compose.yaml', "committed by hand\n"); + $this->git(['-c', 'user.name=t', '-c', 'user.email=t@example.invalid', 'commit', '-q', '-am', 'by hand'], $dir); + + $patch = $clone->saveChangesAsPatch($deployed, $this->stackDir); + + $this->assertNotNull($patch); + $this->assertStringContainsString('+committed by hand', (string) file_get_contents($patch)); + } + + public function testNothingToSaveGivesNoPatch(): void + { + $clone = $this->createdClone(); + $this->assertNull($clone->saveChangesAsPatch($clone->checkedOutCommit(), $this->stackDir)); + $this->assertDirectoryDoesNotExist($this->stackDir . '/git-changes'); + } + + public function testTwoPatchesInTheSameSecondDoNotOverwriteEachOther(): void + { + $clone = $this->createdClone(); + $deployed = $clone->checkedOutCommit(); + file_put_contents($clone->settings()->cloneDir . '/whoami/compose.yaml', "one\n"); + $first = $clone->saveChangesAsPatch($deployed, $this->stackDir); + $second = $clone->saveChangesAsPatch($deployed, $this->stackDir); + + $this->assertNotSame($first, $second); + $this->assertFileExists((string) $first); + $this->assertFileExists((string) $second); + } + // ----- what changed ----- public function testChangeInAnotherStackFolderDoesNotCount(): void From 20aba758cb16d3da13310d917d8ed9a58ba5041c Mon Sep 17 00:00:00 2001 From: Grant Emsley Date: Sat, 3 Oct 2026 14:33:04 -0400 Subject: [PATCH 006/109] feat(git): keep a git stack's managed override and .env out of its clone A git stack is an indirect stack whose target is a plugin-managed clone, so anything the plugin writes into that folder would show up as a local change and could be lost to the next pull. For stacks with git.json, StackInfo now: - reads, creates and saves the plugin-managed override in the stack folder; - applies both overrides when composing: the repository's own (if it has one), then the managed one, so a repository override no longer hides the plugin's labels. An override file that does not exist is left out; - uses the stack folder's .env if there is one, so secrets can stay out of the repository, and otherwise a .env committed next to the compose file, for people who keep their settings versioned with the stack. The editor creates a new .env in the stack folder; - never uses default compose file discovery, so its files are always named. Ordinary stacks behave exactly as before. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_011iUWDG8wTp8MhsFDqNC4NL --- source/compose.manager/include/Util.php | 116 ++++++++++++++++- tests/unit/GitStackInfoTest.php | 166 ++++++++++++++++++++++++ 2 files changed, 275 insertions(+), 7 deletions(-) create mode 100644 tests/unit/GitStackInfoTest.php diff --git a/source/compose.manager/include/Util.php b/source/compose.manager/include/Util.php index 828603f2..55058005 100644 --- a/source/compose.manager/include/Util.php +++ b/source/compose.manager/include/Util.php @@ -2619,6 +2619,50 @@ public function applyIdentityChoice(string $projectName): void $this->cachedContainerCounts = null; } + /** + * Whether this stack is deployed from a git repository (has git.json). + * + * A git stack is an indirect stack whose target is a clone the plugin + * manages. The plugin-managed override always lives in the stack folder, + * never in the clone, so pulling new commits never touches it. The .env + * does too unless the repository brings its own (see + * gitStackDefaultEnvPath()). + */ + public function isGitStack(): bool + { + return is_file($this->path . '/git.json'); + } + + /** + * The .env a git stack uses when no envpath is set. + * + * The stack folder's .env if there is one, so secrets can stay out of the + * repository. Otherwise a .env committed in the repository next to the + * compose file, for people who keep their settings versioned with the + * stack (editing that one changes the clone, which the next deploy reports + * as a local change). With neither, the stack folder's .env, which is where + * the editor creates it. + * + * Only one of the two is used. Overrides can apply both (see + * getComposeFilePaths()) because the plugin already passes compose a list + * of -f files. It passes exactly one env file everywhere (compose.sh's -e, + * autostart, auto-update's COMPOSE_ENV_FILE, the editor), so loading both, + * with the stack folder's values winning, would change all of those. That + * may come later, but was too big a change to make here. + */ + private function gitStackDefaultEnvPath(): string + { + $stackFolderEnv = $this->path . '/.env'; + if (is_file($stackFolderEnv)) { + return $stackFolderEnv; + } + $repositoryEnv = $this->composeSource . '/.env'; + if (is_file($repositoryEnv)) { + return $repositoryEnv; + } + return $stackFolderEnv; + } + /** * Get the stack description. * @return string @@ -2640,7 +2684,7 @@ public function getEnvFilePath(): ?string { $val = $this->readMetadata('envpath'); if ($val === null || $val === '') { - return null; + return $this->isGitStack() ? $this->gitStackDefaultEnvPath() : null; } if (Path::isAbsolutePath($val)) { return $val; @@ -2660,7 +2704,8 @@ public function getCredentialId(): ?string * Resolution order: * 1) `envpath` metadata (explicit stack setting) when it points to a * readable file - * 2) Local `.env` in compose source (direct or indirect) + * 2) Local `.env` in compose source (direct or indirect). A git stack + * uses gitStackDefaultEnvPath() here instead. * * If `envpath` exists but is empty, it is treated as unset. * If `envpath` exists but points to a missing file, it is treated as @@ -2675,7 +2720,7 @@ public function getEffectiveEnvFilePath(): ?string return $explicitEnvPath; } - $defaultEnvPath = $this->composeSource . '/.env'; + $defaultEnvPath = $this->isGitStack() ? $this->gitStackDefaultEnvPath() : $this->composeSource . '/.env'; return is_file($defaultEnvPath) ? $defaultEnvPath : null; } @@ -3038,9 +3083,24 @@ private function getComposeFilePaths(): array $mainComposeFile = $this->composeFilePath ?? ($this->composeSource . '/' . COMPOSE_FILE_NAMES[0]); $paths[] = $mainComposeFile; - $overridePath = $this->getOverridePath(); - if ($overridePath !== null) { - $paths[] = $overridePath; + if ($this->isGitStack()) { + // Git stacks apply both overrides: the repository's own (if it has + // one), then the plugin-managed one from the stack folder, so a + // repository override never hides the plugin's labels. Missing + // files are left out rather than passed to compose. + $repoOverride = $this->overrideInfo->indirectOverride; + if ($repoOverride !== null && is_file($repoOverride)) { + $paths[] = $repoOverride; + } + $managedOverride = $this->overrideInfo->getProjectOverridePath(); + if ($managedOverride !== null && is_file($managedOverride)) { + $paths[] = $managedOverride; + } + } else { + $overridePath = $this->getOverridePath(); + if ($overridePath !== null) { + $paths[] = $overridePath; + } } foreach ($this->getAdditionalComposeFilesFromEnv() as $extraFile) { @@ -3082,6 +3142,10 @@ private function getComposeFilePaths(): array */ public function useDefaultComposeFileDiscovery(): bool { + // A git stack always names its files: discovery could pick up files from the wrong folder. + if ($this->isGitStack()) { + return false; + } $raw = $this->readMetadata('use_default_compose_files'); if ($raw === null) { return false; @@ -3312,6 +3376,10 @@ private function isProfilesCacheStale(): bool return true; } + if ($this->isRepositoryOverrideNewerThan($cacheMtime)) { + return true; + } + $envFilePath = $this->getEffectiveEnvFilePath(); if ($envFilePath !== null && is_file($envFilePath) && filemtime($envFilePath) > $cacheMtime) { return true; @@ -3377,19 +3445,45 @@ public function getOverrideInfo(): OverrideInfo */ public function getOverridePath(): ?string { + if ($this->isGitStack()) { + // The repository's own override is applied too (see + // getComposeFilePaths), but the one the plugin reads and manages + // is in the stack folder. + return $this->overrideInfo->getProjectOverridePath(); + } return $this->overrideInfo->getOverridePath(); } + /** + * Whether a git stack's repository override changed after a cache was written. + * + * getOverridePath() returns the managed override for a git stack, so the + * cache staleness checks compare that one; the repository's own override + * is a compose input too (see getComposeFilePaths). Always false for other stacks. + */ + private function isRepositoryOverrideNewerThan(int|false $cacheMtime): bool + { + if (!$this->isGitStack()) { + return false; + } + $repoOverride = $this->overrideInfo->indirectOverride; + return $repoOverride !== null && is_file($repoOverride) && filemtime($repoOverride) > $cacheMtime; + } + /** * Get the path where a new override template should be created. * * Uses the indirect target when the stack is indirect, otherwise the - * project override path. + * project override path. A git stack always uses the project override. * * @return string|null */ public function getPreferredOverridePath(): ?string { + // Never create or save an override inside a git stack's clone. + if ($this->isGitStack()) { + return $this->overrideInfo->getProjectOverridePath(); + } if (!$this->isIndirect || $this->indirectPath === null || $this->indirectPath === '') { return $this->overrideInfo->getProjectOverridePath(); } @@ -3469,6 +3563,10 @@ private function isServicesCacheStale(): bool } } + if ($this->isRepositoryOverrideNewerThan($cacheMtime)) { + return true; + } + return false; } @@ -3732,6 +3830,10 @@ private function isHasBuildCacheStale(): bool } } + if ($this->isRepositoryOverrideNewerThan($cacheMtime)) { + return true; + } + return false; } diff --git a/tests/unit/GitStackInfoTest.php b/tests/unit/GitStackInfoTest.php new file mode 100644 index 00000000..d456519d --- /dev/null +++ b/tests/unit/GitStackInfoTest.php @@ -0,0 +1,166 @@ +root = $this->createTempDir(); + $this->stackDir = $this->root . '/whoami'; + $this->clone = $this->root . '/clones/whoami-0123abcd'; + mkdir($this->stackDir); + mkdir($this->clone . '/whoami', 0755, true); + file_put_contents($this->clone . '/whoami/compose.yaml', "services:\n whoami:\n image: traefik/whoami\n"); + file_put_contents($this->stackDir . '/indirect', $this->clone . '/whoami/compose.yaml'); + file_put_contents($this->stackDir . '/indirect_mode', 'file'); + file_put_contents($this->stackDir . '/git.json', '{}'); + } + + public function testStackWithGitJsonIsAGitStack(): void + { + $this->assertTrue($this->stack()->isGitStack()); + unlink($this->stackDir . '/git.json'); + \StackInfo::clearCache(); + $this->assertFalse($this->stack()->isGitStack()); + } + + // ----- .env ----- + + public function testStackFolderEnvWinsOverOneInTheRepository(): void + { + file_put_contents($this->clone . '/whoami/.env', "FROM_REPO=1\n"); + file_put_contents($this->stackDir . '/.env', "FROM_HOST=1\n"); + + $stack = $this->stack(); + + $this->assertSame($this->stackDir . '/.env', $stack->getEnvFilePath()); + $this->assertSame($this->stackDir . '/.env', $stack->getEffectiveEnvFilePath()); + $this->assertStringContainsString($this->stackDir . '/.env', $stack->buildComposeArgs()['envFile']); + } + + public function testEnvFileKeptInTheRepositoryIsUsedWhenTheStackFolderHasNone(): void + { + file_put_contents($this->clone . '/whoami/.env', "FROM_REPO=1\n"); + + $stack = $this->stack(); + + $this->assertSame($this->clone . '/whoami/.env', $stack->getEffectiveEnvFilePath()); + $this->assertSame($this->clone . '/whoami/.env', $stack->buildComposeArgs()['envFilePath']); + // The editor opens the one that is used. + $this->assertSame($this->clone . '/whoami/.env', $stack->getEnvFilePath()); + } + + public function testWithNoEnvFileTheEditorCreatesItInTheStackFolder(): void + { + $stack = $this->stack(); + + $this->assertNull($stack->getEffectiveEnvFilePath()); + $this->assertSame($this->stackDir . '/.env', $stack->getEnvFilePath()); + } + + public function testExplicitEnvPathStillWins(): void + { + $custom = $this->root . '/custom.env'; + file_put_contents($custom, "A=1\n"); + file_put_contents($this->stackDir . '/envpath', $custom); + + $this->assertSame($custom, $this->stack()->getEnvFilePath()); + } + + // ----- overrides ----- + + public function testRepositoryOverrideAndManagedOverrideAreBothApplied(): void + { + $repoOverride = $this->clone . '/whoami/compose.override.yaml'; + $managedOverride = $this->stackDir . '/compose.override.yaml'; + file_put_contents($repoOverride, "services: {}\n"); + file_put_contents($managedOverride, "services: {}\n"); + + $paths = $this->stack()->buildComposeArgs()['filePaths']; + + $this->assertSame([$this->clone . '/whoami/compose.yaml', $repoOverride, $managedOverride], $paths); + } + + public function testAChangedRepositoryOverrideMakesTheCachesStale(): void + { + $repoOverride = $this->clone . '/whoami/compose.override.yaml'; + file_put_contents($repoOverride, "services: {}\n"); + file_put_contents($this->stackDir . '/profiles', json_encode(['cached'])); + file_put_contents($this->stackDir . '/has_build', '1'); + $old = time() - 3600; + touch($this->clone . '/whoami/compose.yaml', $old); + touch($repoOverride, $old); + + // The caches are newer than every compose file, so they are used. + $this->assertSame(['cached'], $this->stack()->getProfiles()); + $this->assertTrue($this->stack()->hasBuildConfig()); + + // A commit that changes only the repository's override. + touch($repoOverride, time() + 60); + + // Read again from compose, which the test image does not have, so the cached answers are gone. + $this->assertNotSame(['cached'], $this->stack()->getProfiles()); + $this->assertFalse($this->stack()->hasBuildConfig()); + } + + public function testMissingOverridesAreLeftOut(): void + { + @unlink($this->stackDir . '/compose.override.yaml'); + + $paths = $this->stack()->buildComposeArgs()['filePaths']; + + $this->assertSame([$this->clone . '/whoami/compose.yaml'], $paths); + } + + public function testOverrideEditsGoToTheStackFolder(): void + { + file_put_contents($this->clone . '/whoami/compose.override.yaml', "services: {}\n"); + $stack = $this->stack(); + + $this->assertSame($this->stackDir . '/compose.override.yaml', $stack->getPreferredOverridePath()); + $this->assertSame($this->stackDir . '/compose.override.yaml', $stack->getOverridePath()); + } + + public function testOrdinaryIndirectStackKeepsItsOldOverrideRule(): void + { + unlink($this->stackDir . '/git.json'); + $repoOverride = $this->clone . '/whoami/compose.override.yaml'; + file_put_contents($repoOverride, "services: {}\n"); + file_put_contents($this->stackDir . '/compose.override.yaml', "services: {}\n"); + + $stack = $this->stack(); + + $this->assertSame([$this->clone . '/whoami/compose.yaml', $repoOverride], $stack->buildComposeArgs()['filePaths']); + $this->assertSame($repoOverride, $stack->getPreferredOverridePath()); + } + + // ----- project directory ----- + + public function testProjectDirectoryIsTheComposeFileFolderInTheClone(): void + { + $this->assertSame($this->clone . '/whoami', $this->stack()->buildComposeArgs()['projectDirectory']); + } + + private function stack(): \StackInfo + { + \StackInfo::clearCache(); + return \StackInfo::fromProject($this->root, 'whoami'); + } +} From 820e99c68dc568f654aba0c9ad1fb18f249baf33 Mon Sep 17 00:00:00 2001 From: Grant Emsley Date: Sat, 3 Oct 2026 14:33:04 -0400 Subject: [PATCH 007/109] feat(git): check a git stack before a deploy touches any container GitDeployCheck runs `docker compose config` with the stack's real files, env file and profiles, and works on the JSON it prints; the plugin never parses YAML itself. A deploy is stopped, naming each problem, when: - compose reports an error, the file defines no services, or a variable used by the compose file is not set (a blank value can create state that outlives fixing it); - the repository's .env.example lists keys missing from the .env in use (key names only, never values); - an external network or volume does not exist; - a bind-mount source is under /mnt but not on a mounted disk, pool or share (Docker would create it in RAM), its share does not exist (a typo, or it would quietly become a new share), or it is outside /mnt and does not exist. Missing folders below an existing share, such as a new stack's appdata/newapp/config, are left for Docker to create and named in the deploy log. Sources inside the clone (relative paths such as ./data) are allowed; - a config or secret file, or a local build folder, does not exist; - a container name or published port is already taken by another stack or a plain container. Docker is run through DockerCommand (no shell, time limit). The tests use a scripted stand-in for the docker CLI. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_011iUWDG8wTp8MhsFDqNC4NL --- source/compose.manager/include/Defines.php | 2 + .../compose.manager/include/DockerCommand.php | 38 ++ .../include/GitDeployCheck.php | 452 ++++++++++++++++++ tests/bootstrap.php | 2 + tests/unit/GitDeployCheckTest.php | 383 +++++++++++++++ tests/unit/fixtures/fake-docker.sh | 55 +++ tests/unit/support/FakeDocker.php | 115 +++++ 7 files changed, 1047 insertions(+) create mode 100644 source/compose.manager/include/DockerCommand.php create mode 100644 source/compose.manager/include/GitDeployCheck.php create mode 100644 tests/unit/GitDeployCheckTest.php create mode 100644 tests/unit/fixtures/fake-docker.sh create mode 100644 tests/unit/support/FakeDocker.php diff --git a/source/compose.manager/include/Defines.php b/source/compose.manager/include/Defines.php index a9697c6f..5eaa5381 100644 --- a/source/compose.manager/include/Defines.php +++ b/source/compose.manager/include/Defines.php @@ -43,6 +43,8 @@ function locate_compose_root($name) defined('COMPOSE_GIT_MNT_DIR') || define('COMPOSE_GIT_MNT_DIR', '/mnt'); defined('COMPOSE_GIT_HOME_DIR') || define('COMPOSE_GIT_HOME_DIR', '/var/tmp/compose-manager-git-home'); defined('COMPOSE_GIT_DEFAULT_CLONES_ROOT') || define('COMPOSE_GIT_DEFAULT_CLONES_ROOT', '/mnt/user/appdata/compose.manager/git'); +// The docker CLI (the git deploy checks run it; tests point this at a scripted stand-in). +defined('COMPOSE_DOCKER_BIN') || define('COMPOSE_DOCKER_BIN', 'docker'); /** * Reserved filename at the compose root level used by the plugin installer diff --git a/source/compose.manager/include/DockerCommand.php b/source/compose.manager/include/DockerCommand.php new file mode 100644 index 00000000..f19cb809 --- /dev/null +++ b/source/compose.manager/include/DockerCommand.php @@ -0,0 +1,38 @@ + '/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin', + 'HOME' => getenv('HOME') ?: '/root', + 'LC_ALL' => 'C', + ]; + // Keep the registry credential compose.sh selected for this run, if any. + $dockerConfig = getenv('DOCKER_CONFIG'); + if (is_string($dockerConfig) && $dockerConfig !== '') { + $environment['DOCKER_CONFIG'] = $dockerConfig; + } + + return ProcessRunner::run( + array_merge([COMPOSE_DOCKER_BIN], $args), + $workingDirectory ?? '/', + $environment, + $timeoutSeconds + ); + } +} diff --git a/source/compose.manager/include/GitDeployCheck.php b/source/compose.manager/include/GitDeployCheck.php new file mode 100644 index 00000000..bf5ac339 --- /dev/null +++ b/source/compose.manager/include/GitDeployCheck.php @@ -0,0 +1,452 @@ + Drivers of external networks, asked once per check. */ + private array $externalNetworkDrivers = []; + + /** + * @param string $projectName The stack's compose project name + * @param string[] $composeArgs The stack's -f, --env-file and --profile arguments + * @param string $cloneDir The stack's clone folder + * @param string $composeDir The folder of the compose file inside the clone + * @param string|null $envFile The .env the stack uses, if any + */ + public function __construct( + private readonly string $projectName, + private readonly array $composeArgs, + private readonly string $cloneDir, + private readonly string $composeDir, + private readonly ?string $envFile + ) { + } + + /** + * Run every check. + * + * @return string[] One message per problem found; empty when the stack can be deployed + */ + public function run(): array + { + $this->foldersDockerWillCreate = []; + $config = DockerCommand::run( + array_merge(['compose'], $this->composeArgs, ['-p', $this->projectName, 'config', '--format', 'json']), + $this->composeDir + ); + if (!$config->succeeded()) { + return ['The compose file is not valid: ' . trim($config->stderr !== '' ? $config->stderr : $config->errorSummary())]; + } + + $problems = $this->unsetVariables($config->stderr); + + $resolved = json_decode($config->stdout, true); + if (!is_array($resolved)) { + $problems[] = 'docker compose config did not print valid JSON.'; + return $problems; + } + // An emptied compose file is almost always a mistake, and "up" would fail on it anyway. + if (!is_array($resolved['services'] ?? null) || $resolved['services'] === []) { + $problems[] = 'The compose file defines no services (with the profiles in use).'; + return $problems; + } + + return array_merge( + $problems, + $this->missingEnvKeys(), + $this->missingExternalResources($resolved), + $this->missingPaths($resolved), + $this->takenContainerNames($resolved), + $this->takenPorts($resolved) + ); + } + + /** + * Keys in a .env-style file, in the order they appear. + * + * Understands blank lines, # comments, "export KEY=value", quoted values, + * a bare "KEY" line, and Windows line endings. + * + * @return string[] + */ + public static function readEnvKeys(string $content): array + { + if (str_starts_with($content, "\xEF\xBB\xBF")) { + $content = substr($content, 3); + } + $keys = []; + foreach (preg_split('/\r\n|\r|\n/', $content) ?: [] as $line) { + $line = trim($line); + if ($line === '' || str_starts_with($line, '#')) { + continue; + } + if (preg_match('/^(?:export\s+)?([A-Za-z_][A-Za-z0-9_]*)\s*(?:=|$)/', $line, $match) === 1) { + $keys[] = $match[1]; + } + } + return array_values(array_unique($keys)); + } + + /** + * @return string[] + */ + private function unsetVariables(string $stderr): array + { + preg_match_all('/The \\\\?"([A-Za-z_][A-Za-z0-9_]*)\\\\?" variable is not set/', $stderr, $matches); + $names = array_values(array_unique($matches[1])); + if ($names === []) { + return []; + } + return ['These variables are used by the compose file but not set: ' . implode(', ', $names) + . '. Add them to the .env file (an empty value must be written as NAME=).']; + } + + /** + * @return string[] + */ + private function missingEnvKeys(): array + { + $example = $this->composeDir . '/.env.example'; + if (!is_file($example)) { + return []; + } + $wanted = self::readEnvKeys((string) file_get_contents($example)); + $have = ($this->envFile !== null && is_file($this->envFile)) + ? self::readEnvKeys((string) file_get_contents($this->envFile)) + : []; + $missing = array_values(array_diff($wanted, $have)); + if ($missing === []) { + return []; + } + return ['The repository\'s .env.example lists keys missing from the stack\'s .env: ' + . implode(', ', $missing) . '.']; + } + + /** + * @param array $resolved + * @return string[] + */ + private function missingExternalResources(array $resolved): array + { + $problems = []; + foreach (['networks' => 'network', 'volumes' => 'volume'] as $section => $kind) { + foreach ((array) ($resolved[$section] ?? []) as $key => $definition) { + if (!is_array($definition) || ($definition['external'] ?? false) !== true) { + continue; + } + $name = (string) ($definition['name'] ?? $key); + if (!DockerCommand::run([$kind, 'inspect', '--', $name], null, 30)->succeeded()) { + $problems[] = "The external $kind '$name' does not exist. Create it before deploying."; + } + } + } + return $problems; + } + + /** + * @param array $resolved + * @return string[] + */ + private function missingPaths(array $resolved): array + { + $problems = []; + $cloneReal = realpath($this->cloneDir) ?: $this->cloneDir; + + foreach ((array) ($resolved['services'] ?? []) as $serviceName => $service) { + if (!is_array($service)) { + continue; + } + foreach ((array) ($service['volumes'] ?? []) as $volume) { + if (!is_array($volume) || ($volume['type'] ?? '') !== 'bind') { + continue; + } + $source = (string) ($volume['source'] ?? ''); + $problem = $this->checkBindSource($source, $cloneReal); + if ($problem !== null) { + $problems[] = "Service '$serviceName': $problem"; + } + } + + $build = $service['build'] ?? null; + if (is_array($build) && isset($build['context']) && str_starts_with((string) $build['context'], '/')) { + if (!is_dir((string) $build['context'])) { + $problems[] = "Service '$serviceName': the build folder {$build['context']} does not exist."; + } + } + } + + foreach (['configs' => 'config', 'secrets' => 'secret'] as $section => $kind) { + foreach ((array) ($resolved[$section] ?? []) as $key => $definition) { + if (is_array($definition) && isset($definition['file']) && !file_exists((string) $definition['file'])) { + $problems[] = "The $kind '$key' file {$definition['file']} does not exist."; + } + } + } + + return $problems; + } + + private function checkBindSource(string $source, string $cloneReal): ?string + { + if ($source === '' || !str_starts_with($source, '/')) { + return null; + } + + // Inside the clone (a relative path such as ./data): Docker may create it, which is harmless there. + if ($source === $cloneReal || str_starts_with($source, $cloneReal . '/')) { + return null; + } + + $mntDir = rtrim(COMPOSE_GIT_MNT_DIR, '/'); + if (!str_starts_with($source, $mntDir . '/')) { + if (file_exists($source)) { + return null; + } + // Outside /mnt a missing top folder is almost always a typo of /mnt + // itself (/mtn/user/...), which would build a tree in RAM. A missing + // folder below an existing one, such as /tmp/transcode after a + // reboot, is left for Docker to create. + $topFolder = '/' . explode('/', substr($source, 1))[0]; + if (!is_dir($topFolder)) { + return "the bind mount source $source does not exist, and neither does $topFolder. " + . 'Check the path for a typo, or create the folder first.'; + } + $this->foldersDockerWillCreate[] = $source; + return null; + } + + if (GitPathGuard::findMountFor($source) === null) { + return "the bind mount $source is not on a mounted disk, pool or share. " + . 'Docker would create it in RAM. Check the path for a typo.'; + } + if (file_exists($source)) { + return null; + } + + // A new stack's folders, such as appdata/newapp/config, are left for Docker to + // create. The share (such as /mnt/user/appdata) must already exist: a missing + // one is a typo, or would quietly become a new share. + $share = GitPathGuard::shareFolder($source); + if ($share === null || !is_dir($share)) { + return "the bind mount source $source does not exist, and neither does " . ($share ?? 'its share') . '. ' + . 'Check the path for a typo, or create the folder first.'; + } + $this->foldersDockerWillCreate[] = $source; + return null; + } + + /** + * Bind-mount folders that do not exist yet and that Docker will create, found by the last run(). + * + * @return string[] + */ + public function foldersDockerWillCreate(): array + { + return $this->foldersDockerWillCreate; + } + + /** + * @param array $resolved + * @return string[] + */ + private function takenContainerNames(array $resolved): array + { + $problems = []; + foreach ((array) ($resolved['services'] ?? []) as $service) { + if (!is_array($service) || !isset($service['container_name'])) { + continue; + } + $name = (string) $service['container_name']; + $inspect = DockerCommand::run(['container', 'inspect', '--', $name], null, 30); + if (!$inspect->succeeded()) { + continue; + } + $containers = json_decode($inspect->stdout, true); + // docker inspect also matches the start of a container id, so a short + // hex-looking name such as "cafe" can find an unrelated container. + $foundName = is_array($containers) ? ltrim((string) ($containers[0]['Name'] ?? ''), '/') : ''; + if ($foundName !== $name) { + continue; + } + $owner = $containers[0]['Config']['Labels']['com.docker.compose.project'] ?? null; + if ($owner !== $this->projectName) { + $problems[] = "A container named '$name' already exists and does not belong to this stack" + . ($owner !== null ? " (it belongs to '$owner')" : '') . '.'; + } + } + return $problems; + } + + /** + * @param array $resolved + * @return string[] + */ + private function takenPorts(array $resolved): array + { + $wanted = []; + foreach ((array) ($resolved['services'] ?? []) as $serviceName => $service) { + if (!is_array($service)) { + continue; + } + // Docker publishes no ports for a container on macvlan or ipvlan + // networks only (such as Unraid's br0), so they cannot clash. + if (($service['ports'] ?? []) !== [] && $this->isOnlyOnMacvlanOrIpvlan($service, $resolved)) { + continue; + } + foreach ((array) ($service['ports'] ?? []) as $port) { + if (!is_array($port) || !isset($port['published'])) { + continue; + } + // A published range ("8000-8010") lets Docker pick any free + // port in it, so only single ports are checked. + $published = (string) $port['published']; + if (preg_match('/^\d+$/', $published) !== 1) { + continue; + } + $wanted[] = [ + 'service' => (string) $serviceName, + 'ip' => (string) ($port['host_ip'] ?? ''), + 'port' => (int) $published, + 'protocol' => strtolower((string) ($port['protocol'] ?? 'tcp')), + ]; + } + } + if ($wanted === []) { + return []; + } + + $heldByOthers = $this->portsHeldByOtherContainers(); + + $problems = []; + foreach ($wanted as $want) { + foreach ($heldByOthers as $held) { + if ($held['port'] !== $want['port'] || $held['protocol'] !== $want['protocol']) { + continue; + } + if (!self::addressesOverlap($want['ip'], $held['ip'])) { + continue; + } + $problems[] = "Service '{$want['service']}': port {$want['port']}/{$want['protocol']} is already in use by {$held['owner']}."; + break; + } + } + return array_values(array_unique($problems)); + } + + /** + * Ports published by running containers of other stacks, or by plain containers. + * + * @return list + */ + private function portsHeldByOtherContainers(): array + { + $result = []; + $ids = DockerCommand::run(['ps', '-q'], null, 30); + $idList = array_values(array_filter(explode("\n", trim($ids->stdout)))); + if (!$ids->succeeded() || $idList === []) { + return $result; + } + $inspect = DockerCommand::run(array_merge(['container', 'inspect', '--'], $idList), null, 60); + $containers = json_decode($inspect->stdout, true); + if (!is_array($containers)) { + return $result; + } + + foreach ($containers as $container) { + if (!is_array($container)) { + continue; + } + $project = $container['Config']['Labels']['com.docker.compose.project'] ?? null; + $name = ltrim((string) ($container['Name'] ?? ''), '/'); + foreach ((array) ($container['NetworkSettings']['Ports'] ?? []) as $containerPort => $bindings) { + $protocol = str_contains((string) $containerPort, '/') ? explode('/', (string) $containerPort)[1] : 'tcp'; + foreach ((array) $bindings as $binding) { + if (!is_array($binding) || !isset($binding['HostPort'])) { + continue; + } + if ($project !== $this->projectName) { + $result[] = [ + 'ip' => (string) ($binding['HostIp'] ?? ''), + 'port' => (int) $binding['HostPort'], + 'protocol' => $protocol, + 'owner' => "container '$name'", + ]; + } + } + } + } + return $result; + } + + /** + * Whether every network a service joins has the macvlan or ipvlan driver. + * + * A service with no networks listed joins the project's default network + * (a bridge). The driver of an external network is asked from Docker. + * + * @param array $service + * @param array $resolved + */ + private function isOnlyOnMacvlanOrIpvlan(array $service, array $resolved): bool + { + if (isset($service['network_mode'])) { + return false; + } + $networkKeys = array_keys((array) ($service['networks'] ?? [])); + if ($networkKeys === []) { + return false; + } + foreach ($networkKeys as $key) { + $definition = (array) ($resolved['networks'][$key] ?? []); + $driver = (string) ($definition['driver'] ?? ''); + if (($definition['external'] ?? false) === true) { + $driver = $this->externalNetworkDriver((string) ($definition['name'] ?? $key)); + } + if ($driver !== 'macvlan' && $driver !== 'ipvlan') { + return false; + } + } + return true; + } + + private function externalNetworkDriver(string $name): string + { + if (!isset($this->externalNetworkDrivers[$name])) { + $inspect = DockerCommand::run(['network', 'inspect', '--format', '{{.Driver}}', '--', $name], null, 30); + $this->externalNetworkDrivers[$name] = $inspect->succeeded() ? trim($inspect->stdout) : ''; + } + return $this->externalNetworkDrivers[$name]; + } + + private static function addressesOverlap(string $a, string $b): bool + { + $any = ['', '0.0.0.0', '::']; + return in_array($a, $any, true) || in_array($b, $any, true) || $a === $b; + } +} diff --git a/tests/bootstrap.php b/tests/bootstrap.php index 133343be..235f5803 100644 --- a/tests/bootstrap.php +++ b/tests/bootstrap.php @@ -52,6 +52,8 @@ function composeLogger($message, $data = null, $type = 'daemon', $level = 'info' define('COMPOSE_UNRAID_VAR_INI', sys_get_temp_dir() . '/compose_git_var.ini'); define('COMPOSE_MOUNTS_FILE', sys_get_temp_dir() . '/compose_git_mounts'); define('COMPOSE_GIT_HOME_DIR', sys_get_temp_dir() . '/compose_git_home'); +// A scripted stand-in for the docker CLI (see FakeDocker in the git deploy tests). +define('COMPOSE_DOCKER_BIN', sys_get_temp_dir() . '/compose_fake_docker/docker'); // Point to the dev-env resvg binary when present; plugin path used on real Unraid define('COMPOSE_RESVG_BIN', is_executable('/tmp/resvg') ? '/tmp/resvg' : '/usr/local/emhttp/plugins/compose.manager/bin/resvg'); diff --git a/tests/unit/GitDeployCheckTest.php b/tests/unit/GitDeployCheckTest.php new file mode 100644 index 00000000..3a282150 --- /dev/null +++ b/tests/unit/GitDeployCheckTest.php @@ -0,0 +1,383 @@ +docker = new FakeDocker(); + $this->mnt = COMPOSE_GIT_MNT_DIR; + FakeDocker::removeTree($this->mnt); + mkdir($this->mnt . '/user/appdata/existing', 0755, true); + file_put_contents(COMPOSE_UNRAID_VAR_INI, "mdState=\"STARTED\"\n"); + file_put_contents(COMPOSE_MOUNTS_FILE, "rootfs {$this->mnt} rootfs rw 0 0\nshfs {$this->mnt}/user fuse.shfs rw 0 0\n"); + + $this->clone = $this->mnt . '/user/appdata/git/whoami-0123abcd'; + mkdir($this->clone . '/whoami', 0755, true); + $this->stackDir = sys_get_temp_dir() . '/compose_git_check_stack'; + FakeDocker::removeTree($this->stackDir); + mkdir($this->stackDir); + } + + protected function tearDown(): void + { + $this->docker->cleanUp(); + FakeDocker::removeTree($this->mnt); + FakeDocker::removeTree($this->stackDir); + parent::tearDown(); + } + + public function testCleanStackHasNoProblems(): void + { + $this->docker->setConfig($this->config(['image' => 'busybox'])); + $this->assertSame([], $this->check()); + } + + public function testComposeFileWithNoServicesIsAProblem(): void + { + $this->docker->setConfig(['name' => 'whoami', 'services' => []]); + $this->assertStringContainsString('defines no services', $this->check()[0]); + } + + // ----- compose itself ----- + + public function testComposeErrorIsReportedWithComposesMessage(): void + { + $this->docker->setConfig([], "yaml: line 2: found character that cannot start any token\n", 1); + $problems = $this->check(); + $this->assertCount(1, $problems); + $this->assertStringContainsString('found character that cannot start any token', $problems[0]); + } + + public function testUnsetVariablesAreProblems(): void + { + $stderr = 'time="2026-10-02T21:10:16-04:00" level=warning msg="The \\"DB_PASS\\" variable is not set. Defaulting to a blank string."' . "\n" + . 'time="2026-10-02T21:10:16-04:00" level=warning msg="The \\"TZ\\" variable is not set. Defaulting to a blank string."' . "\n"; + $this->docker->setConfig($this->config(['image' => 'busybox']), $stderr); + + $problems = $this->check(); + + $this->assertCount(1, $problems); + $this->assertStringContainsString('DB_PASS, TZ', $problems[0]); + } + + public function testComposeIsRunWithTheStacksArgumentsAndProjectName(): void + { + $this->docker->setConfig($this->config(['image' => 'busybox'])); + $this->check(); + $this->assertContains('compose -f ' . $this->clone . '/whoami/compose.yaml -p whoami config --format json', $this->docker->calls()); + } + + // ----- .env.example ----- + + public function testEnvExampleKeysMissingFromEnvAreProblems(): void + { + file_put_contents($this->clone . '/whoami/.env.example', "TZ=\nDB_PASS=changeme\n"); + file_put_contents($this->stackDir . '/.env', "TZ=Europe/London\n"); + $this->docker->setConfig($this->config(['image' => 'busybox'])); + + $problems = $this->check(); + + $this->assertCount(1, $problems); + $this->assertStringContainsString('DB_PASS', $problems[0]); + $this->assertStringNotContainsString('changeme', $problems[0]); + } + + public function testMissingEnvFileMeansEveryExampleKeyIsMissing(): void + { + file_put_contents($this->clone . '/whoami/.env.example', "A=\nB=\n"); + $this->docker->setConfig($this->config(['image' => 'busybox'])); + $this->assertStringContainsString('A, B', $this->check()[0]); + } + + public function testEnvKeysAreReadFromEveryUsualForm(): void + { + $content = "\xEF\xBB\xBF# comment\r\n\r\nexport A=1\r\nB = 2\r\nC=\"x=y\"\r\nD\r\n E=5\r\nnot a key\r\n1BAD=x\r\nA=again\r\n"; + $this->assertSame(['A', 'B', 'C', 'D', 'E'], GitDeployCheck::readEnvKeys($content)); + } + + // ----- external networks and volumes ----- + + public function testMissingExternalNetworkAndVolumeAreProblems(): void + { + $config = $this->config(['image' => 'busybox']); + $config['networks'] = ['proxy' => ['name' => 'zz-proxy', 'external' => true], 'default' => ['name' => 'whoami_default']]; + $config['volumes'] = ['ext' => ['name' => 'zz-ext', 'external' => true], 'local' => ['name' => 'whoami_local']]; + $this->docker->setConfig($config); + + $problems = $this->check(); + $this->assertCount(2, $problems); + $this->assertStringContainsString("network 'zz-proxy'", $problems[0]); + $this->assertStringContainsString("volume 'zz-ext'", $problems[1]); + + $this->docker->addNetwork('zz-proxy'); + $this->docker->addVolume('zz-ext'); + $this->assertSame([], $this->check()); + } + + // ----- paths ----- + + public function testMisspelledShareInBindSourceIsAProblem(): void + { + $this->docker->setConfig($this->config(['image' => 'busybox', 'volumes' => [ + ['type' => 'bind', 'source' => $this->mnt . '/user/apdata/whoami', 'target' => '/config'], + ]])); + $problems = $this->check(); + $this->assertCount(1, $problems); + $this->assertStringContainsString('neither does ' . $this->mnt . '/user/apdata', $problems[0]); + } + + public function testNewShareInBindSourceIsAProblem(): void + { + $this->docker->setConfig($this->config(['image' => 'busybox', 'volumes' => [ + ['type' => 'bind', 'source' => $this->mnt . '/user/newshare', 'target' => '/config'], + ]])); + $this->assertCount(1, $this->check()); + } + + public function testMissingFoldersBelowAnExistingShareAreLeftForDocker(): void + { + $this->docker->setConfig($this->config(['image' => 'busybox', 'volumes' => [ + ['type' => 'bind', 'source' => $this->mnt . '/user/appdata/mystack/thisapp', 'target' => '/config'], + ]])); + $check = new GitDeployCheck( + 'whoami', + ['-f', $this->clone . '/whoami/compose.yaml'], + $this->clone, + $this->clone . '/whoami', + $this->stackDir . '/.env' + ); + + $this->assertSame([], $check->run()); + $this->assertSame([$this->mnt . '/user/appdata/mystack/thisapp'], $check->foldersDockerWillCreate()); + } + + public function testBindSourceOutsideMntWithAMissingTopFolderIsAProblem(): void + { + // Such as /mtn/user/appdata, a typo of /mnt. + $this->docker->setConfig($this->config(['image' => 'busybox', 'volumes' => [ + ['type' => 'bind', 'source' => '/compose-git-test-missing/user/appdata', 'target' => '/config'], + ]])); + $this->assertStringContainsString('neither does /compose-git-test-missing', $this->check()[0]); + } + + public function testMissingFolderBelowAnExistingTopFolderOutsideMntIsLeftForDocker(): void + { + // /tmp/transcode is emptied at every reboot. + $source = sys_get_temp_dir() . '/compose-git-test-transcode'; + FakeDocker::removeTree($source); + $this->docker->setConfig($this->config(['image' => 'busybox', 'volumes' => [ + ['type' => 'bind', 'source' => $source, 'target' => '/transcode'], + ]])); + $check = new GitDeployCheck('whoami', ['-f', $this->clone . '/whoami/compose.yaml'], $this->clone, $this->clone . '/whoami', null); + + $this->assertSame([], $check->run()); + $this->assertSame([$source], $check->foldersDockerWillCreate()); + } + + public function testMissingFoldersBelowAShareThatIsAZfsDatasetAreLeftForDocker(): void + { + // On ZFS the share is its own mount; the share must exist, not the folder below it. + mkdir($this->mnt . '/cache/appdata', 0755, true); + file_put_contents( + COMPOSE_MOUNTS_FILE, + "rootfs {$this->mnt} rootfs rw 0 0\nshfs {$this->mnt}/user fuse.shfs rw 0 0\n" + . "cache {$this->mnt}/cache zfs rw 0 0\ncache/appdata {$this->mnt}/cache/appdata zfs rw 0 0\n" + ); + $this->docker->setConfig($this->config(['image' => 'busybox', 'volumes' => [ + ['type' => 'bind', 'source' => $this->mnt . '/cache/appdata/newapp/config', 'target' => '/config'], + ]])); + $this->assertSame([], $this->check()); + } + + public function testBindSourceOnAMissingMountIsCalledOutAsRam(): void + { + mkdir($this->mnt . '/disk9/appdata', 0755, true); + $this->docker->setConfig($this->config(['image' => 'busybox', 'volumes' => [ + ['type' => 'bind', 'source' => $this->mnt . '/disk9/appdata', 'target' => '/config'], + ]])); + $this->assertStringContainsString('RAM', $this->check()[0]); + } + + public function testExistingBindSourceAndMissingSourceInsideTheCloneAreFine(): void + { + $this->docker->setConfig($this->config(['image' => 'busybox', 'volumes' => [ + ['type' => 'bind', 'source' => $this->mnt . '/user/appdata/existing', 'target' => '/config'], + ['type' => 'bind', 'source' => $this->clone . '/whoami/data', 'target' => '/data'], + ['type' => 'volume', 'source' => 'named', 'target' => '/named'], + ]])); + $this->assertSame([], $this->check()); + } + + public function testMissingConfigSecretAndBuildFolderAreProblems(): void + { + $config = $this->config(['build' => ['context' => $this->clone . '/whoami/build', 'dockerfile' => 'Dockerfile']]); + $config['configs'] = ['app' => ['file' => $this->clone . '/whoami/app.cfg']]; + $config['secrets'] = ['token' => ['file' => $this->clone . '/whoami/token.txt']]; + $this->docker->setConfig($config); + + $this->assertCount(3, $this->check()); + } + + // ----- names and ports ----- + + public function testContainerNameTakenByAnotherStackIsAProblem(): void + { + $this->docker->setConfig($this->config(['image' => 'busybox', 'container_name' => 'whoami'])); + $this->docker->addNamedContainer('whoami', 'otherstack'); + $this->assertStringContainsString("belongs to 'otherstack'", $this->check()[0]); + } + + public function testContainerNameHeldByThisStackOrUnusedIsFine(): void + { + $this->docker->setConfig($this->config(['image' => 'busybox', 'container_name' => 'whoami'])); + $this->assertSame([], $this->check()); + $this->docker->addNamedContainer('whoami', 'whoami'); + $this->assertSame([], $this->check()); + } + + public function testContainerIdThatOnlyStartsWithTheNameIsNotAClash(): void + { + // docker inspect "cafe" also finds a container whose id starts with cafe. + $this->docker->setConfig($this->config(['image' => 'busybox', 'container_name' => 'cafe'])); + file_put_contents( + $this->docker->dir . '/containers/cafe.json', + json_encode([['Name' => '/unrelated', 'Config' => ['Labels' => ['com.docker.compose.project' => 'otherstack']]]]) + ); + + $this->assertSame([], $this->check()); + } + + public function testContainerNameTakenByAPlainDockerContainerIsAProblem(): void + { + $this->docker->setConfig($this->config(['image' => 'busybox', 'container_name' => 'plex'])); + $this->docker->addNamedContainer('plex', null); + $this->assertCount(1, $this->check()); + } + + public function testPortPublishedByAnotherContainerIsAProblem(): void + { + $this->docker->setConfig($this->config(['image' => 'busybox', 'ports' => [ + ['target' => 80, 'published' => '8080', 'protocol' => 'tcp'], + ]])); + $this->docker->setRunning([ + ['name' => 'nginx', 'project' => null, 'ports' => ['80/tcp' => [['HostIp' => '0.0.0.0', 'HostPort' => '8080']]]], + ]); + + $problems = $this->check(); + $this->assertCount(1, $problems); + $this->assertStringContainsString("container 'nginx'", $problems[0]); + } + + public function testPortHeldByThisStacksOwnContainerIsFine(): void + { + $this->docker->setConfig($this->config(['image' => 'busybox', 'ports' => [ + ['target' => 80, 'published' => '8080', 'protocol' => 'tcp'], + ]])); + $this->docker->setRunning([ + ['name' => 'whoami-whoami-1', 'project' => 'whoami', 'ports' => ['80/tcp' => [['HostIp' => '0.0.0.0', 'HostPort' => '8080']]]], + ]); + + $this->assertSame([], $this->check()); + } + + public function testSamePortOnUdpOrADifferentAddressIsFine(): void + { + $this->docker->setConfig($this->config(['image' => 'busybox', 'ports' => [ + ['target' => 53, 'published' => '5353', 'protocol' => 'udp'], + ['target' => 80, 'published' => '8080', 'protocol' => 'tcp', 'host_ip' => '127.0.0.1'], + ]])); + $this->docker->setRunning([ + ['name' => 'other', 'project' => 'other', 'ports' => [ + '53/tcp' => [['HostIp' => '0.0.0.0', 'HostPort' => '5353']], + '80/tcp' => [['HostIp' => '192.168.1.5', 'HostPort' => '8080']], + ]], + ]); + $this->assertSame([], $this->check()); + } + + public function testPublishedRangeIsLeftToDocker(): void + { + // "6000-6010:6000" means any free port in the range. + $this->docker->setConfig($this->config(['image' => 'busybox', 'ports' => [ + ['target' => 6000, 'published' => '6000-6010', 'protocol' => 'tcp'], + ]])); + $this->docker->setRunning([ + ['name' => 'other', 'project' => 'other', 'ports' => ['6005/tcp' => [['HostIp' => '0.0.0.0', 'HostPort' => '6005']]]], + ]); + $this->assertSame([], $this->check()); + } + + public function testPortsOfAServiceOnlyOnAMacvlanNetworkAreNotChecked(): void + { + // Unraid's br0 is an external macvlan network; Docker publishes no ports there. + $this->docker->addNetwork('br0', 'macvlan'); + $this->docker->setConfig([ + 'name' => 'whoami', + 'services' => ['whoami' => ['image' => 'busybox', 'networks' => ['br0' => null], 'ports' => [ + ['target' => 80, 'published' => '80', 'protocol' => 'tcp'], + ]]], + 'networks' => ['br0' => ['name' => 'br0', 'external' => true]], + ]); + $this->docker->setRunning([ + ['name' => 'nginx', 'project' => null, 'ports' => ['80/tcp' => [['HostIp' => '0.0.0.0', 'HostPort' => '80']]]], + ]); + $this->assertSame([], $this->check()); + } + + public function testPortsOfAServiceAlsoOnABridgeNetworkAreChecked(): void + { + $this->docker->addNetwork('br0', 'macvlan'); + $this->docker->setConfig([ + 'name' => 'whoami', + 'services' => ['whoami' => ['image' => 'busybox', 'networks' => ['br0' => null, 'default' => null], 'ports' => [ + ['target' => 80, 'published' => '80', 'protocol' => 'tcp'], + ]]], + 'networks' => ['br0' => ['name' => 'br0', 'external' => true], 'default' => ['name' => 'whoami_default']], + ]); + $this->docker->setRunning([ + ['name' => 'nginx', 'project' => null, 'ports' => ['80/tcp' => [['HostIp' => '0.0.0.0', 'HostPort' => '80']]]], + ]); + $this->assertCount(1, $this->check()); + } + + // ----- helpers ----- + + /** + * @param array $service + * @return array + */ + private function config(array $service): array + { + return ['name' => 'whoami', 'services' => ['whoami' => $service]]; + } + + /** @return string[] */ + private function check(): array + { + $check = new GitDeployCheck( + 'whoami', + ['-f', $this->clone . '/whoami/compose.yaml'], + $this->clone, + $this->clone . '/whoami', + $this->stackDir . '/.env' + ); + return $check->run(); + } +} diff --git a/tests/unit/fixtures/fake-docker.sh b/tests/unit/fixtures/fake-docker.sh new file mode 100644 index 00000000..fbbec24a --- /dev/null +++ b/tests/unit/fixtures/fake-docker.sh @@ -0,0 +1,55 @@ +#!/bin/bash +# A scripted stand-in for the docker CLI, used by the git deploy tests. +# +# It answers from files next to itself (the tests write them): +# config.json / config.stderr / config.exit docker compose ... config --format json +# networks/, volumes/ these exist (network/volume inspect succeeds); +# a network file holds its driver, if any +# containers/.json docker container inspect +# running.ids, running.json docker ps -q, and docker container inspect +# Every call is appended to calls.log, one line per call. +here="$(cd "$(dirname "$0")" && pwd)" +printf '%s\n' "$*" >> "$here/calls.log" + +case "$1" in + compose) + for arg in "$@"; do + if [ "$arg" = "config" ]; then + [ -f "$here/config.stderr" ] && cat "$here/config.stderr" >&2 + [ -f "$here/config.json" ] && cat "$here/config.json" + exit "$(cat "$here/config.exit" 2>/dev/null || echo 0)" + fi + done + exit 0 + ;; + network|volume) + kind="$1s" + name="${*: -1}" + if [ -e "$here/$kind/$name" ]; then + # A network file may hold its driver, printed for --format {{.Driver}}. + cat "$here/$kind/$name" + exit 0 + fi + echo "Error: No such $1: $name" >&2 + exit 1 + ;; + ps) + cat "$here/running.ids" 2>/dev/null + exit 0 + ;; + container) + # container inspect -- + shift 3 + if [ "$#" -eq 1 ] && [ -f "$here/containers/$1.json" ]; then + cat "$here/containers/$1.json" + exit 0 + fi + if [ -f "$here/running.ids" ] && [ "$*" = "$(tr '\n' ' ' < "$here/running.ids" | sed 's/ $//')" ]; then + cat "$here/running.json" + exit 0 + fi + echo "Error: No such container: $*" >&2 + exit 1 + ;; +esac +exit 0 diff --git a/tests/unit/support/FakeDocker.php b/tests/unit/support/FakeDocker.php new file mode 100644 index 00000000..c3f5a80f --- /dev/null +++ b/tests/unit/support/FakeDocker.php @@ -0,0 +1,115 @@ +dir = dirname(COMPOSE_DOCKER_BIN); + self::removeTree($this->dir); + mkdir($this->dir . '/networks', 0755, true); + mkdir($this->dir . '/volumes'); + mkdir($this->dir . '/containers'); + copy(__DIR__ . '/../fixtures/fake-docker.sh', COMPOSE_DOCKER_BIN); + chmod(COMPOSE_DOCKER_BIN, 0755); + } + + public function cleanUp(): void + { + self::removeTree($this->dir); + } + + /** + * What `docker compose config --format json` prints. + * + * @param array $config + */ + public function setConfig(array $config, string $stderr = '', int $exitCode = 0): void + { + file_put_contents($this->dir . '/config.json', json_encode($config, JSON_UNESCAPED_SLASHES)); + file_put_contents($this->dir . '/config.stderr', $stderr); + file_put_contents($this->dir . '/config.exit', (string) $exitCode); + } + + /** + * A network that exists, with its driver (empty for the default, bridge). + */ + public function addNetwork(string $name, string $driver = ''): void + { + file_put_contents($this->dir . '/networks/' . $name, $driver === '' ? '' : $driver . "\n"); + } + + public function addVolume(string $name): void + { + touch($this->dir . '/volumes/' . $name); + } + + /** + * A container that exists (running or not), found by name. + */ + public function addNamedContainer(string $name, ?string $project): void + { + $labels = $project === null ? [] : ['com.docker.compose.project' => $project]; + file_put_contents( + $this->dir . '/containers/' . $name . '.json', + json_encode([['Name' => '/' . $name, 'Config' => ['Labels' => $labels]]]) + ); + } + + /** + * Running containers and the ports they publish. + * + * @param list>}> $containers + */ + public function setRunning(array $containers): void + { + $ids = []; + $inspect = []; + foreach ($containers as $i => $container) { + $ids[] = 'id' . $i; + $inspect[] = [ + 'Name' => '/' . $container['name'], + 'Config' => ['Labels' => $container['project'] === null ? [] : ['com.docker.compose.project' => $container['project']]], + 'NetworkSettings' => ['Ports' => $container['ports']], + ]; + } + file_put_contents($this->dir . '/running.ids', implode("\n", $ids) . "\n"); + file_put_contents($this->dir . '/running.json', json_encode($inspect)); + } + + /** + * @return string[] Every docker call made, as one line of arguments each + */ + public function calls(): array + { + $log = @file_get_contents($this->dir . '/calls.log'); + return $log === false ? [] : array_values(array_filter(explode("\n", $log))); + } + + public static function removeTree(string $path): void + { + // readlink(), not is_link(): the test stream wrapper hides symlinks from is_link(). + if (@readlink($path) !== false || is_file($path)) { + unlink($path); + return; + } + if (!is_dir($path)) { + return; + } + foreach (scandir($path) ?: [] as $entry) { + if ($entry !== '.' && $entry !== '..') { + self::removeTree($path . '/' . $entry); + } + } + rmdir($path); + } +} From 277acdfef873e2de878906415632521cbd0869ab Mon Sep 17 00:00:00 2001 From: Grant Emsley Date: Sun, 4 Oct 2026 16:10:36 -0400 Subject: [PATCH 008/109] feat(git): record which commit a git stack has deployed GitStackState is git_state.json in the stack folder: the last commit that deployed successfully, and the commit whose deploy failed, if the last one did. It is written only when a deploy finishes, so the flash sees one small write per deploy. A broken file is reported, never guessed at, and the format follows the same change rules as git.json. The next commit, gitdeploy, reads and writes it. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_011iUWDG8wTp8MhsFDqNC4NL --- .../compose.manager/include/GitStackState.php | 112 ++++++++++++++++++ tests/unit/GitStackStateTest.php | 61 ++++++++++ 2 files changed, 173 insertions(+) create mode 100644 source/compose.manager/include/GitStackState.php create mode 100644 tests/unit/GitStackStateTest.php diff --git a/source/compose.manager/include/GitStackState.php b/source/compose.manager/include/GitStackState.php new file mode 100644 index 00000000..c1812062 --- /dev/null +++ b/source/compose.manager/include/GitStackState.php @@ -0,0 +1,112 @@ +getMessage(), 0, $error); + } + } + + /** + * @throws RuntimeException if it cannot be written safely + */ + public function save(string $stackDir): void + { + $stackDir = rtrim($stackDir, '/'); + if (str_starts_with($stackDir, rtrim(COMPOSE_GIT_MNT_DIR, '/') . '/')) { + GitPathGuard::assertSafeToWrite($stackDir); + } + $json = json_encode([ + 'version' => self::FORMAT_VERSION, + 'deployedCommit' => $this->deployedCommit, + 'failedCommit' => $this->failedCommit, + ], JSON_PRETTY_PRINT | JSON_UNESCAPED_SLASHES); + + $file = $stackDir . '/' . self::FILE_NAME; + $tmpFile = $file . '.tmp-' . bin2hex(random_bytes(4)); + // A short write (a full flash) counts as a failure, so a cut-off file never replaces the good one. + if ($json === false || file_put_contents($tmpFile, $json . "\n") !== strlen($json . "\n")) { + @unlink($tmpFile); + throw new RuntimeException("Could not write $tmpFile."); + } + if (!rename($tmpFile, $file)) { + @unlink($tmpFile); + throw new RuntimeException("Could not replace $file."); + } + } +} diff --git a/tests/unit/GitStackStateTest.php b/tests/unit/GitStackStateTest.php new file mode 100644 index 00000000..477836f0 --- /dev/null +++ b/tests/unit/GitStackStateTest.php @@ -0,0 +1,61 @@ +stackDir = $this->createTempDir(); + } + + public function testNoFileMeansNothingDeployedYet(): void + { + $state = GitStackState::load($this->stackDir); + $this->assertNull($state->deployedCommit); + $this->assertNull($state->failedCommit); + } + + public function testSaveThenLoad(): void + { + $deployed = str_repeat('a', 40); + $failed = str_repeat('b', 40); + (new GitStackState($deployed, $failed))->save($this->stackDir); + + $state = GitStackState::load($this->stackDir); + $this->assertSame($deployed, $state->deployedCommit); + $this->assertSame($failed, $state->failedCommit); + } + + /** @return array */ + public static function brokenFiles(): array + { + return [ + 'not json' => ['{'], + 'extra key' => ['{"version":1,"deployedCommit":null,"failedCommit":null,"paused":false}'], + 'other version' => ['{"version":2,"deployedCommit":null,"failedCommit":null}'], + 'short commit' => ['{"version":1,"deployedCommit":"abc123","failedCommit":null}'], + 'option as commit' => ['{"version":1,"deployedCommit":"--upload-pack=x","failedCommit":null}'], + 'number' => ['{"version":1,"deployedCommit":5,"failedCommit":null}'], + ]; + } + + #[\PHPUnit\Framework\Attributes\DataProvider('brokenFiles')] + public function testBrokenFileIsReportedNotUsed(string $content): void + { + file_put_contents($this->stackDir . '/' . GitStackState::FILE_NAME, $content); + $this->expectException(RuntimeException::class); + GitStackState::load($this->stackDir); + } +} From e74272f97cf4606dc859a7c2e0c9e2afa9f9131b Mon Sep 17 00:00:00 2001 From: Grant Emsley Date: Sun, 4 Oct 2026 16:10:36 -0400 Subject: [PATCH 009/109] feat(git): deploy a git stack with compose.sh gitdeploy `compose.sh -c gitdeploy` deploys a git stack under the stack's usual lock: 1. git_stack.php prepare fetches the branch (or takes a given commit), refuses if the clone has local changes (or saves them as a patch and discards them when asked), refuses if the compose file no longer exists at that commit, checks it out, and runs every pre-flight check. If any of that fails, the previous commit is put back and no container has been touched. 2. The images are pulled, then services with a build section are built, as steps of their own. If either fails, the previous commit is put back. 3. `up -d --remove-orphans` starts the stack from the new commit. On success the commit is recorded as deployed in git_state.json and superseded images are removed, as Update and auto-update do. On failure the commit is recorded as failed. A half-finished up is not rolled back, and the next deploy starts from the failed commit. Every answer from the git helper is checked: PHP notices go to stderr, the previous commit must be a full commit id, and a failure to read the compose files stops the deploy. If putting the previous commit back fails, the message says the clone is still at the new commit. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_011iUWDG8wTp8MhsFDqNC4NL --- source/compose.manager/include/GitDeploy.php | 283 ++++++++++++++ source/compose.manager/scripts/compose.sh | 183 ++++++++- source/compose.manager/scripts/git_stack.php | 86 +++++ tests/unit/GitDeployTest.php | 384 +++++++++++++++++++ tests/unit/compose.bats | 2 +- tests/unit/gitdeploy.bats | 230 +++++++++++ 6 files changed, 1163 insertions(+), 5 deletions(-) create mode 100644 source/compose.manager/include/GitDeploy.php create mode 100644 source/compose.manager/scripts/git_stack.php create mode 100644 tests/unit/GitDeployTest.php create mode 100644 tests/unit/gitdeploy.bats diff --git a/source/compose.manager/include/GitDeploy.php b/source/compose.manager/include/GitDeploy.php new file mode 100644 index 00000000..08c321e2 --- /dev/null +++ b/source/compose.manager/include/GitDeploy.php @@ -0,0 +1,283 @@ +say = $say ?? static function (string $message): void { + }; + } + + /** + * Fetch, check out and check the commit to deploy. + * + * @param string $projectName The stack's compose project name + * @param string[] $profiles Compose profiles the deploy will use + * @param string|null $commit A specific commit to deploy, or null for the branch's latest + * @param bool $saveAndDiscardLocalChanges Save local changes as a patch, then discard them, instead of refusing + * @return string The commit that was checked out before, to put back if the pull fails + * @throws RuntimeException naming why nothing was deployed + */ + public function prepare(string $projectName, array $profiles, ?string $commit, bool $saveAndDiscardLocalChanges): string + { + if (!GitPathGuard::isArrayStarted()) { + throw new RuntimeException('The array is not started, so nothing was deployed.'); + } + $settings = $this->loadSettings(); + $clone = new GitClone($settings); + $clone->assertOwned(); + $state = GitStackState::load($this->stackDir); + + $previous = $clone->checkedOutCommit(); + + // Fetch and check the target first: fetching changes only .git, so a + // network failure or a removed compose file stops the deploy before + // any local change is discarded. + ($this->say)("Fetching {$settings->branch} from {$settings->url}..."); + $latest = $clone->fetch(); + $target = $commit ?? $latest; + if (!GitClone::isCommitId($target)) { + throw new RuntimeException("Not a full commit id: $target"); + } + if (!$clone->composeFileExistsAt($target)) { + throw new RuntimeException( + "{$settings->composePath} does not exist at commit " . substr($target, 0, 12) + . '. It may have been moved or removed in the repository. Nothing was deployed, and the stack was left as it is.' + ); + } + + $this->dealWithLocalChanges($clone, $state, $previous, $saveAndDiscardLocalChanges); + + try { + if ($target !== $previous) { + ($this->say)('Checking out ' . substr($target, 0, 12) . '...'); + $clone->checkOut($target); + } + $this->runChecks($settings, $projectName, $profiles); + } catch (Throwable $error) { + throw $this->putBack($clone, $previous, $error); + } + + return $previous; + } + + /** + * Put the clone back at a commit (after a failed pull, before any container changed). + */ + public function restore(string $commit): void + { + $clone = new GitClone($this->loadSettings()); + if ($clone->checkedOutCommit() !== $commit) { + $clone->checkOut($commit); + } + } + + /** + * Record how the deploy of the checked-out commit ended. + */ + public function finish(bool $succeeded): void + { + $clone = new GitClone($this->loadSettings()); + $commit = $clone->checkedOutCommit(); + $state = GitStackState::load($this->stackDir); + $newState = $succeeded + ? new GitStackState($commit, null) + : new GitStackState($state->deployedCommit, $commit); + $newState->save($this->stackDir); + } + + /** + * The -f and --env-file arguments for the stack as it is checked out now. + * + * @return string[] + */ + public function composeArgs(): array + { + $composeRoot = dirname(rtrim($this->stackDir, '/')); + $folder = basename(rtrim($this->stackDir, '/')); + StackInfo::clearCache(); + $stack = StackInfo::fromProject($composeRoot, $folder); + $built = $stack->buildComposeArgs(); + + $args = []; + foreach ($built['filePaths'] as $file) { + $args[] = '-f'; + $args[] = $file; + } + if ($args === []) { + throw new RuntimeException("No compose file was found for the stack in {$this->stackDir}."); + } + if (isset($built['envFilePath'])) { + $args[] = '--env-file'; + $args[] = $built['envFilePath']; + } + return $args; + } + + /** + * After a failed check or checkout, check the previous commit out again. + * Returns the error to throw: the original problem, or, if putting the + * commit back failed too, both, so nobody is told the clone is where it + * was when it is not. + */ + private function putBack(GitClone $clone, string $previous, Throwable $error): RuntimeException + { + try { + if ($clone->checkedOutCommit() !== $previous) { + ($this->say)('Putting back ' . substr($previous, 0, 12) . '...'); + $clone->checkOut($previous); + } + } catch (Throwable $putBackError) { + return new RuntimeException( + $error->getMessage() . "\nThe previous commit " . substr($previous, 0, 12) + . ' could not be checked out again (' . $putBackError->getMessage() . '). ' + . 'No container was changed, but the clone is not at the deployed commit.', + 0, + $error + ); + } + return $error instanceof RuntimeException ? $error : new RuntimeException($error->getMessage(), 0, $error); + } + + private function loadSettings(): GitStackSettings + { + $settings = GitStackSettings::load($this->stackDir); + if ($settings === null) { + throw new RuntimeException("{$this->stackDir} is not a git stack (it has no git.json)."); + } + + // compose is run with the file named in the stack's indirect file, + // while the checks use git.json's compose path. If git.json was edited + // by hand, the two can disagree, and the checks would pass for a file + // that is not the one deployed. + $indirectFile = rtrim($this->stackDir, '/') . '/indirect'; + $indirect = trim((string) @file_get_contents($indirectFile)); + if ($indirect !== $settings->composeFileInClone()) { + throw new RuntimeException( + "git.json and the stack's indirect file name different compose files (" + . $settings->composeFileInClone() . " and " . ($indirect === '' ? 'none' : $indirect) . "). " + . "Make $indirectFile hold the first, then deploy again." + ); + } + return $settings; + } + + /** + * Refuse when the clone has local changes, or save them as a patch and discard them. + * + * A commit made by hand in the clone counts as a local change too. Call + * after fetch(), so the branch it is compared with is up to date. + */ + private function dealWithLocalChanges(GitClone $clone, GitStackState $state, string $checkedOut, bool $saveAndDiscard): void + { + $changedFiles = $clone->locallyChangedFiles(); + // A checked-out commit that is on the branch came from the repository: + // the deployed one, one whose "up" failed, or one left checked out when + // a deploy was interrupted. The deployed and failed commits also count + // when a rewritten branch no longer holds them. Anything else was + // committed in the clone by hand. + $madeByHand = $checkedOut !== $state->deployedCommit + && $checkedOut !== $state->failedCommit + && !$clone->isOnBranch($checkedOut); + if ($changedFiles === [] && !$madeByHand) { + return; + } + + $base = $checkedOut; + if ($madeByHand) { + // Save the hand-made commits too, from where they left the branch. + $base = $clone->lastCommitSharedWithBranch($checkedOut) ?? $state->deployedCommit; + if ($base === null) { + throw new RuntimeException( + 'The clone is at ' . substr($checkedOut, 0, 12) . ', which shares no history with the branch ' + . "{$clone->settings()->branch}, so nothing was deployed. Clone it again with compose-git reclone." + ); + } + } + + if (!$saveAndDiscard) { + $what = $changedFiles !== [] + ? 'local changes to ' . implode(', ', array_slice($changedFiles, 0, 5)) . (count($changedFiles) > 5 ? ', ...' : '') + : 'a commit made in the clone by hand (' . substr($checkedOut, 0, 12) . ')'; + throw new RuntimeException( + "The clone has $what, so nothing was deployed. " + . 'Deploy again with the option to save them as a patch and discard them, or undo them by hand.' + ); + } + + $clone->unstageAll(); + $clone->assertDiscardKeepsUntrackedFiles(); + $patch = $clone->saveChangesAsPatch($base, $this->stackDir); + if ($patch !== null) { + ($this->say)("Saved the local changes to $patch"); + } + $clone->discardLocalChanges(); + } + + /** + * @param string[] $profiles + * @throws RuntimeException listing every problem found + */ + private function runChecks(GitStackSettings $settings, string $projectName, array $profiles): void + { + ($this->say)('Checking the stack before deploying...'); + + // The compose file must stay inside the clone, even through a symlink in the repository. + $composeFile = realpath($settings->composeFileInClone()); + $cloneReal = realpath($settings->cloneDir); + if ($composeFile === false || $cloneReal === false || !str_starts_with($composeFile, $cloneReal . '/')) { + throw new RuntimeException("{$settings->composePath} leads outside the repository (through a symlink), so it was not used."); + } + + $args = $this->composeArgs(); + foreach ($profiles as $profile) { + $args[] = '--profile'; + $args[] = $profile; + } + $envFile = null; + $envIndex = array_search('--env-file', $args, true); + if ($envIndex !== false) { + $envFile = $args[$envIndex + 1]; + // Say it: the stack folder's .env wins over the repository's, so + // which one is used can change between two deploys. + ($this->say)("Using the .env at $envFile"); + } else { + $envFile = rtrim($this->stackDir, '/') . '/.env'; + } + + $check = new GitDeployCheck($projectName, $args, $settings->cloneDir, dirname($composeFile), $envFile); + $problems = $check->run(); + if ($problems !== []) { + throw new RuntimeException("The stack was not deployed:\n- " . implode("\n- ", $problems)); + } + foreach ($check->foldersDockerWillCreate() as $folder) { + ($this->say)("Docker will create the missing folder $folder"); + } + } +} diff --git a/source/compose.manager/scripts/compose.sh b/source/compose.manager/scripts/compose.sh index ba850613..a3b25e9a 100755 --- a/source/compose.manager/scripts/compose.sh +++ b/source/compose.manager/scripts/compose.sh @@ -12,7 +12,7 @@ LOCK_TIMEOUT=${COMPOSE_LOCK_TIMEOUT:-30} LOCK_DIR="/var/run/compose.manager" SHORT=e:,c:,f:,p:,d:,o:,g:,s:,w: -LONG=env,command:,file:,project_name:,project_dir:,override:,profile:,debug,recreate,remove-orphans,stack-path:,workdir:,follow-logs,wait,wait-timeout:,build,credential-id: +LONG=env,command:,file:,project_name:,project_dir:,override:,profile:,debug,recreate,remove-orphans,stack-path:,workdir:,follow-logs,wait,wait-timeout:,build,credential-id:,git-commit:,save-local-changes OPTS=$(getopt -a -n compose --options $SHORT --longoptions $LONG -- "$@") eval set -- "$OPTS" @@ -34,6 +34,8 @@ lock_fd="" operation_exit_code=0 credential_id="" docker_config_dir="" +git_commit="" +save_local_changes=false # Logging helper — delegates to shared composeLogger, adds console echo in debug mode @@ -222,6 +224,14 @@ do credential_id="$2" shift 2 ;; + --git-commit ) + git_commit="$2" + shift 2 + ;; + --save-local-changes ) + save_local_changes=true + shift; + ;; --) shift; break @@ -242,7 +252,7 @@ if [ -n "$credential_id" ]; then credential_label=$(printf '%s\n' "$credential_output" | sed -n '2p') export DOCKER_CONFIG="$docker_config_dir" log_msg "DEBUG" "Using registry credential '${credential_label:-$credential_id}' for $name" -elif [[ "$command" =~ ^(up|pull|update)$ ]]; then +elif [[ "$command" =~ ^(up|pull|update|gitdeploy)$ ]]; then log_msg "DEBUG" "No registry credential configured for $name; using default Docker credentials" fi @@ -272,7 +282,7 @@ fi # Acquire lock for operations that modify state (not for read-only commands) case $command in - up|down|pull|update|stop) + up|down|pull|update|stop|gitdeploy) # Lock by canonical project name so every path uses the same stack identity. lock_name="$name" if ! acquire_lock "$lock_name"; then @@ -481,6 +491,171 @@ case $command in fi ;; + gitdeploy) + # Deploy a git stack. Fetching, checking out, checking, pulling and + # starting all happen under this one lock. Until "up" starts, any failure + # leaves the containers untouched and the clone at its previous commit. + git_helper="$(dirname "$0")/git_stack.php" + # Run the git helper. PHP notices go to stderr, so stdout carries only its answer. + git_stack() { + php -d display_errors=stderr "$git_helper" "$@" + } + # After a failure before "up", check the previous commit out again and say + # where the clone is now. + put_back_previous_commit() { + if git_stack restore "$stack_path" "$previous_commit"; then + echo "The previous commit was put back and no container was changed." + else + log_msg "ERROR" "Could not put back commit $previous_commit for git stack $name" + echo "No container was changed, but the previous commit could not be put back: the clone is still at the new commit." + fi + } + if [ -z "$stack_path" ] || [ ! -f "$stack_path/git.json" ]; then + log_msg "ERROR" "gitdeploy needs the stack folder of a git stack (--stack-path)" + echo "✗ $name is not a git stack." + exit 1 + fi + + # 1. Fetch, check out and check the new commit (git_stack.php prints the + # commit that was checked out before, for step 2). + prepare_args=("$stack_path" "$name") + if [ -n "$git_commit" ]; then + prepare_args+=("--commit" "$git_commit") + fi + if [ "$save_local_changes" = true ]; then + prepare_args+=("--save-local-changes") + fi + for profile_name in "${profile_names[@]}"; do + prepare_args+=("--profile" "$profile_name") + done + if ! previous_commit=$(git_stack prepare "${prepare_args[@]}"); then + save_result "failed" 1 "gitdeploy" + log_msg "ERROR" "Git deploy of $name stopped before any container was changed" + echo "" + echo "✗ Stack $name was not deployed. No container was changed." + exit 1 + fi + if ! [[ "$previous_commit" =~ ^[0-9a-f]{40}([0-9a-f]{24})?$ ]]; then + save_result "failed" 1 "gitdeploy" + log_msg "ERROR" "git_stack.php prepare printed something other than a commit id for $name" + echo "" + echo "✗ Stack $name was not deployed. No container was changed, but the clone may be at the new commit." + exit 1 + fi + + # The compose files may have changed with the new commit, so build the + # command from the stack as it is checked out now. + git_compose_args=() + git_args_file=$(mktemp) + if ! git_stack compose-args "$stack_path" > "$git_args_file"; then + rm -f "$git_args_file" + save_result "failed" 1 "gitdeploy" + log_msg "ERROR" "Could not read the compose files of git stack $name" + echo "" + echo "✗ Stack $name was not deployed." + put_back_previous_commit + exit 1 + fi + mapfile -d '' -t git_compose_args < "$git_args_file" + rm -f "$git_args_file" + git_compose=(docker compose "${git_compose_args[@]}" "${profile_args[@]}") + + # A deploy always builds services that have a build section (the new + # commit may change their Dockerfile), as a step of its own before up. + up_args=("-d" "--remove-orphans") + if [ "$wait_for_healthy" = true ]; then + up_args+=("--wait") + if [ -n "$wait_timeout" ]; then + up_args+=("--wait-timeout" "$wait_timeout") + fi + fi + if [ "$debug" = true ]; then + log_msg "DEBUG" "${git_compose[*]} -p $name pull --ignore-buildable" + log_msg "DEBUG" "${git_compose[*]} -p $name build" + log_msg "DEBUG" "${git_compose[*]} -p $name up ${up_args[*]}" + fi + + # Remember the stack's current images, to remove the ones it no longer uses afterwards. + old_images=() + mapfile -t old_images < <("${git_compose[@]}" -p "$name" images -q 2>/dev/null) + + # 2. Pull and build images. If either fails, put the previous commit back: + # still no container has changed. + echo "Pulling images..." + if ! "${git_compose[@]}" -p "$name" pull --ignore-buildable; then + save_result "failed" 1 "gitdeploy" + log_msg "ERROR" "Failed to pull images for git stack $name" + echo "" + echo "✗ Pulling images for $name failed." + put_back_previous_commit + exit 1 + fi + echo "" + echo "Building images..." + if ! "${git_compose[@]}" -p "$name" build; then + save_result "failed" 1 "gitdeploy" + log_msg "ERROR" "Failed to build images for git stack $name" + echo "" + echo "✗ Building images for $name failed." + put_back_previous_commit + exit 1 + fi + + # 3. Start the stack at the new commit. + echo "" + echo "Starting containers..." + "${git_compose[@]}" -p "$name" up "${up_args[@]}" + up_exit=$? + operation_exit_code=$up_exit + + if [ $up_exit -eq 0 ]; then + if ! git_stack finish "$stack_path" success; then + log_msg "ERROR" "Git stack $name started, but its deployed commit could not be recorded" + echo "⚠ The stack started, but the deployed commit could not be recorded. Deploy it again to record it." + fi + + new_images=() + mapfile -t new_images < <("${git_compose[@]}" -p "$name" images -q 2>/dev/null) + superseded_images=() + for old_image in "${old_images[@]}"; do + still_used=false + for new_image in "${new_images[@]}"; do + if [ "$old_image" = "$new_image" ]; then + still_used=true + fi + done + if [ "$still_used" = false ]; then + superseded_images+=("$old_image") + fi + done + if (( ${#superseded_images[@]} )); then + echo "" + echo "Cleaning up old images..." + # Plain docker rmi leaves alone any image another container still uses. + docker rmi "${superseded_images[@]}" 2>/dev/null || true + fi + + if [ -n "$stack_path" ] && [ -d "$stack_path" ]; then + date -Iseconds > "$stack_path/started_at" + persist_running_profiles + fi + save_result "success" 0 "gitdeploy" + echo "" + echo "✓ Stack $name deployed successfully" + else + # A half-finished up cannot be undone reliably, so it is not rolled back: + # the commit is recorded as failed, for a person to fix and deploy again. + if ! git_stack finish "$stack_path" failed; then + log_msg "ERROR" "The failed deploy of git stack $name could not be recorded" + fi + save_result "failed" $up_exit "gitdeploy" + log_msg "ERROR" "Failed to deploy git stack $name (exit code: $up_exit)" + echo "" + echo "✗ Stack $name failed to start (exit code: $up_exit). The commit is recorded as failed (compose-git status shows it)." + echo "Fix it and deploy again, or deploy a known good commit with --commit." + fi + ;; + stop) if [ "$debug" = true ]; then log_msg "DEBUG" "${compose_base[*]} -p $name stop" @@ -537,7 +712,7 @@ case $command in esac case $command in - up|down|pull|update|stop|logs) + up|down|pull|update|stop|logs|gitdeploy) exit "${operation_exit_code:-0}" ;; esac \ No newline at end of file diff --git a/source/compose.manager/scripts/git_stack.php b/source/compose.manager/scripts/git_stack.php new file mode 100644 index 00000000..5d588584 --- /dev/null +++ b/source/compose.manager/scripts/git_stack.php @@ -0,0 +1,86 @@ + [--commit ] [--save-local-changes] [--profile ]... + * Fetch, check out and check the commit to deploy. Progress goes to stderr; + * on success the previously checked-out commit is printed on stdout. + * git_stack.php compose-args + * Print the stack's -f / --env-file arguments, each followed by a NUL byte. + * git_stack.php restore + * Check the clone out at again (after a failed pull). + * git_stack.php finish success|failed + * Record how the deploy ended. + * + * Exit status: 0 on success, 1 with a message on stderr otherwise. + */ + +declare(strict_types=1); + +require_once '/usr/local/emhttp/plugins/compose.manager/include/GitDeploy.php'; + +$say = static function (string $message): void { + fwrite(STDERR, $message . "\n"); +}; + +$args = array_slice($argv, 1); +$action = $args[0] ?? ''; +$stackPath = $args[1] ?? ''; +if ($stackPath === '' || !is_dir($stackPath)) { + $say('git_stack.php: a stack folder is required.'); + exit(1); +} +$deploy = new GitDeploy($stackPath, $say); + +try { + switch ($action) { + case 'prepare': + $projectName = $args[2] ?? ''; + if ($projectName === '') { + throw new RuntimeException('git_stack.php prepare: the project name is required.'); + } + $commit = null; + $saveLocalChanges = false; + $profiles = []; + for ($i = 3; $i < count($args); $i++) { + if ($args[$i] === '--commit' && isset($args[$i + 1])) { + $commit = $args[++$i]; + } elseif ($args[$i] === '--save-local-changes') { + $saveLocalChanges = true; + } elseif ($args[$i] === '--profile' && isset($args[$i + 1])) { + $profiles[] = $args[++$i]; + } else { + throw new RuntimeException("git_stack.php prepare: unknown option {$args[$i]}"); + } + } + echo $deploy->prepare($projectName, $profiles, $commit, $saveLocalChanges) . "\n"; + break; + + case 'compose-args': + foreach ($deploy->composeArgs() as $arg) { + echo $arg . "\0"; + } + break; + + case 'restore': + $deploy->restore($args[2] ?? ''); + break; + + case 'finish': + $outcome = $args[2] ?? ''; + if ($outcome !== 'success' && $outcome !== 'failed') { + throw new RuntimeException('git_stack.php finish: expected success or failed.'); + } + $deploy->finish($outcome === 'success'); + break; + + default: + throw new RuntimeException("git_stack.php: unknown action '$action'."); + } +} catch (Throwable $error) { + $say('✗ ' . $error->getMessage()); + exit(1); +} +exit(0); diff --git a/tests/unit/GitDeployTest.php b/tests/unit/GitDeployTest.php new file mode 100644 index 00000000..af13ebae --- /dev/null +++ b/tests/unit/GitDeployTest.php @@ -0,0 +1,384 @@ +docker = new FakeDocker(); + $this->docker->setConfig(['name' => 'whoami', 'services' => ['whoami' => ['image' => 'busybox']]]); + + $this->mnt = COMPOSE_GIT_MNT_DIR; + FakeDocker::removeTree($this->mnt); + mkdir($this->mnt . '/user/appdata', 0755, true); + mkdir($this->mnt . '/user/repos', 0755, true); + $this->setArrayState('STARTED'); + file_put_contents(COMPOSE_MOUNTS_FILE, "rootfs {$this->mnt} rootfs rw 0 0\nshfs {$this->mnt}/user fuse.shfs rw 0 0\n"); + + $this->upstream = $this->mnt . '/user/repos/upstream.git'; + $this->author = sys_get_temp_dir() . '/compose_git_deploy_author'; + FakeDocker::removeTree($this->author); + $this->git(['init', '-q', '--bare', '-b', 'main', $this->upstream], null); + $this->git(['clone', '-q', $this->upstream, $this->author], null); + $this->git(['checkout', '-q', '-b', 'main'], $this->author); + $this->writeAndPush(['whoami/compose.yaml' => "services:\n whoami:\n image: traefik/whoami\n"], 'first'); + + // A git stack as compose-git add would leave it. + $this->composeRoot = sys_get_temp_dir() . '/compose_git_deploy_projects'; + FakeDocker::removeTree($this->composeRoot); + $this->stackDir = $this->composeRoot . '/whoami'; + mkdir($this->stackDir, 0755, true); + $settings = GitStackSettings::createNew($this->upstream, 'main', 'whoami/compose.yaml', $this->mnt . '/user/appdata/git', 'whoami'); + $settings->save($this->stackDir); + $this->clone = new GitClone($settings); + $first = $this->clone->create(); + file_put_contents($this->stackDir . '/indirect', $settings->composeFileInClone()); + file_put_contents($this->stackDir . '/indirect_mode', 'file'); + file_put_contents($this->stackDir . '/project_name', 'whoami'); + (new GitStackState($first, null))->save($this->stackDir); + } + + protected function tearDown(): void + { + $this->docker->cleanUp(); + FakeDocker::removeTree($this->mnt); + FakeDocker::removeTree($this->author); + FakeDocker::removeTree($this->composeRoot); + parent::tearDown(); + } + + public function testPrepareChecksOutTheLatestCommitAndReturnsThePreviousOne(): void + { + $before = $this->clone->checkedOutCommit(); + $this->writeAndPush(['whoami/compose.yaml' => "services:\n whoami:\n image: traefik/whoami:v2\n"], 'bump'); + + $previous = $this->deploy()->prepare('whoami', [], null, false); + + $this->assertSame($before, $previous); + $this->assertSame($this->upstreamHead(), $this->clone->checkedOutCommit()); + } + + public function testFailedCheckPutsThePreviousCommitBack(): void + { + $before = $this->clone->checkedOutCommit(); + $this->writeAndPush(['whoami/compose.yaml' => "services:\n whoami:\n\timage: broken\n"], 'broken'); + $this->docker->setConfig([], "yaml: found character that cannot start any token\n", 1); + + try { + $this->deploy()->prepare('whoami', [], null, false); + $this->fail('Deployed a broken compose file'); + } catch (RuntimeException $error) { + $this->assertStringContainsString('cannot start any token', $error->getMessage()); + } + $this->assertSame($before, $this->clone->checkedOutCommit()); + } + + public function testComposeFileRemovedUpstreamIsNotDeployed(): void + { + $before = $this->clone->checkedOutCommit(); + $this->git(['rm', '-q', 'whoami/compose.yaml'], $this->author); + $this->commitAndPush('remove'); + + try { + $this->deploy()->prepare('whoami', [], null, false); + $this->fail('Deployed a commit without the compose file'); + } catch (RuntimeException $error) { + $this->assertStringContainsString('does not exist at commit', $error->getMessage()); + } + $this->assertSame($before, $this->clone->checkedOutCommit()); + $this->assertSame([], array_filter($this->docker->calls(), static fn(string $c): bool => str_contains($c, 'config'))); + } + + public function testTheCheckRunsComposeWithTheStacksFilesEnvAndProfiles(): void + { + file_put_contents($this->stackDir . '/.env', "TZ=Europe/London\n"); + $this->writeAndPush(['whoami/compose.yaml' => "services:\n whoami:\n image: traefik/whoami:v2\n"], 'bump'); + + $this->deploy()->prepare('whoami', ['media'], null, false); + + $cloneDir = $this->clone->settings()->cloneDir; + $this->assertContains( + "compose -f $cloneDir/whoami/compose.yaml --env-file {$this->stackDir}/.env --profile media -p whoami config --format json", + $this->docker->calls() + ); + } + + public function testLocalChangesStopTheDeployAndAreKept(): void + { + $file = $this->clone->settings()->cloneDir . '/whoami/compose.yaml'; + file_put_contents($file, "hotfix\n"); + $this->writeAndPush(['whoami/other.txt' => "x\n"], 'upstream change'); + + try { + $this->deploy()->prepare('whoami', [], null, false); + $this->fail('Deployed over local changes'); + } catch (RuntimeException $error) { + $this->assertStringContainsString('whoami/compose.yaml', $error->getMessage()); + } + $this->assertSame("hotfix\n", file_get_contents($file)); + } + + public function testLocalChangesCanBeSavedAsAPatchAndThenDeployed(): void + { + file_put_contents($this->clone->settings()->cloneDir . '/whoami/compose.yaml', "hotfix\n"); + $this->writeAndPush(['whoami/other.txt' => "x\n"], 'upstream change'); + + $this->deploy()->prepare('whoami', [], null, true); + + $patches = glob($this->stackDir . '/git-changes/*.patch') ?: []; + $this->assertCount(1, $patches); + $this->assertStringContainsString('+hotfix', (string) file_get_contents($patches[0])); + $this->assertSame($this->upstreamHead(), $this->clone->checkedOutCommit()); + } + + public function testCommitMadeByHandInTheCloneStopsTheDeploy(): void + { + $dir = $this->clone->settings()->cloneDir; + file_put_contents($dir . '/whoami/compose.yaml', "by hand\n"); + $this->git(['-c', 'user.name=t', '-c', 'user.email=t@example.invalid', 'commit', '-q', '-am', 'by hand'], $dir); + + $this->expectException(RuntimeException::class); + $this->expectExceptionMessage('by hand'); + $this->deploy()->prepare('whoami', [], null, false); + } + + public function testCommitMadeByHandIsSavedInThePatchWithTheOption(): void + { + $dir = $this->clone->settings()->cloneDir; + file_put_contents($dir . '/whoami/compose.yaml', "by hand\n"); + $this->git(['-c', 'user.name=t', '-c', 'user.email=t@example.invalid', 'commit', '-q', '-am', 'by hand'], $dir); + $this->writeAndPush(['whoami/other.txt' => "x\n"], 'upstream change'); + + $this->deploy()->prepare('whoami', [], null, true); + + $patches = glob($this->stackDir . '/git-changes/*.patch') ?: []; + $this->assertCount(1, $patches); + $this->assertStringContainsString('+by hand', (string) file_get_contents($patches[0])); + $this->assertSame($this->upstreamHead(), $this->clone->checkedOutCommit()); + } + + public function testCloneLeftAtANewerCommitOfTheBranchIsNotACommitMadeByHand(): void + { + // A deploy interrupted after the checkout (Ctrl-C, or a put-back that + // failed) leaves the clone at a branch commit that was never deployed. + $this->writeAndPush(['whoami/compose.yaml' => "services:\n whoami:\n image: traefik/whoami:v2\n"], 'bump'); + $this->clone->checkOut($this->clone->fetch()); + + $this->deploy()->prepare('whoami', [], null, false); + + $this->assertSame($this->upstreamHead(), $this->clone->checkedOutCommit()); + $this->assertDirectoryDoesNotExist($this->stackDir . '/git-changes'); + } + + public function testLocalChangesAreKeptWhenTheFetchFails(): void + { + $file = $this->clone->settings()->cloneDir . '/whoami/compose.yaml'; + file_put_contents($file, "hotfix\n"); + rename($this->upstream, $this->upstream . '.away'); + + try { + $this->deploy()->prepare('whoami', [], null, true); + $this->fail('Deployed without reaching the repository'); + } catch (RuntimeException $error) { + $this->assertStringContainsString('Could not fetch', $error->getMessage()); + } finally { + rename($this->upstream . '.away', $this->upstream); + } + $this->assertSame("hotfix\n", file_get_contents($file)); + $this->assertDirectoryDoesNotExist($this->stackDir . '/git-changes'); + } + + public function testComposePathEditedInGitJsonButNotInTheIndirectFileIsRefused(): void + { + $this->writeAndPush(['other/compose.yaml' => "services:\n other:\n image: busybox\n"], 'second stack'); + $json = $this->stackDir . '/git.json'; + file_put_contents($json, str_replace('whoami/compose.yaml', 'other/compose.yaml', (string) file_get_contents($json))); + + try { + $this->deploy()->prepare('whoami', [], null, false); + $this->fail('Checked one compose file and would have deployed another'); + } catch (RuntimeException $error) { + $this->assertStringContainsString('name different compose files', $error->getMessage()); + } + } + + public function testASpecificOlderCommitCanBeDeployed(): void + { + $first = $this->clone->checkedOutCommit(); + $this->writeAndPush(['whoami/compose.yaml' => "services:\n whoami:\n image: traefik/whoami:v2\n"], 'bump'); + $this->deploy()->prepare('whoami', [], null, false); + $this->deploy()->finish(true); + + $this->deploy()->prepare('whoami', [], $first, false); + + $this->assertSame($first, $this->clone->checkedOutCommit()); + } + + public function testCommitOptionMustBeAFullCommitId(): void + { + $this->expectException(RuntimeException::class); + $this->deploy()->prepare('whoami', [], '--upload-pack=touch /tmp/x', false); + } + + public function testNothingHappensWhileTheArrayIsStopped(): void + { + $this->setArrayState('STOPPED'); + $this->expectException(RuntimeException::class); + $this->expectExceptionMessage('array is not started'); + $this->deploy()->prepare('whoami', [], null, false); + } + + public function testRestorePutsTheOldCommitBack(): void + { + $previous = $this->clone->checkedOutCommit(); + $this->writeAndPush(['whoami/compose.yaml' => "services: {}\n"], 'change'); + $this->deploy()->prepare('whoami', [], null, false); + + $this->deploy()->restore($previous); + + $this->assertSame($previous, $this->clone->checkedOutCommit()); + } + + public function testFinishRecordsSuccessAndFailure(): void + { + $first = $this->clone->checkedOutCommit(); + $this->writeAndPush(['whoami/compose.yaml' => "services: {}\n"], 'change'); + $this->deploy()->prepare('whoami', [], null, false); + + $this->deploy()->finish(false); + $state = GitStackState::load($this->stackDir); + $this->assertSame($first, $state->deployedCommit); + $this->assertSame($this->upstreamHead(), $state->failedCommit); + + $this->deploy()->finish(true); + $state = GitStackState::load($this->stackDir); + $this->assertSame($this->upstreamHead(), $state->deployedCommit); + $this->assertNull($state->failedCommit); + } + + public function testDeployAfterAFailedUpIsNotMistakenForACommitMadeByHand(): void + { + $this->writeAndPush(['whoami/compose.yaml' => "services:\n whoami:\n image: traefik/whoami:v2\n"], 'bump'); + $this->deploy()->prepare('whoami', [], null, false); + $this->deploy()->finish(false); + + $this->writeAndPush(['whoami/compose.yaml' => "services:\n whoami:\n image: traefik/whoami:v3\n"], 'fix'); + $this->deploy()->prepare('whoami', [], null, false); + + $this->assertSame($this->upstreamHead(), $this->clone->checkedOutCommit()); + $this->assertDirectoryDoesNotExist($this->stackDir . '/git-changes'); + } + + public function testEnvKeptInTheRepositoryIsUsedAndChecked(): void + { + $this->writeAndPush(['whoami/.env' => "X=from-repo\n", 'whoami/.env.example' => "X=\n"], 'repo env'); + $this->deploy()->prepare('whoami', [], null, false); + + $args = $this->deploy()->composeArgs(); + + $envFile = $args[array_search('--env-file', $args, true) + 1]; + $this->assertSame(realpath($this->clone->settings()->cloneDir . '/whoami/.env'), realpath($envFile)); + } + + public function testComposeArgsUseTheCloneAndTheStackFoldersEnv(): void + { + file_put_contents($this->stackDir . '/.env', "A=1\n"); + $args = $this->deploy()->composeArgs(); + $this->assertSame( + ['-f', $this->clone->settings()->composeFileInClone(), '--env-file', realpath($this->stackDir . '/.env')], + $args + ); + } + + public function testComposeFileReachedThroughASymlinkOutsideTheCloneIsRefused(): void + { + $outside = $this->mnt . '/user/appdata/outside.yaml'; + file_put_contents($outside, "services: {}\n"); + $this->git(['rm', '-q', 'whoami/compose.yaml'], $this->author); + @mkdir($this->author . '/whoami'); + symlink($outside, $this->author . '/whoami/compose.yaml'); + $this->git(['add', 'whoami/compose.yaml'], $this->author); + $this->commitAndPush('symlink out'); + $before = $this->clone->checkedOutCommit(); + + try { + $this->deploy()->prepare('whoami', [], null, false); + $this->fail('Used a compose file outside the clone'); + } catch (RuntimeException $error) { + $this->assertStringContainsString('leads outside the repository', $error->getMessage()); + } + $this->assertSame($before, $this->clone->checkedOutCommit()); + } + + // ----- helpers ----- + + private function deploy(): GitDeploy + { + return new GitDeploy($this->stackDir); + } + + /** @param array $files */ + private function writeAndPush(array $files, string $message): void + { + foreach ($files as $path => $content) { + $full = $this->author . '/' . $path; + if (!is_dir(dirname($full))) { + mkdir(dirname($full), 0755, true); + } + file_put_contents($full, $content); + $this->git(['add', '--', $path], $this->author); + } + $this->commitAndPush($message); + } + + private function commitAndPush(string $message): void + { + $this->git(['-c', 'user.name=t', '-c', 'user.email=t@example.invalid', 'commit', '-q', '-m', $message], $this->author); + $this->git(['push', '-q', 'origin', 'main'], $this->author); + } + + private function upstreamHead(): string + { + return trim(GitCommand::run(['--git-dir=' . $this->upstream, 'rev-parse', 'refs/heads/main'])->stdout); + } + + /** @param string[] $args */ + private function git(array $args, ?string $dir): void + { + $result = GitCommand::run($args, $dir); + $this->assertTrue($result->succeeded(), 'git ' . implode(' ', $args) . ': ' . $result->stderr); + } + + private function setArrayState(string $state): void + { + file_put_contents(COMPOSE_UNRAID_VAR_INI, "mdState=\"$state\"\n"); + } +} diff --git a/tests/unit/compose.bats b/tests/unit/compose.bats index d31fb1d1..8c0cc4b7 100644 --- a/tests/unit/compose.bats +++ b/tests/unit/compose.bats @@ -190,7 +190,7 @@ test_setup() { } @test "compose.sh mutating commands have explicit final exit propagation" { - run grep -E '^\s*up\|down\|pull\|update\|stop\|logs\)' "$COMPOSE_SCRIPT" + run grep -E '^\s*up\|down\|pull\|update\|stop\|logs(\|gitdeploy)?\)' "$COMPOSE_SCRIPT" assert_success run grep -E '^\s*exit "\$\{operation_exit_code:-0\}"' "$COMPOSE_SCRIPT" diff --git a/tests/unit/gitdeploy.bats b/tests/unit/gitdeploy.bats new file mode 100644 index 00000000..eb2d8ed1 --- /dev/null +++ b/tests/unit/gitdeploy.bats @@ -0,0 +1,230 @@ +#!/usr/bin/env bats +# +# compose.sh gitdeploy: the order of steps, and that nothing touches a +# container when an earlier step fails. git_stack.php (tested in PHPUnit) and +# docker are replaced by scripts on PATH that record every call. +# + +load '../bats_setup.bash' + +COMPOSE_SCRIPT="$BATS_TEST_DIRNAME/../../source/compose.manager/scripts/compose.sh" + +test_setup() { + # The framework exports mock docker/php functions; these tests use scripts on PATH instead. + unset -f docker php 2>/dev/null || true + export COMPOSE_LOCK_TIMEOUT=5 + REAL_PHP="$(command -v php)" + export REAL_PHP + export CALLS="$TEST_TEMP_DIR/calls.log" + export IMAGES_CALLS="$TEST_TEMP_DIR/images-calls" + : > "$CALLS" + echo 0 > "$IMAGES_CALLS" + export PREPARE_EXIT=0 PULL_EXIT=0 BUILD_EXIT=0 UP_EXIT=0 + export COMPOSE_ARGS_EXIT=0 RESTORE_EXIT=0 + export PREPARE_OUTPUT="0000000000000000000000000000000000000001" + + STACK="$TEST_TEMP_DIR/whoami" + mkdir -p "$STACK" "$TEST_TEMP_DIR/bin" + echo '{}' > "$STACK/git.json" + + # php: answer for git_stack.php, pass everything else to the real php. + cat > "$TEST_TEMP_DIR/bin/php" <<'SHIM' +#!/bin/bash +if [ "$1" = "-d" ]; then + shift 2 +fi +case "$1" in + */git_stack.php) + shift + echo "git_stack $*" >> "$CALLS" + case "$1" in + prepare) + [ "$PREPARE_EXIT" = 0 ] || { echo "✗ check failed" >&2; exit "$PREPARE_EXIT"; } + echo "$PREPARE_OUTPUT" + ;; + compose-args) + [ "$COMPOSE_ARGS_EXIT" = 0 ] || { echo "✗ no compose file" >&2; exit "$COMPOSE_ARGS_EXIT"; } + printf -- '-f\0/clone/whoami/compose.yaml\0--env-file\0/stack/.env\0' + ;; + restore) + exit "$RESTORE_EXIT" + ;; + finish) + exit "${FINISH_EXIT:-0}" + ;; + esac + exit 0 + ;; +esac +exec "$REAL_PHP" "$@" +SHIM + + # docker: record the call; images -q answers "before" then "after". + cat > "$TEST_TEMP_DIR/bin/docker" <<'SHIM' +#!/bin/bash +echo "docker $*" >> "$CALLS" +for arg in "$@"; do + case "$arg" in + images) + n=$(cat "$IMAGES_CALLS"); echo $((n + 1)) > "$IMAGES_CALLS" + if [ "$n" = 0 ]; then printf 'img-old\nimg-kept\n'; else printf 'img-kept\nimg-new\n'; fi + exit 0 ;; + pull) exit "$PULL_EXIT" ;; + build) exit "$BUILD_EXIT" ;; + up) exit "$UP_EXIT" ;; + esac +done +exit 0 +SHIM + chmod +x "$TEST_TEMP_DIR/bin/php" "$TEST_TEMP_DIR/bin/docker" + export PATH="$TEST_TEMP_DIR/bin:$PATH" +} + +run_gitdeploy() { + run bash "$COMPOSE_SCRIPT" -c gitdeploy -p whoami -s "$STACK" "$@" +} + +calls_matching() { + grep -c -- "$1" "$CALLS" || true +} + +@test "gitdeploy refuses a stack that is not a git stack" { + rm "$STACK/git.json" + run_gitdeploy + [ "$status" -eq 1 ] + [[ "$output" == *"not a git stack"* ]] + [ "$(calls_matching 'docker')" -eq 0 ] +} + +@test "gitdeploy waits for the stack's lock and changes nothing when it stays taken" { + # Another operation on the stack (a Start from the web UI, say) holds its lock. + mkdir -p /var/run/compose.manager + exec 8>/var/run/compose.manager/whoami.lock + flock 8 + export COMPOSE_LOCK_TIMEOUT=1 + run_gitdeploy + exec 8>&- + [ "$status" -eq 1 ] + [[ "$output" == *"Another operation is already in progress"* ]] + [ "$(calls_matching 'git_stack')" -eq 0 ] + [ "$(calls_matching 'docker')" -eq 0 ] +} + +@test "gitdeploy stops before any container when the git checks fail" { + export PREPARE_EXIT=1 + run_gitdeploy + [ "$status" -eq 1 ] + [[ "$output" == *"No container was changed"* ]] + [ "$(calls_matching 'docker')" -eq 0 ] + grep -q '"result":"failed"' "$STACK/last_result.json" +} + +@test "gitdeploy puts the previous commit back when the pull fails, and never runs up" { + export PULL_EXIT=1 + run_gitdeploy + [ "$status" -eq 1 ] + grep -q "git_stack restore $STACK 0000000000000000000000000000000000000001" "$CALLS" + [ "$(calls_matching ' up ')" -eq 0 ] + [ "$(calls_matching 'finish')" -eq 0 ] +} + +@test "gitdeploy puts the previous commit back when the build fails, and never runs up" { + export BUILD_EXIT=1 + run_gitdeploy + [ "$status" -eq 1 ] + [[ "$output" == *"Building images for whoami failed"* ]] + grep -q "git_stack restore $STACK 0000000000000000000000000000000000000001" "$CALLS" + [ "$(calls_matching ' up ')" -eq 0 ] + [ "$(calls_matching 'finish')" -eq 0 ] +} + +@test "gitdeploy says so when the previous commit cannot be put back" { + export PULL_EXIT=1 RESTORE_EXIT=1 + run_gitdeploy + [ "$status" -eq 1 ] + [[ "$output" == *"could not be put back"* ]] + [[ "$output" != *"was put back"* ]] + [ "$(calls_matching ' up ')" -eq 0 ] +} + +@test "gitdeploy stops when the git checks print something other than a commit id" { + export PREPARE_OUTPUT="PHP Notice: something" + run_gitdeploy + [ "$status" -eq 1 ] + [ "$(calls_matching 'docker')" -eq 0 ] + [ "$(calls_matching 'restore')" -eq 0 ] +} + +@test "gitdeploy stops before pulling when the compose files cannot be read" { + export COMPOSE_ARGS_EXIT=1 + run_gitdeploy + [ "$status" -eq 1 ] + [ "$(calls_matching 'docker')" -eq 0 ] + grep -q "git_stack restore $STACK 0000000000000000000000000000000000000001" "$CALLS" +} + +@test "gitdeploy records a failed up and does not roll back or remove images" { + export UP_EXIT=3 + run_gitdeploy + [ "$status" -eq 3 ] + grep -q "git_stack finish $STACK failed" "$CALLS" + [ "$(calls_matching 'restore')" -eq 0 ] + [ "$(calls_matching 'rmi')" -eq 0 ] + [[ "$output" == *"The commit is recorded as failed"* ]] +} + +@test "gitdeploy says so when a successful deploy cannot be recorded, and still succeeds" { + export FINISH_EXIT=1 + run_gitdeploy + [ "$status" -eq 0 ] + [[ "$output" == *"the deployed commit could not be recorded"* ]] + [[ "$output" == *"deployed successfully"* ]] +} + +@test "gitdeploy still reports a failed up when the failure cannot be recorded" { + export UP_EXIT=3 FINISH_EXIT=1 + run_gitdeploy + [ "$status" -eq 3 ] + [[ "$output" == *"failed to start"* ]] + [ "$(calls_matching 'restore')" -eq 0 ] +} + +@test "gitdeploy runs the steps in order and removes only superseded images" { + run_gitdeploy + [ "$status" -eq 0 ] + [[ "$output" == *"deployed successfully"* ]] + + prepare_line=$(grep -n 'git_stack prepare' "$CALLS" | cut -d: -f1) + pull_line=$(grep -n ' pull --ignore-buildable' "$CALLS" | cut -d: -f1) + build_line=$(grep -n ' build$' "$CALLS" | cut -d: -f1) + up_line=$(grep -n ' up ' "$CALLS" | cut -d: -f1) + finish_line=$(grep -n 'git_stack finish' "$CALLS" | cut -d: -f1) + [ "$prepare_line" -lt "$pull_line" ] + [ "$pull_line" -lt "$build_line" ] + [ "$build_line" -lt "$up_line" ] + [ "$up_line" -lt "$finish_line" ] + + grep -q "docker compose -f /clone/whoami/compose.yaml --env-file /stack/.env -p whoami up -d --remove-orphans" "$CALLS" + grep -q "git_stack finish $STACK success" "$CALLS" + grep -q "docker rmi img-old$" "$CALLS" + grep -q '"operation":"gitdeploy"' "$STACK/last_result.json" +} + +@test "gitdeploy passes the commit, save-local-changes and profiles to the git checks" { + run_gitdeploy --git-commit 0123456789abcdef0123456789abcdef01234567 --save-local-changes -g media + [ "$status" -eq 0 ] + grep -q "git_stack prepare $STACK whoami --commit 0123456789abcdef0123456789abcdef01234567 --save-local-changes --profile media" "$CALLS" + grep -q -- "--profile media -p whoami up" "$CALLS" +} + +@test "gitdeploy waits for healthy containers when asked" { + run_gitdeploy --wait --wait-timeout 90 + [ "$status" -eq 0 ] + grep -q -- "up -d --remove-orphans --wait --wait-timeout 90" "$CALLS" +} + +@test "gitdeploy never rebuilds during up, so a build failure cannot happen part-way" { + run_gitdeploy --build + [ "$status" -eq 0 ] + [ "$(grep -c -- ' up .*--build' "$CALLS" || true)" -eq 0 ] +} From 9aee9eaf8c06c1dd456061dfb3962010a93ed785 Mon Sep 17 00:00:00 2001 From: Grant Emsley Date: Sat, 3 Oct 2026 14:33:05 -0400 Subject: [PATCH 010/109] feat(git): recreate a git stack's containers when its folder changes Plain "up -d" recreates a container only when its compose definition changed, so a commit that only edits a bind-mounted config file was checked out but never took effect. A new per-stack setting in git.json, recreateOnFolderChange (on by default), makes the deploy add --force-recreate when anything in the stack's folder changed since the previous commit. Recreate, not restart: a restart keeps a single-file mount on the old copy of the file. If the commits cannot be compared, the previous commit is put back and no container is changed, the same as a failed pull or build. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_011iUWDG8wTp8MhsFDqNC4NL --- source/compose.manager/include/GitClone.php | 14 +++++ source/compose.manager/include/GitDeploy.php | 46 +++++++++++++++ .../include/GitStackSettings.php | 41 ++++++++++--- source/compose.manager/scripts/compose.sh | 30 +++++++++- source/compose.manager/scripts/git_stack.php | 8 +++ tests/unit/GitDeployTest.php | 59 +++++++++++++++++++ tests/unit/GitStackSettingsTest.php | 19 ++++++ .../fixtures/git-stack-settings/format-1.json | 3 +- tests/unit/gitdeploy.bats | 33 ++++++++++- 9 files changed, 242 insertions(+), 11 deletions(-) diff --git a/source/compose.manager/include/GitClone.php b/source/compose.manager/include/GitClone.php index 1d2871c9..f25c831c 100644 --- a/source/compose.manager/include/GitClone.php +++ b/source/compose.manager/include/GitClone.php @@ -578,6 +578,20 @@ public function lastCommitSharedWithBranch(string $commit): ?string return $shared; } + /** + * Whether the clone has a commit (after a rewritten branch, an old commit may be gone). + */ + public function hasCommit(string $commit): bool + { + $this->assertOwned(); + $result = $this->git( + ['cat-file', '-e', $this->commitArgument($commit) . '^{commit}'], + $this->settings->cloneDir, + 30 + ); + return $result->succeeded(); + } + /** * Whether a string looks like a full commit id (SHA-1 or SHA-256). */ diff --git a/source/compose.manager/include/GitDeploy.php b/source/compose.manager/include/GitDeploy.php index 08c321e2..bf4e064a 100644 --- a/source/compose.manager/include/GitDeploy.php +++ b/source/compose.manager/include/GitDeploy.php @@ -87,6 +87,52 @@ public function prepare(string $projectName, array $profiles, ?string $commit, b return $previous; } + /** + * Extra arguments for "up" after prepare(). + * + * Plain "up -d" recreates a container only when its compose definition + * changed, so a commit that only edits a bind-mounted config file would + * leave the containers on the old file. When the stack's setting is on and + * anything in the stack's folder changed, the whole stack is recreated + * (not restarted: a restart keeps a single-file mount on the old copy of + * the file). + * + * "Changed" is measured from both the previously checked-out commit and + * the last successfully deployed one. After a failed "up" the two differ, + * and the running containers may hold either commit's files. + * + * @param string $previousCommit The commit prepare() returned + * @return string[] ['--force-recreate'] or [] + */ + public function upArguments(string $previousCommit): array + { + $settings = $this->loadSettings(); + if (!$settings->recreateOnFolderChange) { + return []; + } + $clone = new GitClone($settings); + $current = $clone->checkedOutCommit(); + + $compareFrom = [$previousCommit]; + $deployed = GitStackState::load($this->stackDir)->deployedCommit; + if ($deployed !== null && $deployed !== $previousCommit) { + $compareFrom[] = $deployed; + } + foreach ($compareFrom as $from) { + if ($from !== $current && !$clone->hasCommit($from)) { + // The branch was rewritten and git has since pruned the old + // commit, so the change cannot be measured: assume one. + ($this->say)('The commit ' . substr($from, 0, 12) . ' is no longer in the clone, so every container is recreated.'); + return ['--force-recreate']; + } + if ($from !== $current && $clone->stackFilesChanged($from, $current)) { + ($this->say)("Files in the stack's folder changed, so every container is recreated."); + return ['--force-recreate']; + } + } + return []; + } + /** * Put the clone back at a commit (after a failed pull, before any container changed). */ diff --git a/source/compose.manager/include/GitStackSettings.php b/source/compose.manager/include/GitStackSettings.php index 80414390..2c79bf92 100644 --- a/source/compose.manager/include/GitStackSettings.php +++ b/source/compose.manager/include/GitStackSettings.php @@ -34,14 +34,20 @@ final class GitStackSettings public const FORMAT_VERSION = 1; /** @var string[] The keys git.json must contain, no more and no fewer. */ - private const KEYS = ['version', 'url', 'branch', 'composePath', 'cloneId', 'cloneDir']; + private const KEYS = ['version', 'url', 'branch', 'composePath', 'cloneId', 'cloneDir', 'recreateOnFolderChange']; + /** + * @param bool $recreateOnFolderChange Recreate the stack's containers when a deploy changes + * anything in the stack's folder, not only when the compose definition changes. A commit + * that only edits a bind-mounted config file then takes effect. + */ private function __construct( public readonly string $url, public readonly string $branch, public readonly string $composePath, public readonly string $cloneId, - public readonly string $cloneDir + public readonly string $cloneDir, + public readonly bool $recreateOnFolderChange ) { } @@ -55,12 +61,21 @@ public static function createNew( string $branch, string $composePath, string $clonesRoot, - string $stackFolderName + string $stackFolderName, + bool $recreateOnFolderChange = true ): self { GitPathGuard::assertValidClonesRoot($clonesRoot); $cloneId = bin2hex(random_bytes(8)); $cloneDir = $clonesRoot . '/' . self::cloneFolderName($stackFolderName, $cloneId); - return self::fromValues($url, $branch, $composePath, $cloneId, $cloneDir); + return self::fromValues($url, $branch, $composePath, $cloneId, $cloneDir, $recreateOnFolderChange); + } + + /** + * The same settings with "recreate on any change in the stack's folder" turned on or off. + */ + public function withRecreateOnFolderChange(bool $recreateOnFolderChange): self + { + return new self($this->url, $this->branch, $this->composePath, $this->cloneId, $this->cloneDir, $recreateOnFolderChange); } /** @@ -114,9 +129,19 @@ public static function load(string $stackDir): ?self throw new RuntimeException("$file: '$key' must be a string."); } } + if (!is_bool($data['recreateOnFolderChange'])) { + throw new RuntimeException("$file: 'recreateOnFolderChange' must be true or false."); + } try { - return self::fromValues($data['url'], $data['branch'], $data['composePath'], $data['cloneId'], $data['cloneDir']); + return self::fromValues( + $data['url'], + $data['branch'], + $data['composePath'], + $data['cloneId'], + $data['cloneDir'], + $data['recreateOnFolderChange'] + ); } catch (InvalidArgumentException $error) { throw new RuntimeException("$file: " . $error->getMessage(), 0, $error); } @@ -145,6 +170,7 @@ public function save(string $stackDir): void 'composePath' => $this->composePath, 'cloneId' => $this->cloneId, 'cloneDir' => $this->cloneDir, + 'recreateOnFolderChange' => $this->recreateOnFolderChange, ], JSON_PRETTY_PRINT | JSON_UNESCAPED_SLASHES); if ($json === false) { throw new RuntimeException('Could not encode the git settings.'); @@ -327,7 +353,8 @@ private static function fromValues( string $branch, string $composePath, string $cloneId, - string $cloneDir + string $cloneDir, + bool $recreateOnFolderChange ): self { self::validateUrl($url); self::validateBranch($branch); @@ -344,6 +371,6 @@ private static function fromValues( throw new InvalidArgumentException("The clone folder $cloneDir does not belong to clone id $cloneId."); } - return new self($url, $branch, $composePath, $cloneId, $cloneDir); + return new self($url, $branch, $composePath, $cloneId, $cloneDir, $recreateOnFolderChange); } } diff --git a/source/compose.manager/scripts/compose.sh b/source/compose.manager/scripts/compose.sh index a3b25e9a..875901da 100755 --- a/source/compose.manager/scripts/compose.sh +++ b/source/compose.manager/scripts/compose.sh @@ -601,7 +601,35 @@ case $command in exit 1 fi - # 3. Start the stack at the new commit. + # 3. Ask whether every container should be recreated (the stack's + # "recreate on any change in its folder" setting). + if ! git_up_extra=$(git_stack up-arguments "$stack_path" "$previous_commit"); then + save_result "failed" 1 "gitdeploy" + log_msg "ERROR" "Could not compare commits for git stack $name" + echo "" + echo "✗ Stack $name was not deployed." + put_back_previous_commit + exit 1 + fi + # Only the two answers git_stack.php gives are accepted, so a stray line + # of output can never become an argument to up. + case "$git_up_extra" in + "") + ;; + "--force-recreate") + up_args+=("--force-recreate") + ;; + *) + save_result "failed" 1 "gitdeploy" + log_msg "ERROR" "git_stack.php up-arguments printed something unexpected for $name" + echo "" + echo "✗ Stack $name was not deployed." + put_back_previous_commit + exit 1 + ;; + esac + + # 4. Start the stack at the new commit. echo "" echo "Starting containers..." "${git_compose[@]}" -p "$name" up "${up_args[@]}" diff --git a/source/compose.manager/scripts/git_stack.php b/source/compose.manager/scripts/git_stack.php index 5d588584..104d2c7b 100644 --- a/source/compose.manager/scripts/git_stack.php +++ b/source/compose.manager/scripts/git_stack.php @@ -9,6 +9,8 @@ * on success the previously checked-out commit is printed on stdout. * git_stack.php compose-args * Print the stack's -f / --env-file arguments, each followed by a NUL byte. + * git_stack.php up-arguments + * Print the extra arguments for "up" (--force-recreate or nothing), one per line. * git_stack.php restore * Check the clone out at again (after a failed pull). * git_stack.php finish success|failed @@ -64,6 +66,12 @@ } break; + case 'up-arguments': + foreach ($deploy->upArguments($args[2] ?? '') as $arg) { + echo $arg . "\n"; + } + break; + case 'restore': $deploy->restore($args[2] ?? ''); break; diff --git a/tests/unit/GitDeployTest.php b/tests/unit/GitDeployTest.php index af13ebae..0a241fe3 100644 --- a/tests/unit/GitDeployTest.php +++ b/tests/unit/GitDeployTest.php @@ -284,6 +284,65 @@ public function testFinishRecordsSuccessAndFailure(): void $this->assertNull($state->failedCommit); } + // ----- recreate on any change in the stack's folder ----- + + public function testConfigOnlyChangeInTheStackFolderRecreatesEveryContainer(): void + { + $this->writeAndPush(['whoami/config.yml' => "level: debug\n"], 'config only'); + $previous = $this->deploy()->prepare('whoami', [], null, false); + + $this->assertSame(['--force-recreate'], $this->deploy()->upArguments($previous)); + } + + public function testChangeOutsideTheStackFolderRecreatesNothing(): void + { + $this->writeAndPush(['other/compose.yaml' => "services: {}\n"], 'another stack'); + $previous = $this->deploy()->prepare('whoami', [], null, false); + + $this->assertSame([], $this->deploy()->upArguments($previous)); + } + + public function testRedeployOfTheSameCommitRecreatesNothing(): void + { + $previous = $this->deploy()->prepare('whoami', [], null, false); + + $this->assertSame([], $this->deploy()->upArguments($previous)); + } + + public function testWithTheSettingOffNothingIsRecreated(): void + { + GitStackSettings::load($this->stackDir)->withRecreateOnFolderChange(false)->save($this->stackDir); + $this->writeAndPush(['whoami/config.yml' => "level: debug\n"], 'config only'); + $previous = $this->deploy()->prepare('whoami', [], null, false); + + $this->assertSame([], $this->deploy()->upArguments($previous)); + } + + public function testFolderChangeSinceTheDeployedCommitRecreatesAfterAFailedUp(): void + { + // A deployed; B edits a mounted config file and its up fails; C changes + // nothing in the folder. The containers may still have A's file. + $this->writeAndPush(['whoami/config.yml' => "level: debug\n"], 'B: config'); + $this->deploy()->prepare('whoami', [], null, false); + $this->deploy()->finish(false); + $this->writeAndPush(['other/compose.yaml' => "services: {}\n"], 'C: another stack'); + + $previous = $this->deploy()->prepare('whoami', [], null, false); + + $this->assertSame(['--force-recreate'], $this->deploy()->upArguments($previous)); + } + + public function testDeployedCommitNoLongerInTheCloneRecreatesInsteadOfFailing(): void + { + // The branch was rewritten and git pruned the deployed commit. + (new GitStackState(str_repeat('a', 40), null))->save($this->stackDir); + $this->writeAndPush(['other/compose.yaml' => "services: {}\n"], 'another stack'); + + $previous = $this->deploy()->prepare('whoami', [], null, false); + + $this->assertSame(['--force-recreate'], $this->deploy()->upArguments($previous)); + } + public function testDeployAfterAFailedUpIsNotMistakenForACommitMadeByHand(): void { $this->writeAndPush(['whoami/compose.yaml' => "services:\n whoami:\n image: traefik/whoami:v2\n"], 'bump'); diff --git a/tests/unit/GitStackSettingsTest.php b/tests/unit/GitStackSettingsTest.php index b028c6af..5f7c13c1 100644 --- a/tests/unit/GitStackSettingsTest.php +++ b/tests/unit/GitStackSettingsTest.php @@ -218,9 +218,23 @@ public function testSaveThenLoadGivesTheSameSettings(): void $this->assertSame($settings->composePath, $loaded->composePath); $this->assertSame($settings->cloneId, $loaded->cloneId); $this->assertSame($settings->cloneDir, $loaded->cloneDir); + $this->assertTrue($loaded->recreateOnFolderChange); $this->assertSame([], glob($this->stackDir . '/git.json.tmp-*') ?: []); } + public function testRecreateOnFolderChangeIsOnByDefaultAndCanBeTurnedOff(): void + { + $settings = $this->makeSettings(); + $this->assertTrue($settings->recreateOnFolderChange); + + $settings->withRecreateOnFolderChange(false)->save($this->stackDir); + $loaded = GitStackSettings::load($this->stackDir); + + $this->assertNotNull($loaded); + $this->assertFalse($loaded->recreateOnFolderChange); + $this->assertSame($settings->cloneDir, $loaded->cloneDir); + } + public function testStackWithoutGitJsonIsNotAGitStack(): void { $this->assertNull(GitStackSettings::load($this->stackDir)); @@ -256,6 +270,10 @@ public static function brokenFiles(): array $data['branch'] = 5; return (string) json_encode($data); }], + 'string instead of true or false' => [static function (array $data): string { + $data['recreateOnFolderChange'] = 'yes'; + return (string) json_encode($data); + }], 'hand-edited bad branch' => [static function (array $data): string { $data['branch'] = '--upload-pack=touch /tmp/x'; return (string) json_encode($data); @@ -325,6 +343,7 @@ public function testFormatOneLoadsWithTheBehaviourItWasWrittenWith(): void $this->assertSame('https://github.com/owner/repo.git', $settings->url); $this->assertSame('whoami/compose.yaml', $settings->composePath); + $this->assertTrue($settings->recreateOnFolderChange); } public function testSettingFromANewerVersionIsRefusedByName(): void diff --git a/tests/unit/fixtures/git-stack-settings/format-1.json b/tests/unit/fixtures/git-stack-settings/format-1.json index 56fca42b..2efc6327 100644 --- a/tests/unit/fixtures/git-stack-settings/format-1.json +++ b/tests/unit/fixtures/git-stack-settings/format-1.json @@ -4,5 +4,6 @@ "branch": "main", "composePath": "whoami/compose.yaml", "cloneId": "0123456789abcdef", - "cloneDir": "@MNT@/user/appdata/compose.manager/git/whoami-01234567" + "cloneDir": "@MNT@/user/appdata/compose.manager/git/whoami-01234567", + "recreateOnFolderChange": true } diff --git a/tests/unit/gitdeploy.bats b/tests/unit/gitdeploy.bats index eb2d8ed1..d06d461c 100644 --- a/tests/unit/gitdeploy.bats +++ b/tests/unit/gitdeploy.bats @@ -19,7 +19,7 @@ test_setup() { export IMAGES_CALLS="$TEST_TEMP_DIR/images-calls" : > "$CALLS" echo 0 > "$IMAGES_CALLS" - export PREPARE_EXIT=0 PULL_EXIT=0 BUILD_EXIT=0 UP_EXIT=0 + export PREPARE_EXIT=0 PULL_EXIT=0 BUILD_EXIT=0 UP_EXIT=0 UP_ARGUMENTS_EXIT=0 UP_ARGUMENTS="" export COMPOSE_ARGS_EXIT=0 RESTORE_EXIT=0 export PREPARE_OUTPUT="0000000000000000000000000000000000000001" @@ -52,6 +52,10 @@ case "$1" in finish) exit "${FINISH_EXIT:-0}" ;; + up-arguments) + [ "$UP_ARGUMENTS_EXIT" = 0 ] || { echo "✗ could not compare commits" >&2; exit "$UP_ARGUMENTS_EXIT"; } + printf '%s' "$UP_ARGUMENTS" + ;; esac exit 0 ;; @@ -204,7 +208,7 @@ calls_matching() { [ "$build_line" -lt "$up_line" ] [ "$up_line" -lt "$finish_line" ] - grep -q "docker compose -f /clone/whoami/compose.yaml --env-file /stack/.env -p whoami up -d --remove-orphans" "$CALLS" + grep -q "docker compose -f /clone/whoami/compose.yaml --env-file /stack/.env -p whoami up -d --remove-orphans$" "$CALLS" grep -q "git_stack finish $STACK success" "$CALLS" grep -q "docker rmi img-old$" "$CALLS" grep -q '"operation":"gitdeploy"' "$STACK/last_result.json" @@ -223,6 +227,31 @@ calls_matching() { grep -q -- "up -d --remove-orphans --wait --wait-timeout 90" "$CALLS" } +@test "gitdeploy recreates every container when the stack's folder changed" { + export UP_ARGUMENTS=$'--force-recreate\n' + run_gitdeploy --wait + [ "$status" -eq 0 ] + grep -q "git_stack up-arguments $STACK 0000000000000000000000000000000000000001" "$CALLS" + grep -q -- "up -d --remove-orphans --wait --force-recreate$" "$CALLS" +} + +@test "gitdeploy puts the previous commit back when it cannot tell whether to recreate, and never runs up" { + export UP_ARGUMENTS_EXIT=1 + run_gitdeploy + [ "$status" -eq 1 ] + grep -q "git_stack restore $STACK 0000000000000000000000000000000000000001" "$CALLS" + [ "$(calls_matching ' up ')" -eq 0 ] + [ "$(calls_matching 'finish')" -eq 0 ] +} + +@test "gitdeploy never passes unexpected up-arguments output to up" { + export UP_ARGUMENTS=$'PHP Notice: something\n--force-recreate\n' + run_gitdeploy + [ "$status" -eq 1 ] + grep -q "git_stack restore $STACK 0000000000000000000000000000000000000001" "$CALLS" + [ "$(calls_matching ' up ')" -eq 0 ] +} + @test "gitdeploy never rebuilds during up, so a build failure cannot happen part-way" { run_gitdeploy --build [ "$status" -eq 0 ] From a2a72f6cb6d3779f23fb45f452961d7d5fcd567e Mon Sep 17 00:00:00 2001 From: Grant Emsley Date: Sat, 3 Oct 2026 19:43:33 -0400 Subject: [PATCH 011/109] feat(git): add the compose-git command compose-git creates, deploys and looks after git stacks from the command line (scripts/compose-git): - add: clone a repository and make a new git stack from it; - convert: turn an existing stack into a git stack. Its folder, name, project name, .env and plugin override stay; the old compose file and indirect settings are moved into a dated backup folder, and a .env that sat next to an indirect stack's old compose file is copied in; - check: compare the deployed commit with the branch, changing nothing; - deploy: run compose.sh gitdeploy, with the stack's credential; - reclone: move the clone aside (never delete it: untracked data lives there) and clone again at the deployed commit, moving the old one back if the new clone fails; - status [--json]: what is deployed, failed and changed locally. Stacks are named by their exact folder, convert and reclone take compose.sh's lock for the stack, and the exit status is the result (0 ok, 1 failed, 2 usage, 3 check found a newer commit) so scripts and git hooks can use it. Cloning now also refuses, and removes the new clone, when the compose file is not on the branch. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_011iUWDG8wTp8MhsFDqNC4NL --- source/compose.manager/include/Defines.php | 2 + source/compose.manager/include/GitClone.php | 23 + .../include/GitStackManager.php | 476 +++++++++++++ source/compose.manager/scripts/compose-git | 3 + .../compose.manager/scripts/compose_git.php | 575 ++++++++++++++++ tests/bootstrap.php | 1 + tests/unit/GitStackManagerTest.php | 648 ++++++++++++++++++ 7 files changed, 1728 insertions(+) create mode 100644 source/compose.manager/include/GitStackManager.php create mode 100755 source/compose.manager/scripts/compose-git create mode 100644 source/compose.manager/scripts/compose_git.php create mode 100644 tests/unit/GitStackManagerTest.php diff --git a/source/compose.manager/include/Defines.php b/source/compose.manager/include/Defines.php index 5eaa5381..4a2ca601 100644 --- a/source/compose.manager/include/Defines.php +++ b/source/compose.manager/include/Defines.php @@ -45,6 +45,8 @@ function locate_compose_root($name) defined('COMPOSE_GIT_DEFAULT_CLONES_ROOT') || define('COMPOSE_GIT_DEFAULT_CLONES_ROOT', '/mnt/user/appdata/compose.manager/git'); // The docker CLI (the git deploy checks run it; tests point this at a scripted stand-in). defined('COMPOSE_DOCKER_BIN') || define('COMPOSE_DOCKER_BIN', 'docker'); +// compose.sh's per-stack lock files (its LOCK_DIR), which compose-git takes too. +defined('COMPOSE_LOCK_DIR') || define('COMPOSE_LOCK_DIR', '/var/run/compose.manager'); /** * Reserved filename at the compose root level used by the plugin installer diff --git a/source/compose.manager/include/GitClone.php b/source/compose.manager/include/GitClone.php index f25c831c..d0c74771 100644 --- a/source/compose.manager/include/GitClone.php +++ b/source/compose.manager/include/GitClone.php @@ -133,6 +133,11 @@ public function create(): string } $commit = $this->resolveCommit('refs/remotes/origin/' . $this->settings->branch); + if (!$this->composeFileExistsAt($commit)) { + throw new RuntimeException( + "{$this->settings->composePath} does not exist on branch {$this->settings->branch}, so the clone was removed again." + ); + } $this->runOrThrow(['checkout', '--detach', $commit], 'Could not check out the files'); return $commit; } catch (Throwable $error) { @@ -592,6 +597,24 @@ public function hasCommit(string $commit): bool return $result->succeeded(); } + /** + * The untracked files and folders in the clone, as git lists them (a + * folder holding only untracked files is one entry, ending in /). + * Ignored files are listed too. + * + * @return string[] + * @throws RuntimeException if git fails + */ + public function untrackedEntries(): array + { + $this->assertOwned(); + $result = $this->git(['ls-files', '-z', '--others', '--directory'], $this->settings->cloneDir, 120); + if (!$result->succeeded()) { + throw new RuntimeException('Could not list the untracked files: ' . $result->errorSummary()); + } + return array_values(array_filter(explode("\0", $result->stdout), static fn(string $path): bool => $path !== '')); + } + /** * Whether a string looks like a full commit id (SHA-1 or SHA-256). */ diff --git a/source/compose.manager/include/GitStackManager.php b/source/compose.manager/include/GitStackManager.php new file mode 100644 index 00000000..1d02979e --- /dev/null +++ b/source/compose.manager/include/GitStackManager.php @@ -0,0 +1,476 @@ +say = $say ?? static function (string $message): void { + }; + } + + /** + * Create a new git stack: clone the repository, then make the stack folder. + * + * @return string The new stack's folder name + * @throws RuntimeException|InvalidArgumentException naming why nothing was created + */ + public function add( + string $stackName, + string $url, + string $branch, + string $composePath, + ?string $clonesRoot, + string $description = '' + ): string { + $this->assertArrayStarted(); + $folder = StackInfo::sanitizeProjectString($stackName); + if ($folder === '') { + throw new InvalidArgumentException("'$stackName' cannot be used as a stack name."); + } + $stackDir = $this->composeRoot . '/' . $folder; + if (file_exists($stackDir) || @readlink($stackDir) !== false) { + throw new RuntimeException("A stack folder named '$folder' already exists. Choose another name, or use convert for that stack."); + } + + $settings = GitStackSettings::createNew($url, $branch, $composePath, $clonesRoot ?? COMPOSE_GIT_DEFAULT_CLONES_ROOT, $folder); + ($this->say)("Cloning $url ($branch) into {$settings->cloneDir}..."); + (new GitClone($settings))->create(); + + try { + $stack = StackInfo::createNew($this->composeRoot, $stackName, $description, $settings->composeFileInClone()); + $stackDir = $this->composeRoot . '/' . $stack->projectFolder; + $settings->save($stackDir); + (new GitStackState(null, null))->save($stackDir); + } catch (Throwable $error) { + // Nothing here deletes files, so name what was left: a retry makes a new clone, + // and stops at a stack folder that already exists. + if (is_dir($stackDir)) { + $leftBehind = "The clone at {$settings->cloneDir} was made, and the stack folder $stackDir was" + . ' only partly made. Remove both folders before trying again.'; + } else { + $leftBehind = "The clone at {$settings->cloneDir} was made, but the stack was not." + . ' Remove that folder before trying again.'; + } + throw new RuntimeException($error->getMessage() . "\n" . $leftBehind, 0, $error); + } + ($this->say)("Created stack '{$stack->projectFolder}'. It is not deployed yet."); + return $stack->projectFolder; + } + + /** + * Turn an existing stack into a git stack. + * + * The stack keeps its folder, name, project name, .env and plugin-managed + * override. Its old compose file (for a stack kept in the projects folder) + * and its old indirect settings are moved into a dated backup folder in + * the stack folder, so the change can be undone by moving them back. + * Running containers are left alone until the next deploy. + * + * @return string The backup folder + * @throws RuntimeException|InvalidArgumentException naming why nothing was changed + */ + public function convert(string $folder, string $url, string $branch, string $composePath, ?string $clonesRoot): string + { + $this->assertArrayStarted(); + $stack = $this->stack($folder); + if ($stack->isGitStack()) { + throw new RuntimeException("'$folder' is already a git stack."); + } + $stackDir = $stack->path; + + return $this->withStackLock($stack, function () use ($stack, $stackDir, $folder, $url, $branch, $composePath, $clonesRoot): string { + $oldOverride = $stack->getOverridePath(); + $oldEnv = $stack->getEffectiveEnvFilePath(); + $hasExplicitEnvPath = trim((string) @file_get_contents($stackDir . '/envpath')) !== ''; + $oldComposeFile = $stack->isIndirect ? null : $stack->composeFilePath; + $foldersNextToOldComposeFile = $this->foldersIn($stack->composeSource); + + $settings = GitStackSettings::createNew($url, $branch, $composePath, $clonesRoot ?? COMPOSE_GIT_DEFAULT_CLONES_ROOT, $folder); + ($this->say)("Cloning $url ($branch) into {$settings->cloneDir}..."); + (new GitClone($settings))->create(); + + // From here the stack folder changes; everything replaced goes into the backup. + $backupDir = $stackDir . '/pre-git-' . date('Y-m-d_His'); + $this->assertSafeToWriteStackFolder($backupDir); + if (!@mkdir($backupDir, 0755)) { + throw new RuntimeException("Could not create $backupDir. The clone at {$settings->cloneDir} was made; nothing else changed."); + } + foreach (['indirect', 'indirect_mode', 'use_default_compose_files'] as $metadataFile) { + $source = $stackDir . '/' . $metadataFile; + if (is_file($source) && !@copy($source, $backupDir . '/' . $metadataFile)) { + throw new RuntimeException( + "Could not copy $source into $backupDir. Nothing else in the stack was changed; " + . "the clone at {$settings->cloneDir} was made." + ); + } + } + if ($oldComposeFile !== null && is_file($oldComposeFile) && dirname($oldComposeFile) === $stackDir) { + if (!rename($oldComposeFile, $backupDir . '/' . basename($oldComposeFile))) { + throw new RuntimeException( + "Could not move $oldComposeFile into $backupDir. Nothing else in the stack was changed; " + . "the clone at {$settings->cloneDir} was made." + ); + } + ($this->say)('Moved the old ' . basename($oldComposeFile) . " to $backupDir."); + } + + try { + $indirect = $settings->composeFileInClone(); + if (@file_put_contents($stackDir . '/indirect', $indirect) !== strlen($indirect)) { + throw new RuntimeException("Could not write $stackDir/indirect."); + } + if (@file_put_contents($stackDir . '/indirect_mode', 'file') !== strlen('file')) { + throw new RuntimeException("Could not write $stackDir/indirect_mode."); + } + + // An indirect stack may have used a .env next to its old compose + // file. Keep using it: copy it into the stack folder. + $stackEnv = $stackDir . '/.env'; + if (!$hasExplicitEnvPath && $oldEnv !== null && $oldEnv !== $stackEnv && !is_file($stackEnv)) { + if (!@copy($oldEnv, $stackEnv)) { + throw new RuntimeException("Could not copy $oldEnv to $stackEnv."); + } + ($this->say)("Copied $oldEnv to $stackEnv."); + } + + $settings->save($stackDir); + (new GitStackState(null, null))->save($stackDir); + + // The managed override's name follows the compose file's name, so + // it can change. Carry the old one over if the new one is missing. + StackInfo::clearCache(); + $newOverride = StackInfo::fromProject($this->composeRoot, $folder)->getOverridePath(); + if ($oldOverride !== null && is_file($oldOverride) && $newOverride !== null + && $newOverride !== $oldOverride && !$this->hasContent($newOverride)) { + if (!@copy($oldOverride, $newOverride)) { + throw new RuntimeException("Could not copy the plugin's override $oldOverride to $newOverride."); + } + ($this->say)("Copied the plugin's override $oldOverride to $newOverride."); + } + } catch (Throwable $error) { + // The old compose file may already be in the backup folder, so say how to undo. + throw new RuntimeException( + $error->getMessage() . "\nThe stack was only partly converted. To undo it, delete whichever of" + . " git.json, git_state.json, indirect and indirect_mode are in $stackDir, then copy the files in" + . " $backupDir back into it. The clone at {$settings->cloneDir} was made.", + 0, + $error + ); + } + + ($this->say)("'$folder' is now a git stack. Its containers change at the next deploy."); + if ($foldersNextToOldComposeFile !== []) { + // The compose file is not parsed here; this is only a reminder. + ($this->say)( + 'Note: these folders are next to the old compose file: ' . implode(', ', $foldersNextToOldComposeFile) + . '. If the compose file mounts any of them with a relative path (such as ./data), that path now' + . ' points inside the clone, and the container would start on an empty folder. Use an absolute' + . ' path, or move the data, before deploying.' + ); + } + return $backupDir; + }); + } + + /** + * The names of the folders in a folder, leaving out the ones the plugin makes. + * + * @return string[] + */ + private function foldersIn(string $folder): array + { + $names = []; + foreach (@scandir($folder) ?: [] as $entry) { + if ($entry === '.' || $entry === '..' || $entry === 'git-changes' || str_starts_with($entry, 'pre-git-')) { + continue; + } + if (is_dir($folder . '/' . $entry)) { + $names[] = $entry; + } + } + return $names; + } + + /** + * Compare the deployed commit with the branch on the remote, changing nothing. + * + * @return array{stack: string, branch: string, remoteCommit: string, deployedCommit: ?string, failedCommit: ?string, upToDate: bool} + * @throws RuntimeException if the stack or the remote cannot be read + */ + public function check(string $folder): array + { + $stack = $this->stack($folder); + $settings = $this->settings($stack); + $state = GitStackState::load($stack->path); + $remote = (new GitClone($settings))->remoteBranchCommit(); + return [ + 'stack' => $folder, + 'branch' => $settings->branch, + 'remoteCommit' => $remote, + 'deployedCommit' => $state->deployedCommit, + 'failedCommit' => $state->failedCommit, + 'upToDate' => $state->deployedCommit === $remote, + ]; + } + + /** + * What is known about a git stack locally, without asking the remote. + * + * @return array{stack: string, url: string, branch: string, composePath: string, cloneDir: string, recreateOnFolderChange: bool, deployedCommit: ?string, failedCommit: ?string, checkedOutCommit: ?string, localChanges: list, problem: ?string} + */ + public function status(string $folder): array + { + $stack = $this->stack($folder); + $settings = $this->settings($stack); + $state = GitStackState::load($stack->path); + $checkedOut = null; + $changes = []; + $problem = null; + try { + $clone = new GitClone($settings); + $checkedOut = $clone->checkedOutCommit(); + $changes = $clone->locallyChangedFiles(); + } catch (Throwable $error) { + $problem = $error->getMessage(); + } + return [ + 'stack' => $folder, + 'url' => $settings->url, + 'branch' => $settings->branch, + 'composePath' => $settings->composePath, + 'cloneDir' => $settings->cloneDir, + 'recreateOnFolderChange' => $settings->recreateOnFolderChange, + 'deployedCommit' => $state->deployedCommit, + 'failedCommit' => $state->failedCommit, + 'checkedOutCommit' => $checkedOut, + 'localChanges' => $changes, + 'problem' => $problem, + ]; + } + + /** + * Replace a stack's clone with a fresh one at the deployed commit. + * + * The old clone is moved aside, never deleted: untracked data such as a + * relative ./data bind mount lives in it, and stays there to be copied + * back by hand. If the new clone fails, the old one is moved back. + * + * @return string|null Where the old clone was moved, or null if there was none + * @throws RuntimeException naming the problem + */ + public function reclone(string $folder): ?string + { + $this->assertArrayStarted(); + $stack = $this->stack($folder); + $settings = $this->settings($stack); + + return $this->withStackLock($stack, function () use ($stack, $settings): ?string { + $state = GitStackState::load($stack->path); + $cloneDir = $settings->cloneDir; + $movedTo = null; + + $untrackedInOldClone = []; + if (file_exists($cloneDir) || @readlink($cloneDir) !== false) { + if (@readlink($cloneDir) !== false) { + throw new RuntimeException("$cloneDir is a symlink, so it was left alone."); + } + // Data a container wrote through a relative bind mount stays in + // the old clone; list it so the person knows what to copy back. + try { + $untrackedInOldClone = (new GitClone($settings))->untrackedEntries(); + } catch (Throwable) { + // A clone that is not provably ours is not read. + } + $movedTo = $cloneDir . '.replaced-' . date('Y-m-d_His'); + GitPathGuard::assertSafeToWrite($movedTo); + if (file_exists($movedTo) || !@rename($cloneDir, $movedTo)) { + throw new RuntimeException("Could not move $cloneDir aside to $movedTo. Nothing was changed."); + } + ($this->say)("Moved the old clone to $movedTo."); + } + + try { + $clone = new GitClone($settings); + ($this->say)("Cloning {$settings->url} ({$settings->branch})..."); + $head = $clone->create(); + } catch (Throwable $error) { + if ($movedTo !== null) { + if (!file_exists($cloneDir) && @rename($movedTo, $cloneDir)) { + ($this->say)('Moved the old clone back.'); + } else { + ($this->say)("Could not move the old clone back. It is still at $movedTo."); + } + } + throw $error instanceof RuntimeException ? $error : new RuntimeException($error->getMessage(), 0, $error); + } + + // Put the files back at the commit that is running, so the next + // deploy compares against what is really deployed. + $deployed = $state->deployedCommit; + if ($deployed !== null && $deployed !== $head) { + if (!$clone->hasCommit($deployed)) { + // The branch was rewritten and no longer holds it. + ($this->say)('The deployed commit ' . substr($deployed, 0, 12) . ' is not on the branch any more, so the clone was left at ' + . substr($head, 0, 12) . '. Deploy to bring the stack up to date.'); + (new GitStackState(null, $state->failedCommit))->save($stack->path); + } else { + try { + $clone->checkOut($deployed); + } catch (Throwable $error) { + throw new RuntimeException( + 'The new clone could not be put at the deployed commit ' . substr($deployed, 0, 12) . ': ' + . $error->getMessage() . "\nIt is at " . substr($head, 0, 12) . '.' + . ($movedTo !== null ? " The old clone is at $movedTo." : ''), + 0, + $error + ); + } + } + } + if ($movedTo !== null && $untrackedInOldClone !== []) { + ($this->say)( + 'The old clone holds files that are not in the repository, and not in the new clone: ' + . implode(', ', array_slice($untrackedInOldClone, 0, 10)) . (count($untrackedInOldClone) > 10 ? ', ...' : '') + . '. If a container keeps data there (a relative bind mount such as ./data), copy it back before deploying.' + ); + } + return $movedTo; + }); + } + + /** + * The folder names of all git stacks, in the projects folder's order. + * + * @return string[] + */ + public function listGitStacks(): array + { + $folders = []; + foreach (StackInfo::listProjectFolders($this->composeRoot) as $folder) { + if (is_file($this->composeRoot . '/' . $folder . '/' . GitStackSettings::FILE_NAME)) { + $folders[] = $folder; + } + } + return $folders; + } + + /** + * A git stack by its exact folder name. + * + * @throws RuntimeException if there is no such stack, or it is not a git stack + */ + public function gitStack(string $folder): StackInfo + { + $stack = $this->stack($folder); + if (!$stack->isGitStack()) { + throw new RuntimeException("'$folder' is not a git stack."); + } + return $stack; + } + + /** + * A stack by its exact folder name. + * + * @throws RuntimeException if there is no such stack folder + */ + private function stack(string $folder): StackInfo + { + if ($folder === '' || $folder === '.' || $folder === '..' || str_contains($folder, '/')) { + throw new RuntimeException("'$folder' is not a stack folder name."); + } + if (!is_dir($this->composeRoot . '/' . $folder) || @readlink($this->composeRoot . '/' . $folder) !== false) { + throw new RuntimeException("There is no stack folder named exactly '$folder' in {$this->composeRoot}."); + } + StackInfo::clearCache(); + return StackInfo::fromProject($this->composeRoot, $folder); + } + + private function settings(StackInfo $stack): GitStackSettings + { + $settings = GitStackSettings::load($stack->path); + if ($settings === null) { + throw new RuntimeException("'{$stack->projectFolder}' is not a git stack."); + } + return $settings; + } + + private function assertArrayStarted(): void + { + if (!GitPathGuard::isArrayStarted()) { + throw new RuntimeException('The array is not started, so nothing was changed.'); + } + } + + /** + * The projects folder is usually on the flash, but may be on the array. + */ + private function assertSafeToWriteStackFolder(string $path): void + { + if (str_starts_with($path, rtrim(COMPOSE_GIT_MNT_DIR, '/') . '/')) { + GitPathGuard::assertSafeToWrite($path); + } + } + + private function hasContent(string $file): bool + { + if (!is_file($file)) { + return false; + } + // The plugin writes an empty template override on its own; that does not count. + return trim((string) file_get_contents($file)) !== trim(OverrideInfo::buildTemplateContent()); + } + + /** + * Run under compose.sh's lock for the stack, so a deploy cannot run at the same time. + * + * @template T + * @param callable(): T $work + * @return T + */ + private function withStackLock(StackInfo $stack, callable $work): mixed + { + if (!is_dir(COMPOSE_LOCK_DIR)) { + @mkdir(COMPOSE_LOCK_DIR, 0755, true); + } + $lockFile = COMPOSE_LOCK_DIR . '/' . $stack->projectName . '.lock'; + $handle = @fopen($lockFile, 'c'); + if ($handle === false) { + throw new RuntimeException("Could not open the lock file $lockFile."); + } + try { + if (!flock($handle, LOCK_EX | LOCK_NB)) { + throw new RuntimeException("Another operation is in progress for '{$stack->projectFolder}'. Try again when it has finished."); + } + return $work(); + } finally { + flock($handle, LOCK_UN); + fclose($handle); + } + } +} diff --git a/source/compose.manager/scripts/compose-git b/source/compose.manager/scripts/compose-git new file mode 100755 index 00000000..386ffae4 --- /dev/null +++ b/source/compose.manager/scripts/compose-git @@ -0,0 +1,3 @@ +#!/bin/bash +# compose-git: create, deploy and look after git-backed stacks. See compose_git.php. +exec php -d display_errors=stderr "$(dirname "$(readlink -f "$0")")/compose_git.php" "$@" diff --git a/source/compose.manager/scripts/compose_git.php b/source/compose.manager/scripts/compose_git.php new file mode 100644 index 00000000..7d732395 --- /dev/null +++ b/source/compose.manager/scripts/compose_git.php @@ -0,0 +1,575 @@ + --url --path [--branch ] [--clones-root ] [--description ] + compose-git convert --url --path [--branch ] [--clones-root ] + compose-git check |--all + compose-git deploy [--commit ] [--save-local-changes] [--wait|--no-wait] [--wait-timeout ] [--profile ]... + compose-git reclone + compose-git status [|--all] [--json] + + is an https address (no user name or password in it) or the path of a repository under /mnt. + is the compose file's path inside the repository, such as stacks/whoami/compose.yaml. +--branch defaults to main. is the stack's exact folder name. +deploy waits for healthy containers when the stack's wait-for-healthy setting says so; +--wait and --no-wait override it. + +Exit status: 0 success, 1 failed, 2 usage error, 3 (check) a newer commit is available. +check --all exits 3 if any stack is behind, even when another could not be checked. + +Run 'compose-git --help' for what a command does and what each option means. +TEXT; + +/** + * The detailed help for one command, or null if there is no such command. + */ +function compose_git_command_help(string $command): ?string +{ + return match ($command) { + 'add' => <<<'TEXT' +Usage: compose-git add --url --path [options] + +Make a new git stack from a repository. The repository is cloned onto the array, and a new +stack folder is made that points at the compose file in the clone. Nothing is deployed yet: +run 'compose-git deploy ' next. If the clone fails, or the branch has no compose file +at --path, nothing is created. + + is the stack's name. Its folder name is the name with unsafe characters replaced, and +is printed at the end. No stack folder by that name may exist yet; to put an existing stack +under git, use 'compose-git convert'. + +Options: + --url The repository: a public https:// address, with no user name or + password in it, or the path of a repository on this server under + /mnt. Required. + --path The compose file's path inside the repository, such as + stacks/whoami/compose.yaml. Required. + --branch The branch to follow. Default: main. + --clones-root The folder the clone goes in, as -. It must be on a + share, disk or pool, and the share must exist. + Default: /mnt/user/appdata/compose.manager/git. + --description The description shown for the stack on the Compose page. + +Example: + compose-git add whoami --url https://github.com/you/stacks.git --path whoami/compose.yaml +TEXT, + 'convert' => <<<'TEXT' +Usage: compose-git convert --url --path [options] + +Put an existing stack under git. Copy its compose file into the repository and push it first. +The repository is cloned, and the stack is pointed at the compose file in the clone. + +The stack keeps its name, project name, .env, override file and settings, so it keeps its +volumes and networks. Its old compose file, and its old indirect settings if it had any, are +moved into a pre-git- folder in the stack folder: that is the way back. Running +containers are left alone until the next deploy. Folders next to the old compose file are +listed as a reminder, because a relative bind mount such as ./data now points inside the +clone. + +If the clone fails, or the branch has no compose file at --path, nothing is changed. It +refuses to start while another operation on the stack is running. + +Options: + --url The repository: a public https:// address, with no user name or + password in it, or the path of a repository on this server under + /mnt. Required. + --path The compose file's path inside the repository. Required. + --branch The branch to follow. Default: main. + --clones-root The folder the clone goes in, as -. It must be on a + share, disk or pool, and the share must exist. + Default: /mnt/user/appdata/compose.manager/git. + +Then deploy it: compose-git deploy +TEXT, + 'check' => <<<'TEXT' +Usage: compose-git check + compose-git check --all + +Ask the repository for the newest commit on the stack's branch, and compare it with the +deployed commit. Changes nothing, on the server or in the clone. + +Options: + --all Check every git stack, one line each. + +Exit status: + 0 up to date + 1 the stack or the repository could not be read + 3 a newer commit is on the branch (or nothing is deployed yet); deploy it with + 'compose-git deploy ' +With --all: 3 if any stack is behind, even when another could not be checked; otherwise 1 if +any could not be checked. +TEXT, + 'deploy' => <<<'TEXT' +Usage: compose-git deploy [options] + +Deploy the newest commit on the stack's branch, or the commit given with --commit. In order: +fetch the commit, check it out in the clone, check it, pull and build the images, then run +docker compose up. The checks stop the deploy when the compose file is not valid, a variable +it uses is not set, a name in the repository's .env.example is missing from the .env, or an +external network or volume, bind folder, config or secret file, or build folder is missing, +or a container name or port is already taken. + +If anything before up fails, the previous commit is put back and no container is changed. If +up itself fails, the commit is recorded as failed ('compose-git status' shows it) and nothing +is rolled back: fix the cause, usually in the repository, and deploy again. + +When the commit changes anything in the stack's folder in the repository, every container in +the stack is recreated, so changed config files take effect. To turn that off, set +"recreateOnFolderChange": false in the stack's git.json. + +If another operation on the stack is running, such as a Start from the web UI, it waits up +to 30 seconds (COMPOSE_LOCK_TIMEOUT) for it to finish, then fails without changing anything. + +Options: + --commit Deploy this commit instead of the newest, for example to go back + to an older version. Give the full 40-character id. A later deploy + without --commit returns to the branch's newest commit. + --save-local-changes If tracked files in the clone were changed, or a commit was made + in it, save the changes as a patch in the stack folder's + git-changes folder, discard them, then deploy. Without it, the + deploy stops and lists the changed files. Files that are not part + of the repository are never touched either way. + --wait Wait for the containers to be running and healthy, and count the + deploy as failed if they are not. Default: the stack's + wait-for-healthy setting. + --no-wait Do not wait, whatever the stack's setting says. + --wait-timeout How long to wait for healthy containers. Implies --wait. + --profile Start the services in this compose profile too. Repeat it for + more than one profile. Default: the profiles the stack is running + with, else its default profiles, as the web UI's Update uses. + +Examples: + compose-git deploy myapp + compose-git deploy myapp --commit 4f1c2b0e9d8a7c6b5a4f3e2d1c0b9a8f7e6d5c4b +TEXT, + 'reclone' => <<<'TEXT' +Usage: compose-git reclone + +Replace the stack's clone with a fresh one at the deployed commit, for when the clone is in +a bad state. No container is touched. + +The old clone is moved aside to .replaced-, never deleted, and any files in it +that are not part of the repository (data a container wrote there, say) are listed. Copy +back what you need, then delete the old clone yourself. If the new clone fails, the old one +is moved back. + +It refuses to start while another operation on the stack is running. +TEXT, + 'status' => <<<'TEXT' +Usage: compose-git status [ | --all] [--json] + +Show what is known about git stacks on this server, without asking the repository: the +repository and branch, the compose file's path in it, the clone folder, the deployed commit, +a failed commit if the last deploy failed during up, the commit checked out in the clone, +and the tracked files changed in the clone. Changes nothing. + +Options: + --all Show every git stack. This is what happens when no stack is named, too. A stack + whose git settings cannot be read is shown with the reason. + --json Print the same as JSON, for scripts. +TEXT, + default => null, + }; +} + +/** + * Split arguments into positional values and --options. Options that take a + * value are named in $withValue; --profile may repeat. + * + * @param string[] $args + * @param string[] $withValue + * @param string[] $flags + * @return array{0: string[], 1: array} + */ +function compose_git_parse(array $args, array $withValue, array $flags): array +{ + $positional = []; + $options = []; + for ($i = 0; $i < count($args); $i++) { + $arg = $args[$i]; + if (!str_starts_with($arg, '--')) { + $positional[] = $arg; + continue; + } + $name = substr($arg, 2); + if (in_array($name, $flags, true)) { + $options[$name] = true; + } elseif (in_array($name, $withValue, true)) { + if (!isset($args[$i + 1])) { + throw new ComposeGitUsageError("--$name needs a value."); + } + $value = $args[++$i]; + if ($name === 'profile') { + $options['profile'] = array_merge((array) ($options['profile'] ?? []), [$value]); + } else { + $options[$name] = $value; + } + } else { + throw new ComposeGitUsageError("Unknown option --$name."); + } + } + return [$positional, $options]; +} + +/** + * @param array $options + */ +function compose_git_required(array $options, string $name): string +{ + $value = $options[$name] ?? null; + if (!is_string($value) || $value === '') { + throw new ComposeGitUsageError("--$name is required."); + } + return $value; +} + +/** + * @param string[] $positional + */ +function compose_git_one_stack(array $positional): string +{ + if (count($positional) !== 1) { + throw new ComposeGitUsageError('Name exactly one stack.'); + } + return $positional[0]; +} + +/** + * compose.sh's --wait arguments for a deploy. + * + * Waits for healthy containers as the web UI's up does: the stack's own + * setting, else the plugin default (both from resolveStackWaitSettings()), + * with --wait, --no-wait and --wait-timeout overriding them. + * + * @param array{enabled: bool, timeout: string, stackName?: string} $waitSettings + * @param array $options + * @return string[] + */ +function compose_git_wait_arguments(array $waitSettings, array $options): array +{ + if (isset($options['wait']) && isset($options['no-wait'])) { + throw new ComposeGitUsageError('Use --wait or --no-wait, not both.'); + } + $wait = $waitSettings['enabled']; + $timeout = $waitSettings['timeout']; + if (isset($options['wait']) || isset($options['wait-timeout'])) { + $wait = true; + } + if (isset($options['no-wait'])) { + $wait = false; + } + if (isset($options['wait-timeout'])) { + $timeout = (string) $options['wait-timeout']; + if (preg_match('/^[1-9][0-9]{0,5}$/', $timeout) !== 1) { + throw new ComposeGitUsageError('--wait-timeout must be a number of seconds.'); + } + } + if (!$wait) { + return []; + } + // A timeout read from the settings files is passed on only when it is a plain number. + if (preg_match('/^[1-9][0-9]{0,5}$/', $timeout) !== 1) { + return ['--wait']; + } + return ['--wait', '--wait-timeout', $timeout]; +} + +/** + * compose.sh's -g (profile) arguments for a deploy. + * + * The --profile options when given. Otherwise the profiles the stack is + * running with, else its default profiles, as the web UI's Update does: a + * deploy without profiles would leave the running profile services on the + * old commit, and compose.sh would forget the stack's running profiles. + * + * @param array $options + * @return string[] + */ +function compose_git_profile_arguments(StackInfo $stack, array $options): array +{ + $profiles = (array) ($options['profile'] ?? []); + if ($profiles === []) { + $profiles = $stack->getRunningProfiles(); + } + if ($profiles === []) { + $profiles = $stack->getDefaultProfiles(); + } + $args = []; + foreach ($profiles as $profile) { + $args[] = '-g' . $profile; + } + return $args; +} + +/** + * Deploy through compose.sh, so the deploy takes the same lock, credentials + * and logging as every other stack action. Its output goes straight to ours. + * + * @param array $options + * @param resource $errors + */ +function compose_git_deploy(GitStackManager $manager, string $folder, array $options, $errors): int +{ + $stack = $manager->gitStack($folder); + if (!$stack->hasResolvedIdentity()) { + fwrite($errors, $stack->getIdentityBlockReason() . "\n"); + return COMPOSE_GIT_EXIT_FAILED; + } + + $command = [__DIR__ . '/compose.sh', '-cgitdeploy', '-p' . $stack->projectName, '-s' . $stack->path]; + $credentialId = trim((string) ($stack->getCredentialId() ?? '')); + if ($credentialId !== '') { + $command[] = '--credential-id'; + $command[] = $credentialId; + } + foreach (compose_git_profile_arguments($stack, $options) as $arg) { + $command[] = $arg; + } + if (isset($options['commit'])) { + $command[] = '--git-commit'; + $command[] = (string) $options['commit']; + } + if (isset($options['save-local-changes'])) { + $command[] = '--save-local-changes'; + } + $waitSettings = resolveStackWaitSettings($stack->path, parse_plugin_cfg('compose.manager')); + foreach (compose_git_wait_arguments($waitSettings, $options) as $arg) { + $command[] = $arg; + } + + $process = proc_open($command, [0 => ['file', '/dev/null', 'r'], 1 => STDOUT, 2 => STDERR], $pipes); + if (!is_resource($process)) { + fwrite($errors, "Could not start compose.sh.\n"); + return COMPOSE_GIT_EXIT_FAILED; + } + $exitCode = proc_close($process); + return $exitCode === 0 ? COMPOSE_GIT_EXIT_OK : COMPOSE_GIT_EXIT_FAILED; +} + +/** + * A status row for a git stack whose settings or state could not be read, + * with the same keys as GitStackManager::status() and the reason as the problem. + * + * @return array + */ +function compose_git_unreadable_status(string $folder, string $problem): array +{ + return [ + 'stack' => $folder, + 'url' => null, + 'branch' => null, + 'composePath' => null, + 'cloneDir' => null, + 'recreateOnFolderChange' => null, + 'deployedCommit' => null, + 'failedCommit' => null, + 'checkedOutCommit' => null, + 'localChanges' => [], + 'problem' => $problem, + ]; +} + +/** + * @param array $status + */ +function compose_git_print_status(array $status): void +{ + $short = static fn(?string $commit): string => $commit === null ? '-' : substr($commit, 0, 12); + echo $status['stack'] . "\n"; + if ($status['url'] === null) { + // The stack's git settings could not be read (see compose_git_unreadable_status()). + echo ' problem: ' . $status['problem'] . "\n"; + return; + } + echo ' repository: ' . $status['url'] . ' (' . $status['branch'] . ")\n"; + echo ' compose file: ' . $status['composePath'] . "\n"; + echo ' clone: ' . $status['cloneDir'] . "\n"; + echo ' deployed: ' . $short($status['deployedCommit']) . "\n"; + if ($status['failedCommit'] !== null) { + echo ' failed: ' . $short($status['failedCommit']) . " (fix it and deploy again)\n"; + } + echo ' checked out: ' . $short($status['checkedOutCommit']) . "\n"; + if ($status['localChanges'] !== []) { + echo ' local changes: ' . implode(', ', $status['localChanges']) . "\n"; + } + if ($status['problem'] !== null) { + echo ' problem: ' . $status['problem'] . "\n"; + } +} + +/** + * @param string[] $argv + * @param resource|null $errors Where progress and errors go (standard error by default) + */ +function compose_git_main(array $argv, string $composeRoot, $errors = null): int +{ + $errors ??= STDERR; + $say = static function (string $message) use ($errors): void { + fwrite($errors, $message . "\n"); + }; + $manager = new GitStackManager(rtrim($composeRoot, '/'), $say); + $command = $argv[1] ?? ''; + $args = array_slice($argv, 2); + + // 'compose-git --help' (or -h) prints that command's help, whatever else is given. + $commandHelp = compose_git_command_help($command); + if ($commandHelp !== null && (in_array('--help', $args, true) || in_array('-h', $args, true))) { + echo $commandHelp . "\n"; + return COMPOSE_GIT_EXIT_OK; + } + + try { + switch ($command) { + case 'add': + [$positional, $options] = compose_git_parse($args, ['url', 'path', 'branch', 'clones-root', 'description'], []); + $folder = $manager->add( + compose_git_one_stack($positional), + compose_git_required($options, 'url'), + (string) ($options['branch'] ?? GitStackManager::DEFAULT_BRANCH), + compose_git_required($options, 'path'), + isset($options['clones-root']) ? (string) $options['clones-root'] : null, + (string) ($options['description'] ?? '') + ); + echo "Deploy it with: compose-git deploy $folder\n"; + return COMPOSE_GIT_EXIT_OK; + + case 'convert': + [$positional, $options] = compose_git_parse($args, ['url', 'path', 'branch', 'clones-root'], []); + $folder = compose_git_one_stack($positional); + $backup = $manager->convert( + $folder, + compose_git_required($options, 'url'), + (string) ($options['branch'] ?? GitStackManager::DEFAULT_BRANCH), + compose_git_required($options, 'path'), + isset($options['clones-root']) ? (string) $options['clones-root'] : null + ); + echo "The replaced files are in $backup.\nDeploy it with: compose-git deploy $folder\n"; + return COMPOSE_GIT_EXIT_OK; + + case 'check': + [$positional, $options] = compose_git_parse($args, [], ['all']); + $folders = isset($options['all']) ? $manager->listGitStacks() : [compose_git_one_stack($positional)]; + $exit = COMPOSE_GIT_EXIT_OK; + foreach ($folders as $folder) { + try { + $result = $manager->check($folder); + $deployed = $result['deployedCommit'] === null ? 'nothing deployed yet' : substr($result['deployedCommit'], 0, 12) . ' deployed'; + if ($result['upToDate']) { + echo "$folder: up to date ($deployed)\n"; + } else { + echo "$folder: " . substr($result['remoteCommit'], 0, 12) . " available on {$result['branch']} ($deployed)\n"; + $exit = max($exit, COMPOSE_GIT_EXIT_BEHIND); + } + } catch (RuntimeException | InvalidArgumentException $error) { + echo "$folder: could not check: " . $error->getMessage() . "\n"; + $exit = $exit === COMPOSE_GIT_EXIT_BEHIND ? $exit : COMPOSE_GIT_EXIT_FAILED; + } + } + return $exit; + + case 'deploy': + [$positional, $options] = compose_git_parse($args, ['commit', 'wait-timeout', 'profile'], ['save-local-changes', 'wait', 'no-wait']); + return compose_git_deploy($manager, compose_git_one_stack($positional), $options, $errors); + + case 'reclone': + [$positional] = compose_git_parse($args, [], []); + $movedTo = $manager->reclone(compose_git_one_stack($positional)); + if ($movedTo !== null) { + echo "The old clone is in $movedTo. Copy back anything you need from it, then delete it yourself.\n"; + } + return COMPOSE_GIT_EXIT_OK; + + case 'status': + [$positional, $options] = compose_git_parse($args, [], ['all', 'json']); + $listingAll = $positional === [] || isset($options['all']); + $folders = $listingAll ? $manager->listGitStacks() : [compose_git_one_stack($positional)]; + $statuses = []; + foreach ($folders as $folder) { + try { + $statuses[] = $manager->status($folder); + } catch (RuntimeException | InvalidArgumentException $error) { + if (!$listingAll) { + throw $error; + } + // One unreadable stack (a hand-edited git.json, say) must not hide the others. + $statuses[] = compose_git_unreadable_status($folder, $error->getMessage()); + } + } + if (isset($options['json'])) { + echo json_encode($statuses, JSON_PRETTY_PRINT | JSON_UNESCAPED_SLASHES) . "\n"; + } else { + foreach ($statuses as $status) { + compose_git_print_status($status); + } + if ($statuses === []) { + echo "There are no git stacks.\n"; + } + } + return COMPOSE_GIT_EXIT_OK; + + case 'help': + // 'compose-git help ' is the same as 'compose-git --help'. + if (isset($args[0])) { + $helpFor = compose_git_command_help($args[0]); + if ($helpFor === null) { + throw new ComposeGitUsageError("Unknown command '{$args[0]}'."); + } + echo $helpFor . "\n"; + return COMPOSE_GIT_EXIT_OK; + } + echo COMPOSE_GIT_USAGE . "\n"; + return COMPOSE_GIT_EXIT_OK; + + case '--help': + case '-h': + echo COMPOSE_GIT_USAGE . "\n"; + return COMPOSE_GIT_EXIT_OK; + + case '': + echo COMPOSE_GIT_USAGE . "\n"; + return COMPOSE_GIT_EXIT_USAGE; + + default: + throw new ComposeGitUsageError("Unknown command '$command'."); + } + } catch (ComposeGitUsageError $error) { + // A mistake in a known command shows that command's help; anything else, the overview. + fwrite($errors, $error->getMessage() . "\n\n" . ($commandHelp ?? COMPOSE_GIT_USAGE) . "\n"); + return COMPOSE_GIT_EXIT_USAGE; + } catch (Throwable $error) { + fwrite($errors, $error->getMessage() . "\n"); + return COMPOSE_GIT_EXIT_FAILED; + } +} + +if (PHP_SAPI === 'cli' && realpath($_SERVER['SCRIPT_FILENAME'] ?? '') === realpath(__FILE__)) { + exit(compose_git_main($argv, $compose_root)); +} diff --git a/tests/bootstrap.php b/tests/bootstrap.php index 235f5803..7615a76c 100644 --- a/tests/bootstrap.php +++ b/tests/bootstrap.php @@ -54,6 +54,7 @@ function composeLogger($message, $data = null, $type = 'daemon', $level = 'info' define('COMPOSE_GIT_HOME_DIR', sys_get_temp_dir() . '/compose_git_home'); // A scripted stand-in for the docker CLI (see FakeDocker in the git deploy tests). define('COMPOSE_DOCKER_BIN', sys_get_temp_dir() . '/compose_fake_docker/docker'); +define('COMPOSE_LOCK_DIR', sys_get_temp_dir() . '/compose_manager_locks'); // Point to the dev-env resvg binary when present; plugin path used on real Unraid define('COMPOSE_RESVG_BIN', is_executable('/tmp/resvg') ? '/tmp/resvg' : '/usr/local/emhttp/plugins/compose.manager/bin/resvg'); diff --git a/tests/unit/GitStackManagerTest.php b/tests/unit/GitStackManagerTest.php new file mode 100644 index 00000000..0ee9e6fd --- /dev/null +++ b/tests/unit/GitStackManagerTest.php @@ -0,0 +1,648 @@ +mnt = COMPOSE_GIT_MNT_DIR; + FakeDocker::removeTree($this->mnt); + mkdir($this->mnt . '/user/appdata', 0755, true); + mkdir($this->mnt . '/user/repos', 0755, true); + file_put_contents(COMPOSE_UNRAID_VAR_INI, "mdState=\"STARTED\"\n"); + file_put_contents(COMPOSE_MOUNTS_FILE, "rootfs {$this->mnt} rootfs rw 0 0\nshfs {$this->mnt}/user fuse.shfs rw 0 0\n"); + + $this->upstream = $this->mnt . '/user/repos/stacks.git'; + $this->author = sys_get_temp_dir() . '/compose_git_manager_author'; + FakeDocker::removeTree($this->author); + $this->git(['init', '-q', '--bare', '-b', 'main', $this->upstream], null); + $this->git(['clone', '-q', $this->upstream, $this->author], null); + $this->git(['checkout', '-q', '-b', 'main'], $this->author); + $this->writeAndPush(['whoami/compose.yaml' => "services:\n whoami:\n image: traefik/whoami\n"], 'first'); + + $this->composeRoot = sys_get_temp_dir() . '/compose_git_manager_projects'; + FakeDocker::removeTree($this->composeRoot); + mkdir($this->composeRoot); + FakeDocker::removeTree(COMPOSE_LOCK_DIR); + $this->clonesRoot = $this->mnt . '/user/appdata/git'; + $this->manager = new GitStackManager($this->composeRoot); + } + + protected function tearDown(): void + { + FakeDocker::removeTree($this->mnt); + FakeDocker::removeTree($this->author); + FakeDocker::removeTree($this->composeRoot); + FakeDocker::removeTree(COMPOSE_LOCK_DIR); + parent::tearDown(); + } + + // ----- add ----- + + public function testAddClonesAndMakesAGitStack(): void + { + $folder = $this->manager->add('whoami', $this->upstream, 'main', 'whoami/compose.yaml', $this->clonesRoot); + + $stackDir = $this->composeRoot . '/' . $folder; + $settings = GitStackSettings::load($stackDir); + $this->assertNotNull($settings); + $this->assertSame($settings->composeFileInClone(), file_get_contents($stackDir . '/indirect')); + $this->assertSame('file', file_get_contents($stackDir . '/indirect_mode')); + $this->assertFileExists($settings->composeFileInClone()); + $this->assertNull(GitStackState::load($stackDir)->deployedCommit); + $this->assertTrue(\StackInfo::fromProject($this->composeRoot, $folder)->isGitStack()); + } + + public function testAddRefusesAnExistingStackFolderAndClonesNothing(): void + { + mkdir($this->composeRoot . '/whoami'); + + try { + $this->manager->add('whoami', $this->upstream, 'main', 'whoami/compose.yaml', $this->clonesRoot); + $this->fail('Added over an existing stack folder'); + } catch (RuntimeException $error) { + $this->assertStringContainsString('already exists', $error->getMessage()); + } + $this->assertDirectoryDoesNotExist($this->clonesRoot); + } + + public function testAddWithAComposePathNotOnTheBranchLeavesNothingBehind(): void + { + try { + $this->manager->add('whoami', $this->upstream, 'main', 'whoami/compose.yml', $this->clonesRoot); + $this->fail('Added a stack whose compose file is not in the repository'); + } catch (RuntimeException $error) { + $this->assertStringContainsString('does not exist on branch main', $error->getMessage()); + } + $this->assertSame([], glob($this->clonesRoot . '/*') ?: []); + $this->assertDirectoryDoesNotExist($this->composeRoot . '/whoami'); + } + + public function testAddThatFailsAfterMakingTheStackFolderSaysToRemoveIt(): void + { + // A projects folder on the array's root file system (RAM): the stack + // folder is made, then saving its git settings there is refused. + $composeRoot = $this->mnt . '/projects'; + mkdir($composeRoot); + $manager = new GitStackManager($composeRoot); + + try { + $manager->add('whoami', $this->upstream, 'main', 'whoami/compose.yaml', $this->clonesRoot); + $this->fail('Added a stack whose settings could not be saved'); + } catch (RuntimeException $error) { + $this->assertStringContainsString("the stack folder $composeRoot/whoami was only partly made", $error->getMessage()); + $this->assertStringContainsString('Remove both folders', $error->getMessage()); + } + $this->assertDirectoryExists($composeRoot . '/whoami'); + } + + public function testAddDoesNothingWhileTheArrayIsStopped(): void + { + file_put_contents(COMPOSE_UNRAID_VAR_INI, "mdState=\"STOPPED\"\n"); + $this->expectException(RuntimeException::class); + $this->manager->add('whoami', $this->upstream, 'main', 'whoami/compose.yaml', $this->clonesRoot); + } + + // ----- convert ----- + + public function testConvertMovesTheOldComposeFileAsideAndKeepsEnvAndOverride(): void + { + $stack = \StackInfo::createNew($this->composeRoot, 'myapp'); + $stackDir = $this->composeRoot . '/' . $stack->projectFolder; + file_put_contents($stackDir . '/.env', "TZ=Europe/London\n"); + file_put_contents($stackDir . '/compose.override.yaml', "services:\n whoami:\n labels:\n net.unraid.docker.icon: x\n"); + $oldCompose = (string) file_get_contents($stackDir . '/compose.yaml'); + + $backup = $this->manager->convert($stack->projectFolder, $this->upstream, 'main', 'whoami/compose.yaml', $this->clonesRoot); + + $this->assertSame($oldCompose, file_get_contents($backup . '/compose.yaml')); + $this->assertFileDoesNotExist($stackDir . '/compose.yaml'); + $this->assertSame("TZ=Europe/London\n", file_get_contents($stackDir . '/.env')); + $this->assertStringContainsString('net.unraid.docker.icon', (string) file_get_contents($stackDir . '/compose.override.yaml')); + + \StackInfo::clearCache(); + $converted = \StackInfo::fromProject($this->composeRoot, $stack->projectFolder); + $this->assertTrue($converted->isGitStack()); + $args = $converted->buildComposeArgs(); + $this->assertSame(GitStackSettings::load($stackDir)->composeFileInClone(), $args['filePaths'][0]); + $this->assertSame($stackDir . '/compose.override.yaml', $args['filePaths'][1]); + $this->assertSame($stackDir . '/.env', $args['envFilePath']); + } + + public function testConvertOfAnIndirectStackCopiesItsEnvIntoTheStackFolder(): void + { + $oldFolder = $this->mnt . '/user/appdata/legacy'; + mkdir($oldFolder); + file_put_contents($oldFolder . '/compose.yaml', "services:\n whoami:\n image: traefik/whoami\n"); + file_put_contents($oldFolder . '/.env', "SECRET=1\n"); + $stack = \StackInfo::createNew($this->composeRoot, 'legacy', '', $oldFolder); + $stackDir = $this->composeRoot . '/' . $stack->projectFolder; + + $backup = $this->manager->convert($stack->projectFolder, $this->upstream, 'main', 'whoami/compose.yaml', $this->clonesRoot); + + $this->assertSame("SECRET=1\n", file_get_contents($stackDir . '/.env')); + $this->assertSame("SECRET=1\n", file_get_contents($oldFolder . '/.env')); + $this->assertFileExists($oldFolder . '/compose.yaml'); + $this->assertSame($oldFolder, file_get_contents($backup . '/indirect')); + } + + public function testConvertOfAStackWithACustomEnvFileCopiesNoEnv(): void + { + $oldFolder = $this->mnt . '/user/appdata/legacy'; + mkdir($oldFolder); + file_put_contents($oldFolder . '/compose.yaml', "services:\n whoami:\n image: traefik/whoami\n"); + file_put_contents($oldFolder . '/.env', "NEXT_TO_COMPOSE=1\n"); + $customEnv = $this->mnt . '/user/appdata/custom.env'; + file_put_contents($customEnv, "CUSTOM=1\n"); + $stack = \StackInfo::createNew($this->composeRoot, 'legacy', '', $oldFolder); + $stackDir = $this->composeRoot . '/' . $stack->projectFolder; + file_put_contents($stackDir . '/envpath', $customEnv); + + $this->manager->convert($stack->projectFolder, $this->upstream, 'main', 'whoami/compose.yaml', $this->clonesRoot); + + // The stack's own env file setting still decides; nothing is copied next to it. + $this->assertFileDoesNotExist($stackDir . '/.env'); + \StackInfo::clearCache(); + $this->assertSame($customEnv, \StackInfo::fromProject($this->composeRoot, $stack->projectFolder)->getEffectiveEnvFilePath()); + } + + public function testConvertOfAnIndirectStackCarriesItsOverrideIntoTheStackFolder(): void + { + $oldFolder = $this->mnt . '/user/appdata/legacy'; + mkdir($oldFolder); + file_put_contents($oldFolder . '/compose.yaml', "services:\n whoami:\n image: traefik/whoami\n"); + $labels = "services:\n whoami:\n labels:\n net.unraid.docker.icon: x\n"; + file_put_contents($oldFolder . '/compose.override.yaml', $labels); + $stack = \StackInfo::createNew($this->composeRoot, 'legacy', '', $oldFolder); + $stackDir = $this->composeRoot . '/' . $stack->projectFolder; + + $this->manager->convert($stack->projectFolder, $this->upstream, 'main', 'whoami/compose.yaml', $this->clonesRoot); + + // A git stack's override lives in the stack folder, not next to the old compose file. + $this->assertSame($labels, file_get_contents($stackDir . '/compose.override.yaml')); + $this->assertSame($labels, file_get_contents($oldFolder . '/compose.override.yaml')); + } + + public function testConvertThatFailsPartWaySaysHowToUndoIt(): void + { + $stack = \StackInfo::createNew($this->composeRoot, 'myapp'); + $stackDir = $this->composeRoot . '/' . $stack->projectFolder; + // A folder where indirect_mode must be written makes that write fail + // after the old compose file was moved into the backup folder. + mkdir($stackDir . '/indirect_mode'); + + try { + $this->manager->convert($stack->projectFolder, $this->upstream, 'main', 'whoami/compose.yaml', $this->clonesRoot); + $this->fail('Converted although indirect_mode could not be written'); + } catch (RuntimeException $error) { + $this->assertStringContainsString("Could not write $stackDir/indirect_mode.", $error->getMessage()); + $this->assertStringContainsString('only partly converted', $error->getMessage()); + $this->assertStringContainsString($stackDir . '/pre-git-', $error->getMessage()); + } + $backups = glob($stackDir . '/pre-git-*') ?: []; + $this->assertCount(1, $backups); + $this->assertFileExists($backups[0] . '/compose.yaml'); + } + + public function testConvertRefusesAGitStack(): void + { + $folder = $this->manager->add('whoami', $this->upstream, 'main', 'whoami/compose.yaml', $this->clonesRoot); + $this->expectException(RuntimeException::class); + $this->expectExceptionMessage('already a git stack'); + $this->manager->convert($folder, $this->upstream, 'main', 'whoami/compose.yaml', $this->clonesRoot); + } + + public function testConvertWaitsForNoOneElse(): void + { + $stack = \StackInfo::createNew($this->composeRoot, 'myapp'); + mkdir(COMPOSE_LOCK_DIR, 0755, true); + $lock = fopen(COMPOSE_LOCK_DIR . '/' . $stack->projectName . '.lock', 'c'); + $this->assertNotFalse($lock); + flock($lock, LOCK_EX); + + try { + $this->manager->convert($stack->projectFolder, $this->upstream, 'main', 'whoami/compose.yaml', $this->clonesRoot); + $this->fail('Converted while another operation held the lock'); + } catch (RuntimeException $error) { + $this->assertStringContainsString('in progress', $error->getMessage()); + } finally { + flock($lock, LOCK_UN); + fclose($lock); + } + $this->assertFileExists($this->composeRoot . '/' . $stack->projectFolder . '/compose.yaml'); + } + + public function testStacksAreNamedExactly(): void + { + $this->manager->add('whoami', $this->upstream, 'main', 'whoami/compose.yaml', $this->clonesRoot); + $this->expectException(RuntimeException::class); + $this->manager->status('Whoami'); + } + + // ----- check and status ----- + + public function testCheckComparesTheDeployedCommitWithTheRemote(): void + { + $folder = $this->manager->add('whoami', $this->upstream, 'main', 'whoami/compose.yaml', $this->clonesRoot); + $stackDir = $this->composeRoot . '/' . $folder; + $this->assertFalse($this->manager->check($folder)['upToDate']); + + (new GitStackState($this->upstreamHead(), null))->save($stackDir); + $this->assertTrue($this->manager->check($folder)['upToDate']); + + $this->writeAndPush(['whoami/compose.yaml' => "services: {}\n"], 'change'); + $result = $this->manager->check($folder); + $this->assertFalse($result['upToDate']); + $this->assertSame($this->upstreamHead(), $result['remoteCommit']); + } + + public function testStatusShowsLocalChangesWithoutAskingTheRemote(): void + { + $folder = $this->manager->add('whoami', $this->upstream, 'main', 'whoami/compose.yaml', $this->clonesRoot); + $settings = GitStackSettings::load($this->composeRoot . '/' . $folder); + file_put_contents($settings->composeFileInClone(), "edited\n"); + rename($this->upstream, $this->upstream . '.gone'); + + $status = $this->manager->status($folder); + + $this->assertSame(['whoami/compose.yaml'], $status['localChanges']); + $this->assertNull($status['problem']); + $this->assertSame([$folder], $this->manager->listGitStacks()); + } + + // ----- reclone ----- + + public function testRecloneMovesTheOldCloneAsideAndReturnsToTheDeployedCommit(): void + { + $folder = $this->manager->add('whoami', $this->upstream, 'main', 'whoami/compose.yaml', $this->clonesRoot); + $stackDir = $this->composeRoot . '/' . $folder; + $deployed = $this->upstreamHead(); + (new GitStackState($deployed, null))->save($stackDir); + $settings = GitStackSettings::load($stackDir); + file_put_contents($settings->cloneDir . '/whoami/data.db', "keep me\n"); + $this->writeAndPush(['whoami/compose.yaml' => "services: {}\n"], 'newer'); + + $movedTo = $this->manager->reclone($folder); + + $this->assertNotNull($movedTo); + $this->assertSame("keep me\n", file_get_contents($movedTo . '/whoami/data.db')); + $this->assertFileDoesNotExist($settings->cloneDir . '/whoami/data.db'); + $this->assertSame($deployed, (new GitClone($settings))->checkedOutCommit()); + } + + public function testRecloneNamesTheUntrackedFilesLeftInTheOldClone(): void + { + $messages = []; + $manager = new GitStackManager($this->composeRoot, static function (string $message) use (&$messages): void { + $messages[] = $message; + }); + $folder = $manager->add('whoami', $this->upstream, 'main', 'whoami/compose.yaml', $this->clonesRoot); + $settings = GitStackSettings::load($this->composeRoot . '/' . $folder); + mkdir($settings->cloneDir . '/whoami/data'); + file_put_contents($settings->cloneDir . '/whoami/data/app.db', "keep me\n"); + + $manager->reclone($folder); + + $this->assertStringContainsString('whoami/data/', implode("\n", $messages)); + } + + public function testRecloneAfterTheDeployedCommitWasRewrittenAwayStaysAtTheNewHead(): void + { + $folder = $this->manager->add('whoami', $this->upstream, 'main', 'whoami/compose.yaml', $this->clonesRoot); + $stackDir = $this->composeRoot . '/' . $folder; + $this->writeAndPush(['whoami/compose.yaml' => "services: {}\n"], 'to be rewritten'); + (new GitStackState($this->upstreamHead(), null))->save($stackDir); + $this->git(['reset', '-q', '--hard', 'HEAD~1'], $this->author); + file_put_contents($this->author . '/whoami/other.txt', "x\n"); + $this->git(['add', '--', 'whoami/other.txt'], $this->author); + $this->git(['-c', 'user.name=t', '-c', 'user.email=t@example.invalid', 'commit', '-q', '-m', 'rewritten'], $this->author); + $this->git(['push', '-q', '-f', 'origin', 'main'], $this->author); + // Until the server prunes it, a rewritten-away commit can still be fetched. + $this->git(['--git-dir=' . $this->upstream, 'reflog', 'expire', '--expire=now', '--all'], null); + $this->git(['--git-dir=' . $this->upstream, 'gc', '-q', '--prune=now'], null); + + $this->manager->reclone($folder); + + $settings = GitStackSettings::load($stackDir); + $this->assertSame($this->upstreamHead(), (new GitClone($settings))->checkedOutCommit()); + $this->assertNull(GitStackState::load($stackDir)->deployedCommit); + } + + public function testConvertNamesTheFoldersNextToTheOldComposeFile(): void + { + $messages = []; + $manager = new GitStackManager($this->composeRoot, static function (string $message) use (&$messages): void { + $messages[] = $message; + }); + $oldFolder = $this->mnt . '/user/appdata/legacy'; + mkdir($oldFolder . '/db', 0755, true); + file_put_contents($oldFolder . '/compose.yaml', "services:\n whoami:\n image: traefik/whoami\n"); + $stack = \StackInfo::createNew($this->composeRoot, 'legacy', '', $oldFolder); + + $manager->convert($stack->projectFolder, $this->upstream, 'main', 'whoami/compose.yaml', $this->clonesRoot); + + $this->assertStringContainsString('next to the old compose file: db.', implode("\n", $messages)); + } + + public function testRecloneThatFailsPutsTheOldCloneBack(): void + { + $folder = $this->manager->add('whoami', $this->upstream, 'main', 'whoami/compose.yaml', $this->clonesRoot); + $settings = GitStackSettings::load($this->composeRoot . '/' . $folder); + rename($this->upstream, $this->upstream . '.gone'); + + try { + $this->manager->reclone($folder); + $this->fail('Recloned from a repository that is gone'); + } catch (RuntimeException) { + } + $this->assertFileExists($settings->composeFileInClone()); + (new GitClone($settings))->assertOwned(); + $this->assertSame([], glob($settings->cloneDir . '.replaced-*') ?: []); + } + + public function testRecloneLeavesASymlinkedCloneFolderAlone(): void + { + $folder = $this->manager->add('whoami', $this->upstream, 'main', 'whoami/compose.yaml', $this->clonesRoot); + $settings = GitStackSettings::load($this->composeRoot . '/' . $folder); + rename($settings->cloneDir, $settings->cloneDir . '.real'); + symlink($settings->cloneDir . '.real', $settings->cloneDir); + + try { + $this->manager->reclone($folder); + $this->fail('Recloned through a symlink'); + } catch (RuntimeException $error) { + $this->assertStringContainsString('is a symlink', $error->getMessage()); + } + $this->assertSame($settings->cloneDir . '.real', readlink($settings->cloneDir)); + $this->assertFileExists($settings->cloneDir . '.real/whoami/compose.yaml'); + $this->assertSame([], glob($settings->cloneDir . '.replaced-*') ?: []); + } + + public function testRecloneThatCannotPutTheOldCloneBackSaysWhereItIs(): void + { + $folder = $this->manager->add('whoami', $this->upstream, 'main', 'whoami/compose.yaml', $this->clonesRoot); + $settings = GitStackSettings::load($this->composeRoot . '/' . $folder); + $messages = []; + // Something appears at the clone folder once the old clone is moved + // aside, so the new clone fails and the old one cannot go back. + $manager = new GitStackManager($this->composeRoot, static function (string $message) use (&$messages, $settings): void { + $messages[] = $message; + if (str_starts_with($message, 'Cloning ')) { + file_put_contents($settings->cloneDir, 'in the way'); + } + }); + + try { + $manager->reclone($folder); + $this->fail('Recloned over a file in the way'); + } catch (RuntimeException) { + } + $moved = glob($settings->cloneDir . '.replaced-*') ?: []; + $this->assertCount(1, $moved); + $this->assertContains("Could not move the old clone back. It is still at {$moved[0]}.", $messages); + $this->assertNotContains('Moved the old clone back.', $messages); + } + + // ----- command line ----- + + public function testCommandLineUsageErrorsExitWithTwo(): void + { + $this->assertSame(2, $this->runCli(['compose-git', 'frobnicate'])); + $this->assertSame(2, $this->runCli(['compose-git', 'add', 'whoami', '--path', 'whoami/compose.yaml'])); + $this->assertSame(2, $this->runCli(['compose-git', 'status', 'whoami', '--colour'])); + } + + public function testCommandLineAddThenCheckReportsNothingDeployedYet(): void + { + $add = ['compose-git', 'add', 'whoami', '--url', $this->upstream, '--path', 'whoami/compose.yaml', '--clones-root', $this->clonesRoot]; + $this->assertSame(0, $this->runCli($add)); + $this->assertSame(3, $this->runCli(['compose-git', 'check', 'whoami'])); + $this->assertSame(1, $this->runCli(['compose-git', 'check', 'nosuchstack'])); + } + + public function testDeployWaitsAsTheStacksSettingSaysUnlessTheCommandLineOverrides(): void + { + $on = ['enabled' => true, 'timeout' => '120']; + $off = ['enabled' => false, 'timeout' => '300']; + + $this->assertSame(['--wait', '--wait-timeout', '120'], compose_git_wait_arguments($on, [])); + $this->assertSame([], compose_git_wait_arguments($off, [])); + $this->assertSame([], compose_git_wait_arguments($on, ['no-wait' => true])); + $this->assertSame(['--wait', '--wait-timeout', '300'], compose_git_wait_arguments($off, ['wait' => true])); + $this->assertSame(['--wait', '--wait-timeout', '45'], compose_git_wait_arguments($off, ['wait-timeout' => '45'])); + // A timeout file holding something other than a number is not passed on. + $this->assertSame(['--wait'], compose_git_wait_arguments(['enabled' => true, 'timeout' => '5m'], [])); + } + + public function testDeployRefusesWaitAndNoWaitTogether(): void + { + $this->expectException(\ComposeGitUsageError::class); + compose_git_wait_arguments(['enabled' => false, 'timeout' => '300'], ['wait' => true, 'no-wait' => true]); + } + + public function testDeployKeepsTheRunningProfilesUnlessTheCommandLineNamesSome(): void + { + $folder = $this->manager->add('whoami', $this->upstream, 'main', 'whoami/compose.yaml', $this->clonesRoot); + $stackDir = $this->composeRoot . '/' . $folder; + // StackInfo caches what it reads, so read the stack again after each change. + $stack = function () use ($folder): \StackInfo { + \StackInfo::clearCache(); + return $this->manager->gitStack($folder); + }; + + // Nothing running and no defaults: no profiles. + $this->assertSame([], compose_git_profile_arguments($stack(), [])); + + // The default profiles, as the web UI's Update uses on a first run. + file_put_contents($stackDir . '/default_profile', 'media'); + $this->assertSame(['-gmedia'], compose_git_profile_arguments($stack(), [])); + + // The profiles the stack is running with come first. + file_put_contents($stackDir . '/running_profiles', 'extras,tools'); + $this->assertSame(['-gextras', '-gtools'], compose_git_profile_arguments($stack(), [])); + + // --profile replaces both. + $this->assertSame(['-gdebug'], compose_git_profile_arguments($stack(), ['profile' => ['debug']])); + } + + public function testCommandLineStatusAsJson(): void + { + $this->manager->add('whoami', $this->upstream, 'main', 'whoami/compose.yaml', $this->clonesRoot); + $errors = fopen('php://memory', 'w'); + ob_start(); + $exit = compose_git_main(['compose-git', 'status', '--json'], $this->composeRoot, $errors); + $output = (string) ob_get_clean(); + fclose($errors); + + $this->assertSame(0, $exit); + $decoded = json_decode($output, true); + $this->assertSame('whoami', $decoded[0]['stack']); + $this->assertSame('main', $decoded[0]['branch']); + } + + public function testCommandLineStatusOfAllShowsAStackWhoseSettingsCannotBeRead(): void + { + $good = $this->manager->add('whoami', $this->upstream, 'main', 'whoami/compose.yaml', $this->clonesRoot); + $bad = $this->manager->add('broken', $this->upstream, 'main', 'whoami/compose.yaml', $this->clonesRoot); + $gitJson = $this->composeRoot . '/' . $bad . '/git.json'; + $settings = json_decode((string) file_get_contents($gitJson), true); + $settings['typo'] = true; + file_put_contents($gitJson, json_encode($settings)); + + [$exit, $output] = $this->runCliCapturingOutput(['compose-git', 'status', '--json']); + + $this->assertSame(0, $exit); + $rows = array_column(json_decode($output, true), null, 'stack'); + $this->assertSame('main', $rows[$good]['branch']); + $this->assertNull($rows[$bad]['url']); + $this->assertNotNull($rows[$bad]['problem']); + // Named on its own, the stack is still an error. + $this->assertSame(1, $this->runCli(['compose-git', 'status', $bad])); + } + + /** + * @return array + */ + public static function commandsAndTheirOptions(): array + { + return [ + 'add' => ['add', ['--url', '--path', '--branch', '--clones-root', '--description']], + 'convert' => ['convert', ['--url', '--path', '--branch', '--clones-root']], + 'check' => ['check', ['--all']], + 'deploy' => ['deploy', ['--commit', '--save-local-changes', '--wait', '--no-wait', '--wait-timeout', '--profile']], + 'reclone' => ['reclone', []], + 'status' => ['status', ['--all', '--json']], + ]; + } + + /** + * @param string[] $options + */ + #[\PHPUnit\Framework\Attributes\DataProvider('commandsAndTheirOptions')] + public function testEachCommandHasHelpNamingEveryOption(string $command, array $options): void + { + [$exit, $output] = $this->runCliCapturingOutput(['compose-git', $command, '--help']); + + $this->assertSame(0, $exit); + $this->assertStringStartsWith("Usage: compose-git $command", $output); + foreach ($options as $option) { + $this->assertStringContainsString($option, $output); + } + foreach (explode("\n", $output) as $line) { + $this->assertLessThanOrEqual(100, strlen($line), "Too long for a terminal: $line"); + } + // The other ways to ask give the same text. + $this->assertSame($output, $this->runCliCapturingOutput(['compose-git', $command, '-h'])[1]); + $this->assertSame($output, $this->runCliCapturingOutput(['compose-git', 'help', $command])[1]); + } + + public function testHelpWinsOverTheRestOfTheCommandLine(): void + { + // Nothing is cloned or checked: --help only prints. + [$exit, $output] = $this->runCliCapturingOutput(['compose-git', 'deploy', 'nosuchstack', '--help']); + + $this->assertSame(0, $exit); + $this->assertStringStartsWith('Usage: compose-git deploy', $output); + } + + public function testAMistakeInACommandShowsThatCommandsHelp(): void + { + [$exit, , $errors] = $this->runCliCapturingOutput(['compose-git', 'add', 'whoami', '--path', 'whoami/compose.yaml']); + + $this->assertSame(2, $exit); + $this->assertStringStartsWith("--url is required.\n\nUsage: compose-git add", $errors); + } + + public function testTheOverviewPointsToTheCommandHelp(): void + { + [$exit, $output] = $this->runCliCapturingOutput(['compose-git', '--help']); + + $this->assertSame(0, $exit); + $this->assertStringContainsString("compose-git --help", $output); + $this->assertSame(2, $this->runCli(['compose-git', 'help', 'frobnicate'])); + } + + // ----- helpers ----- + + /** + * @param string[] $argv + * @return array{0: int, 1: string, 2: string} The exit status, standard output and the errors + */ + private function runCliCapturingOutput(array $argv): array + { + $errors = fopen('php://memory', 'w+'); + ob_start(); + $exit = compose_git_main($argv, $this->composeRoot, $errors); + $output = (string) ob_get_clean(); + rewind($errors); + $errorText = (string) stream_get_contents($errors); + fclose($errors); + return [$exit, $output, $errorText]; + } + + /** @param string[] $argv */ + private function runCli(array $argv): int + { + $errors = fopen('php://memory', 'w'); + ob_start(); + $exit = compose_git_main($argv, $this->composeRoot, $errors); + ob_end_clean(); + fclose($errors); + return $exit; + } + + /** @param array $files */ + private function writeAndPush(array $files, string $message): void + { + foreach ($files as $path => $content) { + $full = $this->author . '/' . $path; + if (!is_dir(dirname($full))) { + mkdir(dirname($full), 0755, true); + } + file_put_contents($full, $content); + $this->git(['add', '--', $path], $this->author); + } + $this->git(['-c', 'user.name=t', '-c', 'user.email=t@example.invalid', 'commit', '-q', '-m', $message], $this->author); + $this->git(['push', '-q', 'origin', 'main'], $this->author); + } + + private function upstreamHead(): string + { + return trim(GitCommand::run(['--git-dir=' . $this->upstream, 'rev-parse', 'refs/heads/main'])->stdout); + } + + /** @param string[] $args */ + private function git(array $args, ?string $dir): void + { + $result = GitCommand::run($args, $dir); + $this->assertTrue($result->succeeded(), 'git ' . implode(' ', $args) . ': ' . $result->stderr); + } +} From 6ced90d48ba0b06fcaf952f263c8bd1e851af44a Mon Sep 17 00:00:00 2001 From: Grant Emsley Date: Sat, 3 Oct 2026 19:52:38 -0400 Subject: [PATCH 012/109] docs: add a guide to git stacks How git stacks work, a quick start, and a step-by-step guide to moving an existing stack into git (including getting its files into a repository and keeping secrets out). Also: how to lay out repositories (every stack keeps its own clone, so keep a shared repository small and give big stacks their own), which .env is used, making folders writable for non-root containers, recreating on config changes, local changes and patches, failed deploys, the pre-flight checks, and the compose-git command reference. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_011iUWDG8wTp8MhsFDqNC4NL --- docs/README.md | 1 + docs/git-stacks.md | 386 +++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 387 insertions(+) create mode 100644 docs/git-stacks.md diff --git a/docs/README.md b/docs/README.md index 73cb7c92..7046d654 100644 --- a/docs/README.md +++ b/docs/README.md @@ -8,6 +8,7 @@ Documentation for the Compose Manager plugin for Unraid. - [User Guide](user-guide.md) - How to use Compose Manager - [Configuration](configuration.md) - Settings and options - [Profiles](profiles.md) - Docker Compose profiles support +- [Git Stacks](git-stacks.md) - Deploy stacks from a git repository ## Screenshots diff --git a/docs/git-stacks.md b/docs/git-stacks.md new file mode 100644 index 00000000..8815f599 --- /dev/null +++ b/docs/git-stacks.md @@ -0,0 +1,386 @@ +# Git Stacks + +A git stack is a stack whose compose file lives in a git repository. Compose Manager keeps a clone of the +repository on your server, and deploying the stack means: fetch the latest commit, check it, pull and build +images, and start the containers from it. To change the stack, you change the repository: commit, push, +deploy. + +For now git stacks are managed from the command line with `compose-git`. Stacks made this way show up in +the Compose page like any other stack, and Start, Stop and the rest work on them as usual. + +```bash +/usr/local/emhttp/plugins/compose.manager/scripts/compose-git +``` + +To type it as `compose-git`, add `alias compose-git=/usr/local/emhttp/plugins/compose.manager/scripts/compose-git` +to your shell profile. + +## Contents + +- [How it works](#how-it-works) +- [Quick start](#quick-start) +- [Moving an existing stack into git](#moving-an-existing-stack-into-git) +- [Day to day](#day-to-day) +- [How to lay out your repositories](#how-to-lay-out-your-repositories) +- [Which .env is used](#which-env-is-used) +- [Folders and files your containers need](#folders-and-files-your-containers-need) +- [Config files and recreating containers](#config-files-and-recreating-containers) +- [Local changes on the server](#local-changes-on-the-server) +- [When a deploy fails](#when-a-deploy-fails) +- [What is checked before a deploy](#what-is-checked-before-a-deploy) +- [Command reference](#command-reference) +- [Other actions on a git stack](#other-actions-on-a-git-stack) +- [Limitations](#limitations) + +## How it works + +- **The clone** lives on your array, by default under `/mnt/user/appdata/compose.manager/git/-`, + never on the flash drive. Each git stack has its own clone of the whole repository. +- **The stack folder** in the projects folder holds the stack's settings (`git.json`), what was last deployed + (`git_state.json`), the stack's `.env` and the plugin's override file. These are never inside the clone, so + pulling new commits never touches them. +- **A deploy changes no container until everything else has worked.** It fetches the commit, checks it out, + checks it (see [below](#what-is-checked-before-a-deploy)), pulls and builds images, and only then runs + `docker compose up`. If anything before `up` fails, the previous commit is put back and your containers are + exactly as they were. +- **Nothing of yours is deleted.** Files the plugin replaces are moved into a backup folder, local changes are + saved as a patch before they are discarded, and an old clone is moved aside rather than removed. The plugin + removes only a clone it has just made itself, when setting it up fails (for example, the branch has no + compose file at the given path), and images a deploy has replaced. +- **The array must be started.** Git stacks never write anything while the array is stopped, and a path that + is not on a mounted disk, pool or share is refused, because on Unraid it would land in RAM. + +## Quick start + +You need a repository with a compose file in it, reachable over `https://` without a password (a public +repository), or a repository on this server under `/mnt` (see [step 2](#2-create-the-repository)). + +```bash +compose-git add whoami --url https://github.com/you/stacks.git --path whoami/compose.yaml +compose-git deploy whoami +``` + +`--path` is the compose file's path inside the repository. `--branch` defaults to `main`. + +## Moving an existing stack into git + +This takes a stack you already run in Compose Manager and puts it under git, keeping its containers, volumes, +`.env` and settings. The example stack is called `myapp`. + +### 1. Decide where it goes + +Read [How to lay out your repositories](#how-to-lay-out-your-repositories) first. In short: one repository +for all your stacks works well if it stays small (compose files and small config files only). A stack with +big files gets a repository of its own. + +A layout for one repository with several stacks: + +``` +stacks/ + myapp/ + compose.yaml + config/ small config files the stack mounts (optional) + .env.example the variable names the stack needs, no values (optional) + otherapp/ + compose.yaml +.gitignore +``` + +### 2. Create the repository + +Either: + +- **On GitHub, Gitea, Forgejo or similar.** It must be public for now (private repositories need credentials, + which are not supported yet; see [Limitations](#limitations)). Anything in a public repository is public, + so keep secrets out of it (step 4). +- **On this server.** A bare repository under `/mnt` works without any credentials: + + ```bash + mkdir -p /mnt/user/appdata/git-repos + git init --bare -b main /mnt/user/appdata/git-repos/stacks.git + ``` + + From your own computer you then push to it over SSH: + `git remote add origin root@tower:/mnt/user/appdata/git-repos/stacks.git`. + +### 3. Copy the stack's files into the repository + +Find the stack's current compose file. A stack created in Compose Manager keeps it in the projects folder, +by default `/boot/config/plugins/compose.manager/projects/myapp/compose.yaml`. A stack that points at another +folder (an "indirect" stack) keeps it there; the `indirect` file in the stack's projects folder says where. + +Copy it into the repository, for example to `stacks/myapp/compose.yaml`. + +Then look at its `volumes:` entries: + +- **Absolute paths** such as `/mnt/user/appdata/myapp:/config` keep working unchanged. This is where data the + container writes belongs. +- **Relative paths** such as `./config:/config` will point inside the clone after the move, because the + compose file now lives there. For each one, either: + - commit those files to the repository next to the compose file, if they are configuration you want versioned + (and small), or + - change the path to an absolute one under appdata, and copy the files there, if the container writes to + them or they are big. + + Data a container writes should never live in the clone: it would show up as local changes, and it is not + backed up by the repository. +- **Named volumes** keep working: the stack keeps its project name, so it keeps its volumes. + +### 4. Keep secrets out + +Do not commit the `.env` if it has passwords or tokens in it. Add a `.gitignore` to the repository: + +``` +.env +``` + +The stack's `.env` stays on the server, in the stack folder, and keeps being used (see +[Which .env is used](#which-env-is-used)). If you like, commit a `.env.example` listing the variable names +with no values: before each deploy, Compose Manager checks that every name in it is set in the stack's `.env`, +so a variable added later can't be forgotten. + +If you do want your `.env` versioned with the stack, that works too; it is your call. + +### 5. Commit and push + +```bash +git add stacks/myapp +git commit -m "Add myapp" +git push +``` + +### 6. Convert the stack + +```bash +compose-git convert myapp --url https://github.com/you/stacks.git --path stacks/myapp/compose.yaml +``` + +This clones the repository and points the stack at the compose file in the clone. It: + +- moves the stack's old compose file, when it is in the stack folder, into a backup folder in the stack + folder, `pre-git-` (an indirect stack's compose file stays where it is, and its old `indirect` + settings go into the backup folder); +- keeps the stack's `.env`, override file, name and settings; +- leaves your running containers alone; +- lists any folders next to the old compose file, as a reminder: a relative bind mount such as `./db` now + points inside the clone (see step 3). + +If the compose file is not on the branch, nothing is changed. + +If the stack had an auto-update schedule, set it again after converting: the schedule is stored against the +old compose file's location. + +### 7. Deploy + +```bash +compose-git deploy myapp +``` + +The stack now runs from the repository's compose file. A container whose definition is unchanged keeps +running; one that changed (a path you moved in step 3, say) is recreated. Because the project name is +unchanged, the stack keeps its volumes and networks. Check that it works. + +### 8. Tidy up + +Once you are happy, delete the `pre-git-` folder. Until then it is your way back: to undo the +conversion, delete `git.json`, `git_state.json`, `indirect` and `indirect_mode` from the stack folder, then +copy the backup folder's files back into it (they include the old `indirect` settings, if the stack had any). + +## Day to day + +1. Change the compose file (or its config files) in the repository, commit and push. +2. See whether the server is behind: `compose-git check myapp` (or `--all`). Exit status 3 means a newer + commit is waiting. +3. Deploy it: `compose-git deploy myapp`. + +To go back to an older version, deploy its commit: `compose-git deploy myapp --commit `. A +later plain `deploy` brings the stack back to the branch's latest. + +`compose-git status` shows every git stack: what is deployed, any failed commit, and local changes. + +## How to lay out your repositories + +**Every git stack keeps its own clone of the whole repository.** Ten stacks from one repository means ten +copies of every file in it on your array. That is fine for compose files and small config files, and it is +what makes a single repository for all your stacks convenient. It stops being fine when the repository holds +big things: + +- Keep a shared stacks repository **lightweight**: compose files, small config files, `.env.example` + files. No container data, backups, media, databases or downloaded binaries. +- **Give a stack its own repository** when it needs large files: the build context and source for an image + you build yourself, big configuration sets, or data files. Its size is then paid once, by that stack + alone. +- Clones download only the files of the commit they check out, not every old version (a partial clone), + so history costs little. What costs is the size of the files at the current commit, times the number of + stacks. + +A change to another stack's folder in a shared repository is fetched by every stack, but only recreates the +containers of the stack whose folder changed (see [below](#config-files-and-recreating-containers)). + +## Which .env is used + +With no custom env file set for the stack, a git stack uses, in order: + +1. the `.env` in the stack folder, if there is one (the usual place, and where secrets stay off git); +2. otherwise a `.env` committed in the repository next to the compose file; +3. otherwise none. The editor creates one in the stack folder. + +Only one of the two is used; they are not merged. Editing a `.env` that came from the repository changes the +clone, so the next deploy stops with "local changes": make the change in the repository instead. + +If the `.env` sets `COMPOSE_FILE` with relative paths, they are relative to the folder the `.env` is in. For +the stack folder's `.env` that is the stack folder, not the clone, so use paths relative to the clone only in +a `.env` committed next to the compose file. + +## Folders and files your containers need + +A bind mount whose folder does not exist yet is created by Docker on the first deploy, owned by root. Compose +Manager allows this inside a share that exists, such as `/mnt/user/appdata`, and names each folder it will +create in the deploy log. A path that looks like a mistake stops the deploy instead: a share that does not +exist (a typo such as `/mnt/user/apdata`), or a path that is not on a mounted disk, pool or share, which on +Unraid would end up in RAM. + +If your container runs as a non-root user, a folder Docker created as root is not writable for it. Two ways +to fix that from the compose file itself, so it works on every deploy: + +**A `pre_start` hook** (Docker Compose 5.3 and later; the plugin ships 5.5): + +```yaml +services: + app: + image: example/app + user: "1000:1000" + volumes: + - /mnt/user/appdata/app/data:/data + pre_start: + - command: ["sh", "-c", "mkdir -p /data && chown -R 1000:1000 /data"] + image: alpine + user: root +``` + +**An init service** that runs first and exits: + +```yaml +services: + init-perms: + image: alpine + command: ["sh", "-c", "mkdir -p /data && chown -R 1000:1000 /data"] + volumes: + - /mnt/user/appdata/app/data:/data + app: + image: example/app + user: "1000:1000" + volumes: + - /mnt/user/appdata/app/data:/data + depends_on: + init-perms: + condition: service_completed_successfully +``` + +## Config files and recreating containers + +`docker compose up` only recreates a container when its compose definition changes. A commit that only edits +a config file the container mounts would otherwise be checked out but never take effect. + +So by default, **when a deploy changes anything in the stack's folder in the repository, every container in +the stack is recreated**, and the new config takes effect. + +Only the stack's own folder counts: a change to a shared folder elsewhere in the repository does not +recreate anything yet. To turn the behaviour off, set `"recreateOnFolderChange": false` in the stack's +`git.json`. + +## Local changes on the server + +The clone is not meant to be edited. If a tracked file in it is changed (or a commit is made in it), the next +deploy stops and lists the files. Then either: + +- undo the change by hand, or +- deploy with `--save-local-changes`: the changes are saved as a patch in the stack folder's `git-changes` + folder, then discarded, and the deploy goes ahead. Apply the patch to your own copy of the repository with + `git apply ` if you want to keep it. + +Files that are not part of the repository, such as data a container wrote into the clone, are never removed +or overwritten, whether `.gitignore` lists them or not. If a new commit would put a file where one of them +is, the deploy stops instead. + +## When a deploy fails + +- **Before `up`** (a check fails, the pull or build fails): the previous commit is put back and no container + was changed. Fix the cause, usually in the repository, and deploy again. +- **During `up`** (a container fails to start, or does not get healthy with `--wait`): a half-finished `up` + cannot be undone safely, so it is not rolled back. The commit is recorded as failed and shown by + `compose-git status`. Fix it in the repository and deploy again, or deploy a known good commit with + `--commit`. + +If the clone itself gets into a bad state, `compose-git reclone myapp` moves it aside (as +`.replaced-`, nothing is deleted) and clones the repository again at the deployed commit. It +lists the untracked files left in the old clone: copy back anything you need, then delete the old clone +yourself. + +## What is checked before a deploy + +Every deploy runs `docker compose config` on the new commit, with the stack's `.env` and profiles, and stops +without changing anything when: + +- the compose file is not valid, defines no services, or uses a variable that is not set (an empty value must + be written as `NAME=` in the `.env`); +- the repository's `.env.example` lists names missing from the `.env` in use; +- an `external: true` network or volume does not exist; +- a bind-mount source is missing in a way described in + [Folders and files](#folders-and-files-your-containers-need), or a config or secret file, or a local build + folder, is missing; +- a `container_name` or published port is already taken by another stack or container. + +The deploy log says which `.env` was used. + +## Command reference + +| Command | What it does | +|---|---| +| `add --url --path [--branch ] [--clones-root ] [--description ]` | Clone a repository and make a new git stack. Not deployed yet. | +| `convert --url --path [--branch ] [--clones-root ]` | Turn an existing stack into a git stack (see above). | +| `check ` or `check --all` | Compare the deployed commit with the branch on the remote. Changes nothing. | +| `deploy [--commit ] [--save-local-changes] [--wait\|--no-wait] [--wait-timeout ] [--profile

]...` | Deploy the branch's latest commit, or the given one. Waits for healthy containers when the stack's wait-for-healthy setting says so, unless `--wait` or `--no-wait` overrides it. | +| `reclone ` | Move the clone aside and clone again at the deployed commit. | +| `status [\|--all] [--json]` | Show what is deployed, failed and changed locally, without asking the remote. | + +`compose-git --help` explains a command and each of its options. + +`` is an `https://` address without a user name or password in it, or the path of a repository under +`/mnt`. `` is the stack's exact folder name in the projects folder. `--clones-root` must be on a share, +disk or pool, and the share must exist. + +Exit status: `0` success (for `check`: up to date), `1` failed, `2` the command line was not understood, `3` +(`check`) a newer commit is available. `check --all` exits `3` if any stack is behind, even when another could +not be checked. If another operation on the same stack is running (a Start or Update from the web UI, say), +`deploy` waits up to 30 seconds (`COMPOSE_LOCK_TIMEOUT`) for it to finish, then fails without changing +anything. `convert` and `reclone` do not wait: they refuse to start while another operation on the stack is +running, so run them again once it has finished. + +Without `--profile`, `deploy` uses the profiles the stack is running with, or else its default profiles, the +same as Update in the web UI. + +## Other actions on a git stack + +The rest of Compose Manager treats a git stack like any other stack, using the commit that is checked out in +the clone. Start, Stop, Update, autostart, auto-update and stopping the array all work as usual, with the same +compose files and `.env` as a deploy. None of them fetch a new commit, run the checks above, or record a +deployed commit: only `compose-git deploy` does that. + +- **Delete** removes the stack folder and leaves the clone (see [Limitations](#limitations)). +- **Backup** saves the stack folder, not the clone, as for any stack whose compose file lives elsewhere. The + repository is the backup of the clone. +- **Settings:** do not change the stack's external compose path. The next deploy refuses, because it no longer + matches `git.json`. + +## Limitations + +- **Public `https` repositories and repositories on this server only.** Private repositories (access tokens, + SSH deploy keys) are planned. +- **No web UI yet.** Git stacks are created and deployed from the command line; the Compose page shows and + runs them like other stacks. +- **No automatic deploys yet.** Use `check` and `deploy` from your own schedule or a git hook. +- No submodules and no Git LFS: the clone checks out plain files only. +- **The web UI's editor edits the file in the clone.** The next deploy treats that as a local change. Changes + that must survive deploys belong in the plugin's override or the stack folder's `.env`. +- **Deleting a git stack leaves its clone** under the clones folder. Remove it by hand. From 9a1d46c55fa41099f62cb34d48e0b0463ae36b06 Mon Sep 17 00:00:00 2001 From: Grant Emsley Date: Sun, 4 Oct 2026 15:17:14 -0400 Subject: [PATCH 013/109] feat(git): create missing external networks in a git deploy when the setting is on With Create Missing External Networks on, compose.sh creates a stack's missing `external: true` networks before up and update. A git deploy did not: its pre-flight check refused the stack over the missing network, so the same stack started by hand but would not deploy from git. With the setting on, the pre-flight check no longer counts a missing external network as a problem (external volumes are still checked), and gitdeploy creates the missing networks right before up. That is after every check, pull and build has passed, so a deploy that stops earlier creates nothing. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_011iUWDG8wTp8MhsFDqNC4NL --- docs/git-stacks.md | 3 +- source/compose.manager/include/GitDeploy.php | 12 ++++++- .../include/GitDeployCheck.php | 12 +++++-- source/compose.manager/scripts/compose.sh | 8 +++++ tests/unit/GitDeployCheckTest.php | 18 ++++++++-- tests/unit/GitDeployTest.php | 22 ++++++++++++ tests/unit/gitdeploy.bats | 36 +++++++++++++++++++ 7 files changed, 105 insertions(+), 6 deletions(-) diff --git a/docs/git-stacks.md b/docs/git-stacks.md index 8815f599..99940736 100644 --- a/docs/git-stacks.md +++ b/docs/git-stacks.md @@ -325,7 +325,8 @@ without changing anything when: - the compose file is not valid, defines no services, or uses a variable that is not set (an empty value must be written as `NAME=` in the `.env`); - the repository's `.env.example` lists names missing from the `.env` in use; -- an `external: true` network or volume does not exist; +- an `external: true` network or volume does not exist. With **Create Missing External Networks** turned on + in the plugin's settings, a missing network is created instead, the same as for any other stack; - a bind-mount source is missing in a way described in [Folders and files](#folders-and-files-your-containers-need), or a config or secret file, or a local build folder, is missing; diff --git a/source/compose.manager/include/GitDeploy.php b/source/compose.manager/include/GitDeploy.php index bf4e064a..ab3eceb9 100644 --- a/source/compose.manager/include/GitDeploy.php +++ b/source/compose.manager/include/GitDeploy.php @@ -317,7 +317,17 @@ private function runChecks(GitStackSettings $settings, string $projectName, arra $envFile = rtrim($this->stackDir, '/') . '/.env'; } - $check = new GitDeployCheck($projectName, $args, $settings->cloneDir, dirname($composeFile), $envFile); + $pluginSettings = parse_plugin_cfg('compose.manager'); + $missingNetworksWillBeCreated = ($pluginSettings['CREATE_MISSING_EXTERNAL_NETWORKS'] ?? 'false') === 'true'; + + $check = new GitDeployCheck( + $projectName, + $args, + $settings->cloneDir, + dirname($composeFile), + $envFile, + $missingNetworksWillBeCreated + ); $problems = $check->run(); if ($problems !== []) { throw new RuntimeException("The stack was not deployed:\n- " . implode("\n- ", $problems)); diff --git a/source/compose.manager/include/GitDeployCheck.php b/source/compose.manager/include/GitDeployCheck.php index bf5ac339..31203cb0 100644 --- a/source/compose.manager/include/GitDeployCheck.php +++ b/source/compose.manager/include/GitDeployCheck.php @@ -15,7 +15,8 @@ * - compose errors, and variables that are not set (a blank value can create * state that outlives fixing it); * - keys listed in the repository's .env.example but missing from the .env; - * - external networks and volumes that do not exist; + * - external networks and volumes that do not exist (networks only when the + * "Create Missing External Networks" setting is off); * - bind-mount sources under /mnt that are not on a live mount (Docker would * create them in RAM) or whose share (such as /mnt/user/appdata) does not * exist, and sources outside /mnt whose top folder (such as /tmp) does not @@ -38,13 +39,16 @@ final class GitDeployCheck * @param string $cloneDir The stack's clone folder * @param string $composeDir The folder of the compose file inside the clone * @param string|null $envFile The .env the stack uses, if any + * @param bool $missingNetworksWillBeCreated True when the "Create Missing External Networks" + * setting is on, so a missing external network is not a problem */ public function __construct( private readonly string $projectName, private readonly array $composeArgs, private readonly string $cloneDir, private readonly string $composeDir, - private readonly ?string $envFile + private readonly ?string $envFile, + private readonly bool $missingNetworksWillBeCreated = false ) { } @@ -160,6 +164,10 @@ private function missingExternalResources(array $resolved): array if (!is_array($definition) || ($definition['external'] ?? false) !== true) { continue; } + // compose.sh creates missing external networks right before "up" when the setting is on. + if ($kind === 'network' && $this->missingNetworksWillBeCreated) { + continue; + } $name = (string) ($definition['name'] ?? $key); if (!DockerCommand::run([$kind, 'inspect', '--', $name], null, 30)->succeeded()) { $problems[] = "The external $kind '$name' does not exist. Create it before deploying."; diff --git a/source/compose.manager/scripts/compose.sh b/source/compose.manager/scripts/compose.sh index 875901da..73c31596 100755 --- a/source/compose.manager/scripts/compose.sh +++ b/source/compose.manager/scripts/compose.sh @@ -629,6 +629,14 @@ case $command in ;; esac + # Optionally create missing `external: true` networks, as up and update do + # (Settings > Compose > Create Missing External Networks). This is done + # here, after every check, pull and build has passed, so a deploy that + # stops earlier creates nothing. + if plugin_setting_enabled CREATE_MISSING_EXTERNAL_NETWORKS; then + create_missing_external_networks "${git_compose[@]}" -p "$name" + fi + # 4. Start the stack at the new commit. echo "" echo "Starting containers..." diff --git a/tests/unit/GitDeployCheckTest.php b/tests/unit/GitDeployCheckTest.php index 3a282150..575ece0d 100644 --- a/tests/unit/GitDeployCheckTest.php +++ b/tests/unit/GitDeployCheckTest.php @@ -131,6 +131,19 @@ public function testMissingExternalNetworkAndVolumeAreProblems(): void $this->assertSame([], $this->check()); } + public function testMissingExternalNetworkIsNotAProblemWhenItWillBeCreated(): void + { + $config = $this->config(['image' => 'busybox']); + $config['networks'] = ['proxy' => ['name' => 'zz-proxy', 'external' => true]]; + $config['volumes'] = ['ext' => ['name' => 'zz-ext', 'external' => true]]; + $this->docker->setConfig($config); + + // The volume is still checked: only networks are created by the setting. + $problems = $this->check(missingNetworksWillBeCreated: true); + $this->assertCount(1, $problems); + $this->assertStringContainsString("volume 'zz-ext'", $problems[0]); + } + // ----- paths ----- public function testMisspelledShareInBindSourceIsAProblem(): void @@ -369,14 +382,15 @@ private function config(array $service): array } /** @return string[] */ - private function check(): array + private function check(bool $missingNetworksWillBeCreated = false): array { $check = new GitDeployCheck( 'whoami', ['-f', $this->clone . '/whoami/compose.yaml'], $this->clone, $this->clone . '/whoami', - $this->stackDir . '/.env' + $this->stackDir . '/.env', + $missingNetworksWillBeCreated ); return $check->run(); } diff --git a/tests/unit/GitDeployTest.php b/tests/unit/GitDeployTest.php index 0a241fe3..d2bcc3ed 100644 --- a/tests/unit/GitDeployTest.php +++ b/tests/unit/GitDeployTest.php @@ -10,6 +10,7 @@ use GitDeploy; use GitStackSettings; use GitStackState; +use PluginTests\Mocks\FunctionMocks; use PluginTests\TestCase; use RuntimeException; @@ -101,6 +102,27 @@ public function testFailedCheckPutsThePreviousCommitBack(): void $this->assertSame($before, $this->clone->checkedOutCommit()); } + public function testMissingExternalNetworkIsLeftToCreateMissingExternalNetworks(): void + { + $this->docker->setConfig([ + 'name' => 'whoami', + 'services' => ['whoami' => ['image' => 'busybox']], + 'networks' => ['proxy' => ['name' => 'zz-proxy', 'external' => true]], + ]); + + try { + $this->deploy()->prepare('whoami', [], null, false); + $this->fail('Deployed with a missing external network and the setting off'); + } catch (RuntimeException $error) { + $this->assertStringContainsString("network 'zz-proxy' does not exist", $error->getMessage()); + } + + // With the setting on, compose.sh creates the network before up, so the check lets it through. + FunctionMocks::setPluginConfig('compose.manager', ['CREATE_MISSING_EXTERNAL_NETWORKS' => 'true']); + $this->deploy()->prepare('whoami', [], null, false); + $this->assertSame($this->upstreamHead(), $this->clone->checkedOutCommit()); + } + public function testComposeFileRemovedUpstreamIsNotDeployed(): void { $before = $this->clone->checkedOutCommit(); diff --git a/tests/unit/gitdeploy.bats b/tests/unit/gitdeploy.bats index d06d461c..c4d9bcbb 100644 --- a/tests/unit/gitdeploy.bats +++ b/tests/unit/gitdeploy.bats @@ -22,6 +22,10 @@ test_setup() { export PREPARE_EXIT=0 PULL_EXIT=0 BUILD_EXIT=0 UP_EXIT=0 UP_ARGUMENTS_EXIT=0 UP_ARGUMENTS="" export COMPOSE_ARGS_EXIT=0 RESTORE_EXIT=0 export PREPARE_OUTPUT="0000000000000000000000000000000000000001" + # The plugin's settings: none, so Create Missing External Networks is off unless a test turns it on. + export COMPOSE_MANAGER_CFG_FILE="$TEST_TEMP_DIR/compose.manager.cfg" + # What "docker compose config --format json" prints, and whether "docker network inspect" finds the network. + export CONFIG_JSON='{}' NETWORK_INSPECT_EXIT=0 STACK="$TEST_TEMP_DIR/whoami" mkdir -p "$STACK" "$TEST_TEMP_DIR/bin" @@ -76,6 +80,8 @@ for arg in "$@"; do pull) exit "$PULL_EXIT" ;; build) exit "$BUILD_EXIT" ;; up) exit "$UP_EXIT" ;; + config) printf '%s' "$CONFIG_JSON"; exit 0 ;; + inspect) exit "$NETWORK_INSPECT_EXIT" ;; esac done exit 0 @@ -257,3 +263,33 @@ calls_matching() { [ "$status" -eq 0 ] [ "$(grep -c -- ' up .*--build' "$CALLS" || true)" -eq 0 ] } + +@test "gitdeploy creates a missing external network after the build and before up when the setting is on" { + echo 'CREATE_MISSING_EXTERNAL_NETWORKS="true"' > "$COMPOSE_MANAGER_CFG_FILE" + export CONFIG_JSON='{"networks":{"proxy":{"name":"zz-proxy","external":true}}}' NETWORK_INSPECT_EXIT=1 + run_gitdeploy + [ "$status" -eq 0 ] + [[ "$output" == *"Created missing external network: zz-proxy"* ]] + + build_line=$(grep -n ' build$' "$CALLS" | cut -d: -f1) + create_line=$(grep -n 'docker network create zz-proxy$' "$CALLS" | cut -d: -f1) + up_line=$(grep -n ' up ' "$CALLS" | cut -d: -f1) + [ "$build_line" -lt "$create_line" ] + [ "$create_line" -lt "$up_line" ] +} + +@test "gitdeploy creates no network when the setting is off" { + export CONFIG_JSON='{"networks":{"proxy":{"name":"zz-proxy","external":true}}}' NETWORK_INSPECT_EXIT=1 + run_gitdeploy + [ "$status" -eq 0 ] + [ "$(calls_matching 'network create')" -eq 0 ] +} + +@test "gitdeploy creates no network when it stops before up" { + echo 'CREATE_MISSING_EXTERNAL_NETWORKS="true"' > "$COMPOSE_MANAGER_CFG_FILE" + export CONFIG_JSON='{"networks":{"proxy":{"name":"zz-proxy","external":true}}}' NETWORK_INSPECT_EXIT=1 + export BUILD_EXIT=1 + run_gitdeploy + [ "$status" -eq 1 ] + [ "$(calls_matching 'network create')" -eq 0 ] +} From c2fb1071f52d7c8547273b32a76f017951970f5c Mon Sep 17 00:00:00 2001 From: Grant Emsley Date: Tue, 6 Oct 2026 14:37:58 -0400 Subject: [PATCH 014/109] feat(git): prune the override's orphan services before a git deploy's checks The web UI prunes the plugin-managed override before every up; a git deploy did not. When a commit renames or removes a service, the override keeps an entry for it with labels only, and compose refuses the whole stack over a file the user did not write. Prune it after the checkout and before the checks, and log each service removed. This relies on #169: before it, pruneOrphanOverrideServices() never removed anything. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01PnMScBowJk7yyfEwraTdgg --- docs/git-stacks.md | 7 ++++- source/compose.manager/include/GitDeploy.php | 22 ++++++++++++++ tests/unit/GitDeployTest.php | 32 ++++++++++++++++++++ tests/unit/fixtures/fake-docker.sh | 18 +++++++++++ tests/unit/support/FakeDocker.php | 10 ++++++ 5 files changed, 88 insertions(+), 1 deletion(-) diff --git a/docs/git-stacks.md b/docs/git-stacks.md index 99940736..bca8d9eb 100644 --- a/docs/git-stacks.md +++ b/docs/git-stacks.md @@ -46,7 +46,8 @@ to your shell profile. - **Nothing of yours is deleted.** Files the plugin replaces are moved into a backup folder, local changes are saved as a patch before they are discarded, and an old clone is moved aside rather than removed. The plugin removes only a clone it has just made itself, when setting it up fails (for example, the branch has no - compose file at the given path), and images a deploy has replaced. + compose file at the given path), images a deploy has replaced, and override entries for services the + compose file no longer has. - **The array must be started.** Git stacks never write anything while the array is stopped, and a path that is not on a mounted disk, pool or share is refused, because on Unraid it would land in RAM. @@ -334,6 +335,10 @@ without changing anything when: The deploy log says which `.env` was used. +Before the checks, the plugin's override loses its entries (UI labels) for services the compose file no +longer has, as it does before every Compose Up from the web UI. An entry for a renamed service would +otherwise make compose refuse the stack. The log names each one removed. + ## Command reference | Command | What it does | diff --git a/source/compose.manager/include/GitDeploy.php b/source/compose.manager/include/GitDeploy.php index ab3eceb9..dda59159 100644 --- a/source/compose.manager/include/GitDeploy.php +++ b/source/compose.manager/include/GitDeploy.php @@ -301,6 +301,8 @@ private function runChecks(GitStackSettings $settings, string $projectName, arra throw new RuntimeException("{$settings->composePath} leads outside the repository (through a symlink), so it was not used."); } + $this->pruneOverrideServicesTheComposeFileNoLongerHas(); + $args = $this->composeArgs(); foreach ($profiles as $profile) { $args[] = '--profile'; @@ -336,4 +338,24 @@ private function runChecks(GitStackSettings $settings, string $projectName, arra ($this->say)("Docker will create the missing folder $folder"); } } + + /** + * Remove services from the plugin-managed override that the new commit's + * compose file no longer has, as the web UI does before every "up". + * + * An override entry for a renamed or removed service defines a service + * with no image, and compose would refuse the whole stack over a file the + * user did not write. The entries only hold the plugin's UI labels. If the + * checks then fail and the previous commit is put back, the labels of a + * removed service are not restored; they are only labels. + */ + private function pruneOverrideServicesTheComposeFileNoLongerHas(): void + { + StackInfo::clearCache(); + $stack = StackInfo::fromProject(dirname(rtrim($this->stackDir, '/')), basename(rtrim($this->stackDir, '/'))); + $pruned = $stack->pruneOrphanOverrideServices(); + foreach ($pruned['removed'] as $service) { + ($this->say)("Removed '$service' from the plugin's override: the compose file no longer has that service."); + } + } } diff --git a/tests/unit/GitDeployTest.php b/tests/unit/GitDeployTest.php index d2bcc3ed..2f76a976 100644 --- a/tests/unit/GitDeployTest.php +++ b/tests/unit/GitDeployTest.php @@ -419,6 +419,38 @@ public function testComposeFileReachedThroughASymlinkOutsideTheCloneIsRefused(): $this->assertSame($before, $this->clone->checkedOutCommit()); } + public function testOverrideEntryForAServiceTheCommitRemovedIsPrunedBeforeTheChecks(): void + { + // The override has UI labels for 'gone', which the new commit renames to 'whoami2'. + $override = (string) \StackInfo::fromProject($this->composeRoot, 'whoami')->getOverridePath(); + $this->assertStringStartsWith($this->stackDir . '/', $override); + file_put_contents( + $override, + "services:\n whoami:\n labels:\n net.unraid.docker.webui: http://x\n" + . " gone:\n labels:\n net.unraid.docker.webui: http://y\n" + ); + $this->writeAndPush(['whoami/compose.yaml' => "services:\n whoami:\n image: traefik/whoami\n whoami2:\n image: busybox\n"], 'rename gone'); + $this->docker->setServices(['whoami', 'whoami2']); + $said = []; + $deploy = new GitDeploy($this->stackDir, static function (string $message) use (&$said): void { + $said[] = $message; + }); + + // StackInfo runs "docker" from the PATH, so put the scripted one first. + $originalPath = (string) getenv('PATH'); + putenv('PATH=' . $this->docker->dir . ':' . $originalPath); + try { + $deploy->prepare('whoami', [], null, false); + } finally { + putenv('PATH=' . $originalPath); + } + + $content = (string) file_get_contents($override); + $this->assertStringNotContainsString('gone', $content); + $this->assertStringContainsString('whoami', $content); + $this->assertContains("Removed 'gone' from the plugin's override: the compose file no longer has that service.", $said); + } + // ----- helpers ----- private function deploy(): GitDeploy diff --git a/tests/unit/fixtures/fake-docker.sh b/tests/unit/fixtures/fake-docker.sh index fbbec24a..09459b66 100644 --- a/tests/unit/fixtures/fake-docker.sh +++ b/tests/unit/fixtures/fake-docker.sh @@ -3,6 +3,8 @@ # # It answers from files next to itself (the tests write them): # config.json / config.stderr / config.exit docker compose ... config --format json +# services.txt docker compose ... config --services (refused, +# as compose does, when given an override file) # networks/, volumes/ these exist (network/volume inspect succeeds); # a network file holds its driver, if any # containers/.json docker container inspect @@ -13,6 +15,22 @@ printf '%s\n' "$*" >> "$here/calls.log" case "$1" in compose) + for arg in "$@"; do + if [ "$arg" = "--services" ]; then + # The override in the tests holds an entry for a service with no image, + # which makes compose refuse the project when the override is loaded. + for file in "$@"; do + case "$file" in + *override*) + echo 'service "gone" has neither an image nor a build context specified: invalid compose project' >&2 + exit 1 + ;; + esac + done + cat "$here/services.txt" 2>/dev/null + exit 0 + fi + done for arg in "$@"; do if [ "$arg" = "config" ]; then [ -f "$here/config.stderr" ] && cat "$here/config.stderr" >&2 diff --git a/tests/unit/support/FakeDocker.php b/tests/unit/support/FakeDocker.php index c3f5a80f..d2de8db8 100644 --- a/tests/unit/support/FakeDocker.php +++ b/tests/unit/support/FakeDocker.php @@ -40,6 +40,16 @@ public function setConfig(array $config, string $stderr = '', int $exitCode = 0) file_put_contents($this->dir . '/config.exit', (string) $exitCode); } + /** + * What `docker compose config --services` prints. + * + * @param string[] $services + */ + public function setServices(array $services): void + { + file_put_contents($this->dir . '/services.txt', implode("\n", $services) . "\n"); + } + /** * A network that exists, with its driver (empty for the default, bridge). */ From 85a7c91dc6f7acb64c0779d883e649d77d281263 Mon Sep 17 00:00:00 2001 From: Grant Emsley Date: Tue, 6 Oct 2026 19:35:20 -0400 Subject: [PATCH 015/109] test(git): run the gitdeploy Bats tests without root, as CI does compose.sh kept its lock files in a fixed /var/run/compose.manager, which only root can create. The GitHub runner is not root, so every gitdeploy test failed at the lock with "Another operation is already in progress". The local test container runs as root, which hid it. The lock folder can now be set with COMPOSE_LOCK_DIR, the same way COMPOSE_LOCK_TIMEOUT already sets the wait. The default is unchanged. The gitdeploy tests point it at their temporary folder. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01PnMScBowJk7yyfEwraTdgg --- source/compose.manager/scripts/compose.sh | 2 +- tests/unit/gitdeploy.bats | 6 ++++-- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/source/compose.manager/scripts/compose.sh b/source/compose.manager/scripts/compose.sh index 73c31596..2ee10b8c 100755 --- a/source/compose.manager/scripts/compose.sh +++ b/source/compose.manager/scripts/compose.sh @@ -9,7 +9,7 @@ export HOME=/root # Configuration - can be overridden via environment LOCK_TIMEOUT=${COMPOSE_LOCK_TIMEOUT:-30} -LOCK_DIR="/var/run/compose.manager" +LOCK_DIR=${COMPOSE_LOCK_DIR:-/var/run/compose.manager} SHORT=e:,c:,f:,p:,d:,o:,g:,s:,w: LONG=env,command:,file:,project_name:,project_dir:,override:,profile:,debug,recreate,remove-orphans,stack-path:,workdir:,follow-logs,wait,wait-timeout:,build,credential-id:,git-commit:,save-local-changes diff --git a/tests/unit/gitdeploy.bats b/tests/unit/gitdeploy.bats index c4d9bcbb..a1c5b8a5 100644 --- a/tests/unit/gitdeploy.bats +++ b/tests/unit/gitdeploy.bats @@ -13,6 +13,8 @@ test_setup() { # The framework exports mock docker/php functions; these tests use scripts on PATH instead. unset -f docker php 2>/dev/null || true export COMPOSE_LOCK_TIMEOUT=5 + # /var/run is not writable when the tests run without root (as in CI). + export COMPOSE_LOCK_DIR="$TEST_TEMP_DIR/locks" REAL_PHP="$(command -v php)" export REAL_PHP export CALLS="$TEST_TEMP_DIR/calls.log" @@ -108,8 +110,8 @@ calls_matching() { @test "gitdeploy waits for the stack's lock and changes nothing when it stays taken" { # Another operation on the stack (a Start from the web UI, say) holds its lock. - mkdir -p /var/run/compose.manager - exec 8>/var/run/compose.manager/whoami.lock + mkdir -p "$COMPOSE_LOCK_DIR" + exec 8>"$COMPOSE_LOCK_DIR/whoami.lock" flock 8 export COMPOSE_LOCK_TIMEOUT=1 run_gitdeploy From 256afa7d352540199b507ca5fca37a67deaf68dd Mon Sep 17 00:00:00 2001 From: Grant Emsley Date: Wed, 7 Oct 2026 09:34:19 -0400 Subject: [PATCH 016/109] fix(git): stop the deploy when Docker cannot say which names and ports are taken The container-name and port checks took any failed docker command as "nothing is taken": a daemon that is not answering, or a timeout, let the deploy go on to "up", which would then find the clash after changing some containers. Now only "No such container" means the name is free. Any other failure, and output that is not JSON, is reported as a problem and stops the deploy before "up", like every other check. A container removed between "docker ps" and the bulk inspect is still fine: inspect prints the others and the removed one holds no port. The fake docker used by the tests can now be told to fail a command. Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_01NavH8cj9aU49Qs1pQPa1Qc --- .../include/GitDeployCheck.php | 62 +++++++++++++++++-- tests/unit/GitDeployCheckTest.php | 56 +++++++++++++++++ tests/unit/fixtures/fake-docker.sh | 8 +++ tests/unit/support/FakeDocker.php | 14 +++++ 4 files changed, 136 insertions(+), 4 deletions(-) diff --git a/source/compose.manager/include/GitDeployCheck.php b/source/compose.manager/include/GitDeployCheck.php index 31203cb0..21d16f08 100644 --- a/source/compose.manager/include/GitDeployCheck.php +++ b/source/compose.manager/include/GitDeployCheck.php @@ -24,6 +24,9 @@ * are left for Docker to create, as are paths inside the clone; * - config and secret files, and local build folders, that do not exist; * - container names and published ports already taken by another container. + * When Docker cannot answer a question (the daemon is not responding, a + * command times out), that is a problem too: "up" must not be the one to find + * out, after it has changed some of the containers. */ final class GitDeployCheck { @@ -293,12 +296,21 @@ private function takenContainerNames(array $resolved): array $name = (string) $service['container_name']; $inspect = DockerCommand::run(['container', 'inspect', '--', $name], null, 30); if (!$inspect->succeeded()) { + if (self::failedOnlyOnMissingContainers($inspect)) { + // Nothing has that name. + continue; + } + $problems[] = "Could not check whether the container name '$name' is free: " . self::failureText($inspect); continue; } $containers = json_decode($inspect->stdout, true); + if (!is_array($containers)) { + $problems[] = "Could not check whether the container name '$name' is free: docker inspect did not print valid JSON."; + continue; + } // docker inspect also matches the start of a container id, so a short // hex-looking name such as "cafe" can find an unrelated container. - $foundName = is_array($containers) ? ltrim((string) ($containers[0]['Name'] ?? ''), '/') : ''; + $foundName = ltrim((string) ($containers[0]['Name'] ?? ''), '/'); if ($foundName !== $name) { continue; } @@ -349,7 +361,11 @@ private function takenPorts(array $resolved): array return []; } - $heldByOthers = $this->portsHeldByOtherContainers(); + try { + $heldByOthers = $this->portsHeldByOtherContainers(); + } catch (RuntimeException $error) { + return ['Could not check whether the published ports are free: ' . $error->getMessage()]; + } $problems = []; foreach ($wanted as $want) { @@ -371,19 +387,28 @@ private function takenPorts(array $resolved): array * Ports published by running containers of other stacks, or by plain containers. * * @return list + * @throws RuntimeException when Docker could not say */ private function portsHeldByOtherContainers(): array { $result = []; $ids = DockerCommand::run(['ps', '-q'], null, 30); + if (!$ids->succeeded()) { + throw new RuntimeException('docker ps failed: ' . self::failureText($ids)); + } $idList = array_values(array_filter(explode("\n", trim($ids->stdout)))); - if (!$ids->succeeded() || $idList === []) { + if ($idList === []) { return $result; } $inspect = DockerCommand::run(array_merge(['container', 'inspect', '--'], $idList), null, 60); + // A container removed between the two commands makes inspect fail, + // but it still prints the others, and the removed one holds no port. + if (!$inspect->succeeded() && !self::failedOnlyOnMissingContainers($inspect)) { + throw new RuntimeException('docker inspect failed: ' . self::failureText($inspect)); + } $containers = json_decode($inspect->stdout, true); if (!is_array($containers)) { - return $result; + throw new RuntimeException('docker inspect did not print valid JSON.'); } foreach ($containers as $container) { @@ -452,6 +477,35 @@ private function externalNetworkDriver(string $name): string return $this->externalNetworkDrivers[$name]; } + /** + * Whether a failed "docker container inspect" failed only because a + * container it was asked about does not exist. Any other failure (the + * daemon not answering, a timeout) means the question was not answered. + */ + private static function failedOnlyOnMissingContainers(ProcessResult $inspect): bool + { + if ($inspect->timedOut) { + return false; + } + $lines = array_filter(array_map('trim', explode("\n", $inspect->stderr)), static fn(string $line): bool => $line !== ''); + if ($lines === []) { + return false; + } + foreach ($lines as $line) { + // "Error response from daemon: No such container: x" (docker container inspect), + // "error: no such object: x" (docker inspect). + if (stripos($line, 'no such container') === false && stripos($line, 'no such object') === false) { + return false; + } + } + return true; + } + + private static function failureText(ProcessResult $result): string + { + return trim($result->stderr) !== '' ? trim($result->stderr) : $result->errorSummary(); + } + private static function addressesOverlap(string $a, string $b): bool { $any = ['', '0.0.0.0', '::']; diff --git a/tests/unit/GitDeployCheckTest.php b/tests/unit/GitDeployCheckTest.php index 575ece0d..1caab066 100644 --- a/tests/unit/GitDeployCheckTest.php +++ b/tests/unit/GitDeployCheckTest.php @@ -284,6 +284,62 @@ public function testContainerNameTakenByAPlainDockerContainerIsAProblem(): void $this->assertCount(1, $this->check()); } + public function testContainerNameCheckThatDockerCannotAnswerIsAProblem(): void + { + $this->docker->setConfig($this->config(['image' => 'busybox', 'container_name' => 'plex'])); + $this->docker->fail('container', 'Cannot connect to the Docker daemon at unix:///var/run/docker.sock. Is the docker daemon running?'); + + $problems = $this->check(); + $this->assertCount(1, $problems); + $this->assertStringContainsString("Could not check whether the container name 'plex' is free: Cannot connect", $problems[0]); + } + + public function testPortCheckThatCannotListTheRunningContainersIsAProblem(): void + { + $this->docker->setConfig($this->config(['image' => 'busybox', 'ports' => [ + ['target' => 80, 'published' => '8080', 'protocol' => 'tcp'], + ]])); + $this->docker->fail('ps', 'Cannot connect to the Docker daemon at unix:///var/run/docker.sock. Is the docker daemon running?'); + + $problems = $this->check(); + $this->assertCount(1, $problems); + $this->assertStringContainsString('Could not check whether the published ports are free: docker ps failed: Cannot connect', $problems[0]); + } + + public function testPortCheckThatCannotInspectTheRunningContainersIsAProblem(): void + { + $this->docker->setConfig($this->config(['image' => 'busybox', 'ports' => [ + ['target' => 80, 'published' => '8080', 'protocol' => 'tcp'], + ]])); + $this->docker->setRunning([ + ['name' => 'nginx', 'project' => null, 'ports' => ['80/tcp' => [['HostIp' => '0.0.0.0', 'HostPort' => '8080']]]], + ]); + $this->docker->fail('container', 'error during connect: Get "http://...": context deadline exceeded'); + + $problems = $this->check(); + $this->assertCount(1, $problems); + $this->assertStringContainsString('docker inspect failed: error during connect', $problems[0]); + } + + public function testContainerRemovedBetweenPsAndInspectHoldsNoPort(): void + { + $this->docker->setConfig($this->config(['image' => 'busybox', 'ports' => [ + ['target' => 80, 'published' => '8080', 'protocol' => 'tcp'], + ]])); + // docker inspect still prints the containers it found, and exits 1 for the one that is gone. + $this->docker->fail('container', 'Error response from daemon: No such container: id1', json_encode([ + ['Name' => '/nginx', 'Config' => ['Labels' => []], 'NetworkSettings' => ['Ports' => ['80/tcp' => [['HostIp' => '0.0.0.0', 'HostPort' => '8080']]]]], + ])); + $this->docker->setRunning([ + ['name' => 'nginx', 'project' => null, 'ports' => []], + ['name' => 'gone', 'project' => null, 'ports' => []], + ]); + + $problems = $this->check(); + $this->assertCount(1, $problems); + $this->assertStringContainsString("container 'nginx'", $problems[0]); + } + public function testPortPublishedByAnotherContainerIsAProblem(): void { $this->docker->setConfig($this->config(['image' => 'busybox', 'ports' => [ diff --git a/tests/unit/fixtures/fake-docker.sh b/tests/unit/fixtures/fake-docker.sh index 09459b66..278218bc 100644 --- a/tests/unit/fixtures/fake-docker.sh +++ b/tests/unit/fixtures/fake-docker.sh @@ -9,10 +9,18 @@ # a network file holds its driver, if any # containers/.json docker container inspect # running.ids, running.json docker ps -q, and docker container inspect +# fail/, fail/.stdout docker fails: the first file is its stderr, +# the second (optional) what it still printed # Every call is appended to calls.log, one line per call. here="$(cd "$(dirname "$0")" && pwd)" printf '%s\n' "$*" >> "$here/calls.log" +if [ -f "$here/fail/$1" ]; then + [ -f "$here/fail/$1.stdout" ] && cat "$here/fail/$1.stdout" + cat "$here/fail/$1" >&2 + exit 1 +fi + case "$1" in compose) for arg in "$@"; do diff --git a/tests/unit/support/FakeDocker.php b/tests/unit/support/FakeDocker.php index d2de8db8..529337a7 100644 --- a/tests/unit/support/FakeDocker.php +++ b/tests/unit/support/FakeDocker.php @@ -19,6 +19,7 @@ public function __construct() mkdir($this->dir . '/networks', 0755, true); mkdir($this->dir . '/volumes'); mkdir($this->dir . '/containers'); + mkdir($this->dir . '/fail'); copy(__DIR__ . '/../fixtures/fake-docker.sh', COMPOSE_DOCKER_BIN); chmod(COMPOSE_DOCKER_BIN, 0755); } @@ -28,6 +29,19 @@ public function cleanUp(): void self::removeTree($this->dir); } + /** + * Make every `docker ...` call fail with this stderr (exit 1), + * after printing $stdout, if any. The command is the first argument + * ("ps", "container"). + */ + public function fail(string $command, string $stderr, string $stdout = ''): void + { + file_put_contents($this->dir . '/fail/' . $command, $stderr . "\n"); + if ($stdout !== '') { + file_put_contents($this->dir . '/fail/' . $command . '.stdout', $stdout); + } + } + /** * What `docker compose config --format json` prints. * From b790367c9d9f0bd2f3f2147b4c5ebc0a8735d94a Mon Sep 17 00:00:00 2001 From: Grant Emsley Date: Wed, 7 Oct 2026 09:37:46 -0400 Subject: [PATCH 017/109] fix(git): catch two services of the stack publishing the same port The port check compared the stack's published ports only with running containers of other stacks. Two services of the stack itself asking for the same host port passed it, and "up" then started one service and failed on the other, after the containers had changed. The ports are now also compared with each other, with the same address and protocol rules as against other containers. Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_01NavH8cj9aU49Qs1pQPa1Qc --- docs/git-stacks.md | 4 ++- .../include/GitDeployCheck.php | 25 ++++++++++++++++--- tests/unit/GitDeployCheckTest.php | 23 +++++++++++++++++ 3 files changed, 48 insertions(+), 4 deletions(-) diff --git a/docs/git-stacks.md b/docs/git-stacks.md index bca8d9eb..648497aa 100644 --- a/docs/git-stacks.md +++ b/docs/git-stacks.md @@ -331,7 +331,9 @@ without changing anything when: - a bind-mount source is missing in a way described in [Folders and files](#folders-and-files-your-containers-need), or a config or secret file, or a local build folder, is missing; -- a `container_name` or published port is already taken by another stack or container. +- a `container_name` or published port is already taken by another stack or container, or two services of + the stack publish the same port; +- Docker could not be asked whether a name or port is free (the daemon is not answering, say). The deploy log says which `.env` was used. diff --git a/source/compose.manager/include/GitDeployCheck.php b/source/compose.manager/include/GitDeployCheck.php index 21d16f08..11db8f19 100644 --- a/source/compose.manager/include/GitDeployCheck.php +++ b/source/compose.manager/include/GitDeployCheck.php @@ -23,7 +23,8 @@ * exist, which catches a typo of /mnt itself. Missing folders below those * are left for Docker to create, as are paths inside the clone; * - config and secret files, and local build folders, that do not exist; - * - container names and published ports already taken by another container. + * - container names and published ports already taken by another container, + * and a port published by two services of the stack. * When Docker cannot answer a question (the daemon is not responding, a * command times out), that is a problem too: "up" must not be the one to find * out, after it has changed some of the containers. @@ -361,13 +362,31 @@ private function takenPorts(array $resolved): array return []; } + // Two services of this stack asking for the same port clash with each + // other, not with a running container: "up" would start one of them + // and fail on the other. + $problems = []; + foreach ($wanted as $i => $want) { + foreach (array_slice($wanted, $i + 1) as $other) { + if ($other['port'] !== $want['port'] || $other['protocol'] !== $want['protocol']) { + continue; + } + if (!self::addressesOverlap($want['ip'], $other['ip'])) { + continue; + } + $problems[] = $other['service'] === $want['service'] + ? "Service '{$want['service']}' publishes port {$want['port']}/{$want['protocol']} twice." + : "Services '{$want['service']}' and '{$other['service']}' both publish port {$want['port']}/{$want['protocol']}."; + } + } + try { $heldByOthers = $this->portsHeldByOtherContainers(); } catch (RuntimeException $error) { - return ['Could not check whether the published ports are free: ' . $error->getMessage()]; + $problems[] = 'Could not check whether the published ports are free: ' . $error->getMessage(); + return $problems; } - $problems = []; foreach ($wanted as $want) { foreach ($heldByOthers as $held) { if ($held['port'] !== $want['port'] || $held['protocol'] !== $want['protocol']) { diff --git a/tests/unit/GitDeployCheckTest.php b/tests/unit/GitDeployCheckTest.php index 1caab066..e5f8478d 100644 --- a/tests/unit/GitDeployCheckTest.php +++ b/tests/unit/GitDeployCheckTest.php @@ -381,6 +381,29 @@ public function testSamePortOnUdpOrADifferentAddressIsFine(): void $this->assertSame([], $this->check()); } + public function testTwoServicesOfTheStackPublishingTheSamePortIsAProblem(): void + { + $this->docker->setConfig(['name' => 'whoami', 'services' => [ + 'web' => ['image' => 'busybox', 'ports' => [['target' => 80, 'published' => '8080', 'protocol' => 'tcp']]], + 'api' => ['image' => 'busybox', 'ports' => [['target' => 80, 'published' => '8080', 'protocol' => 'tcp', 'host_ip' => '0.0.0.0']]], + 'dns' => ['image' => 'busybox', 'ports' => [['target' => 53, 'published' => '8080', 'protocol' => 'udp']]], + ]]); + + $problems = $this->check(); + $this->assertCount(1, $problems); + $this->assertSame("Services 'web' and 'api' both publish port 8080/tcp.", $problems[0]); + } + + public function testOneServicePublishingTheSamePortTwiceIsAProblem(): void + { + $this->docker->setConfig($this->config(['image' => 'busybox', 'ports' => [ + ['target' => 80, 'published' => '8080', 'protocol' => 'tcp'], + ['target' => 81, 'published' => '8080', 'protocol' => 'tcp', 'host_ip' => '127.0.0.1'], + ]])); + + $this->assertSame(["Service 'whoami' publishes port 8080/tcp twice."], $this->check()); + } + public function testPublishedRangeIsLeftToDocker(): void { // "6000-6010:6000" means any free port in the range. From 0f93286088a8c565f2ba84d72c5de4c6c41f22c9 Mon Sep 17 00:00:00 2001 From: Grant Emsley Date: Wed, 7 Oct 2026 09:42:17 -0400 Subject: [PATCH 018/109] fix(git): keep the deployed commit when reclone finds the branch rewritten reclone cleared the deployed commit when the rewritten branch no longer held it. The next deploy then had nothing to compare the new commit with, so it ran a plain "up", and a container whose bind-mounted config changed in the rewrite kept the old file. The deployed commit now stays recorded. The next deploy finds it gone from the clone and recreates every container, which GitDeploy::upArguments already does for a commit that cannot be measured against. Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_01NavH8cj9aU49Qs1pQPa1Qc --- source/compose.manager/include/GitStackManager.php | 8 +++++--- tests/unit/GitStackManagerTest.php | 4 +++- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/source/compose.manager/include/GitStackManager.php b/source/compose.manager/include/GitStackManager.php index 1d02979e..847ff566 100644 --- a/source/compose.manager/include/GitStackManager.php +++ b/source/compose.manager/include/GitStackManager.php @@ -335,10 +335,12 @@ public function reclone(string $folder): ?string $deployed = $state->deployedCommit; if ($deployed !== null && $deployed !== $head) { if (!$clone->hasCommit($deployed)) { - // The branch was rewritten and no longer holds it. + // The branch was rewritten and no longer holds it. The + // deployed commit stays recorded: the next deploy finds it + // gone from the clone and recreates every container, since + // what changed cannot be measured (see GitDeploy::upArguments). ($this->say)('The deployed commit ' . substr($deployed, 0, 12) . ' is not on the branch any more, so the clone was left at ' - . substr($head, 0, 12) . '. Deploy to bring the stack up to date.'); - (new GitStackState(null, $state->failedCommit))->save($stack->path); + . substr($head, 0, 12) . '. Deploy to bring the stack up to date; that deploy recreates every container.'); } else { try { $clone->checkOut($deployed); diff --git a/tests/unit/GitStackManagerTest.php b/tests/unit/GitStackManagerTest.php index 0ee9e6fd..ec913a65 100644 --- a/tests/unit/GitStackManagerTest.php +++ b/tests/unit/GitStackManagerTest.php @@ -349,11 +349,13 @@ public function testRecloneAfterTheDeployedCommitWasRewrittenAwayStaysAtTheNewHe $this->git(['--git-dir=' . $this->upstream, 'reflog', 'expire', '--expire=now', '--all'], null); $this->git(['--git-dir=' . $this->upstream, 'gc', '-q', '--prune=now'], null); + $rewrittenAway = GitStackState::load($stackDir)->deployedCommit; $this->manager->reclone($folder); $settings = GitStackSettings::load($stackDir); $this->assertSame($this->upstreamHead(), (new GitClone($settings))->checkedOutCommit()); - $this->assertNull(GitStackState::load($stackDir)->deployedCommit); + // Still recorded: the next deploy sees it is gone from the clone and recreates every container. + $this->assertSame($rewrittenAway, GitStackState::load($stackDir)->deployedCommit); } public function testConvertNamesTheFoldersNextToTheOldComposeFile(): void From 61b06dfbb0c9829a399a230b007654a0127006ea Mon Sep 17 00:00:00 2001 From: Grant Emsley Date: Wed, 7 Oct 2026 09:42:17 -0400 Subject: [PATCH 019/109] fix(git): notice a repository override that a checkout removed The profiles, services and build caches of a git stack were compared with the repository's override only while that file existed. A commit that removed the override, and changed nothing else, left the caches looking fresh, so the plugin kept answering from a file that was no longer a compose input. When the override is missing, its folder's modification time is compared instead: removing a file updates the folder. Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_01NavH8cj9aU49Qs1pQPa1Qc --- source/compose.manager/include/Util.php | 13 ++++++++++++- tests/unit/GitStackInfoTest.php | 24 ++++++++++++++++++++++++ 2 files changed, 36 insertions(+), 1 deletion(-) diff --git a/source/compose.manager/include/Util.php b/source/compose.manager/include/Util.php index 55058005..00f28528 100644 --- a/source/compose.manager/include/Util.php +++ b/source/compose.manager/include/Util.php @@ -3460,6 +3460,9 @@ public function getOverridePath(): ?string * getOverridePath() returns the managed override for a git stack, so the * cache staleness checks compare that one; the repository's own override * is a compose input too (see getComposeFilePaths). Always false for other stacks. + * + * A checkout that removes the override leaves no file to compare, so its + * folder's modification time stands in: removing a file updates it. */ private function isRepositoryOverrideNewerThan(int|false $cacheMtime): bool { @@ -3467,7 +3470,15 @@ private function isRepositoryOverrideNewerThan(int|false $cacheMtime): bool return false; } $repoOverride = $this->overrideInfo->indirectOverride; - return $repoOverride !== null && is_file($repoOverride) && filemtime($repoOverride) > $cacheMtime; + if ($repoOverride === null) { + return false; + } + if (is_file($repoOverride)) { + return filemtime($repoOverride) > $cacheMtime; + } + $folder = dirname($repoOverride); + clearstatcache(true, $folder); + return is_dir($folder) && filemtime($folder) > $cacheMtime; } /** diff --git a/tests/unit/GitStackInfoTest.php b/tests/unit/GitStackInfoTest.php index d456519d..5735caac 100644 --- a/tests/unit/GitStackInfoTest.php +++ b/tests/unit/GitStackInfoTest.php @@ -120,6 +120,30 @@ public function testAChangedRepositoryOverrideMakesTheCachesStale(): void $this->assertFalse($this->stack()->hasBuildConfig()); } + public function testARemovedRepositoryOverrideMakesTheCachesStale(): void + { + $repoOverride = $this->clone . '/whoami/compose.override.yaml'; + file_put_contents($repoOverride, "services: {}\n"); + file_put_contents($this->stackDir . '/profiles', json_encode(['cached'])); + file_put_contents($this->stackDir . '/has_build', '1'); + $old = time() - 3600; + touch($this->clone . '/whoami/compose.yaml', $old); + touch($repoOverride, $old); + touch($this->clone . '/whoami', $old); + touch($this->stackDir . '/profiles', time() - 60); + touch($this->stackDir . '/has_build', time() - 60); + + $this->assertSame(['cached'], $this->stack()->getProfiles()); + $this->assertTrue($this->stack()->hasBuildConfig()); + + // A commit that removes the repository's override leaves no file to compare, but + // the removal updates its folder. + unlink($repoOverride); + + $this->assertNotSame(['cached'], $this->stack()->getProfiles()); + $this->assertFalse($this->stack()->hasBuildConfig()); + } + public function testMissingOverridesAreLeftOut(): void { @unlink($this->stackDir . '/compose.override.yaml'); From 78c2ff5beeba743a95cf998a8beb3fbecb63fba7 Mon Sep 17 00:00:00 2001 From: Grant Emsley Date: Wed, 7 Oct 2026 09:42:31 -0400 Subject: [PATCH 020/109] docs(git): say what reclone leaves running on the old clone A container with a bind mount into the clone keeps the moved folder until it is recreated, and a rewritten branch leaves the new clone at its latest commit. Say both next to the reclone command. Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_01NavH8cj9aU49Qs1pQPa1Qc --- docs/git-stacks.md | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/docs/git-stacks.md b/docs/git-stacks.md index 648497aa..82c4cf7a 100644 --- a/docs/git-stacks.md +++ b/docs/git-stacks.md @@ -316,7 +316,10 @@ is, the deploy stops instead. If the clone itself gets into a bad state, `compose-git reclone myapp` moves it aside (as `.replaced-`, nothing is deleted) and clones the repository again at the deployed commit. It lists the untracked files left in the old clone: copy back anything you need, then delete the old clone -yourself. +yourself. A running container with a bind mount into the clone (`./config`, say) keeps using the moved +folder until the container is recreated, so stop and start the stack before deleting the old clone. If the +deployed commit is no longer on the branch (the branch was rewritten), the new clone is left at the +branch's latest commit, and the next deploy recreates every container. ## What is checked before a deploy From ff9cb5893cace298edeaf10e51ce23e308bd5120 Mon Sep 17 00:00:00 2001 From: mstrhakr Date: Thu, 8 Oct 2026 09:26:23 -0400 Subject: [PATCH 021/109] fix(compatibility): require Unraid 6.12 and remove legacy WebUI patches Align documented requirements with the plugin's PHP 8 dependency. Remove obsolete patch scripts, assets, installer hooks, settings and action handling, while retaining the pre-7.0 Ace fallback and Git patch features. Add installer compatibility regression coverage for the minimum version and Ace fallback. Fixes #170. --- README.md | 2 +- compose.manager.plg | 50 +- docs/configuration.md | 21 +- docs/getting-started.md | 2 +- docs/user-guide.md | 6 +- .../compose.manager.settings.page | 83 ---- source/compose.manager/default.cfg | 1 - source/compose.manager/include/Exec.php | 21 - .../patches/6.10/docker_client.patch | 119 ----- .../patches/6.11/docker_client.patch | 103 ---- .../patches/6.9/docker_client.patch | 128 ----- .../compose.manager/scripts/install-patch.sh | 54 -- source/compose.manager/scripts/patch.sh | 462 ------------------ source/pkg_build.sh | 2 +- tests/unit/ComposeFilterTest.php | 2 +- tests/unit/InstallerCompatibilityTest.php | 42 ++ tests/unit/patch.bats | 98 ---- 17 files changed, 49 insertions(+), 1147 deletions(-) delete mode 100644 source/compose.manager/patches/6.10/docker_client.patch delete mode 100644 source/compose.manager/patches/6.11/docker_client.patch delete mode 100644 source/compose.manager/patches/6.9/docker_client.patch delete mode 100644 source/compose.manager/scripts/install-patch.sh delete mode 100644 source/compose.manager/scripts/patch.sh create mode 100644 tests/unit/InstallerCompatibilityTest.php delete mode 100644 tests/unit/patch.bats diff --git a/README.md b/README.md index c69b6944..e4ffc63b 100644 --- a/README.md +++ b/README.md @@ -172,7 +172,7 @@ With dcflachs Docker Compose Manager being depreciated, if you want to go back t ## Requirements -- unRAID 6.9.0 or later +- unRAID 6.12.0 or later (PHP 8 is required; first included in Unraid 6.12.0) > NOTE: This is untested on anything older than 7.2.3 currently. I will remove this note if I get a report that this IS compatible that far back. I am working on getting a machine setup for testing on the older versions. Any advice would be welcome, eg how to VM unraid easily. diff --git a/compose.manager.plg b/compose.manager.plg index 07dfbfb7..78118113 100644 --- a/compose.manager.plg +++ b/compose.manager.plg @@ -29,7 +29,7 @@ launch="&launch;" pluginURL="&pluginURL;" icon="cubes" - min="6.9.0" + min="6.12.0" support="https://forums.unraid.net/topic/197334-plugin-compose-manager-plus/" project="https://github.com/mstrhakr/compose_plugin" readme="https://github.com/mstrhakr/compose_plugin#readme" @@ -157,30 +157,6 @@ rm -f /tmp/ace-editor-install.sh - - -echo "" -echo "----------------------------------------------------" -echo "" -echo " Patches are only installed for" -echo " pre-6.12 Unraid versions." -echo "" -echo " If on 6.12 or later, patches are not needed" -echo " and skips will be shown below." -echo "" -echo "----------------------------------------------------" -echo "" - - - - - - -&emhttpLOC;/scripts/install-patch.sh install &emhttpLOC; &pluginLOC; -rm -f /tmp/patch-util-install.sh - - - echo "" @@ -194,30 +170,6 @@ echo "" - - -echo "" -echo "----------------------------------------------------" -echo "" -echo " Patches are only modified for" -echo " pre-6.12 Unraid versions." -echo "" -echo " If on 6.12 or later, patches are not needed" -echo " and skips will be shown below." -echo "" -echo "----------------------------------------------------" -echo "" - - - - - - -&emhttpLOC;/scripts/install-patch.sh remove &emhttpLOC; &pluginLOC; -rm -f /tmp/patch-util-remove.sh - - - diff --git a/docs/configuration.md b/docs/configuration.md index 4a95a3fd..6101cacb 100755 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -23,7 +23,7 @@ Access settings via **Settings → Compose** in the Unraid web UI. | **Show in Header Menu** | No | Display Compose Manager as a separate page in the header navigation bar. | | **Show Dashboard Tile** | Yes | Display a Compose Stacks tile on the Dashboard showing stack status at a glance. | | **Hide Compose Containers (Dashboard Tile)** | No | Hide containers managed by Compose stacks from Unraid's Docker Containers dashboard tile. This avoids duplicate entries when both tiles are visible. Requires "Show Dashboard Tile". | -| **Hide Compose Containers (Docker Page)** | No | Patch the native Docker UI to hide or filter Compose-managed containers. See the Web UI Patches section for version limitations. | +| **Hide Compose Containers (Docker Page)** | No | Hide Compose-managed containers from the Docker Containers table to avoid duplicate entries when Compose stacks appear on the same page. Requires "Show in Header Menu" to be disabled (inline mode). | | **Show Compose Above Docker Containers** | No | When the Docker page is displayed without tabs, move the Compose Stacks section above the built-in Docker Containers section. | | **Expand Stacks by Default** | No | Automatically expand all stack detail rows when the page loads. | @@ -39,7 +39,6 @@ Access settings via **Settings → Compose** in the Unraid web UI. | Setting | Default | Description | |---------|---------|-------------| -| **Legacy Patch Web UI** | No | Enable integration patches for the native Docker manager UI on Unraid ≤ 6.11. Not required on Unraid 6.12 and later. Use the Patch/Unpatch buttons to apply or remove the patch. | | **Debug to Log** | No | Enable debug logging to syslog to troubleshoot Compose command output. | ### Backup Settings @@ -90,24 +89,6 @@ The default location stores all compose configurations on the USB flash drive, e Compose Manager supports all four standard Compose file names and preserves the filenames already present in each stack. -## Web UI Patches - -This setting controls small compatibility patches that change how Compose-managed containers are presented in Unraid's native Docker UI. These patches only affect UI rendering and metadata — they do **not** stop containers or change how they run. - -### Legacy UI Patch - -- **Legacy behavior (Unraid < 6.12)** — On older Unraid releases the patch adds Compose-specific metadata (e.g. `net.unraid.docker.managed`) and tweaks the Docker manager so Compose-managed containers are recognized. Practical effects include skipping native update checks for those containers and preventing them from being treated like native Docker-managed containers in the UI. Use this when running an older Unraid release so Compose can interoperate cleanly with the Docker manager. This is enabled in the settings under "Patch UI" - -### Hide Compose Containers from Docker UI via Patch - -- **Hide patches (Unraid 6.12–7.2)** — For Unraid 6.12 through 7.2 we offer a different patch that **removes** Compose-managed containers from the Docker page and Dashboard tile entirely (they remain visible through the Compose Stacks tile). Use the hide patch to remove duplicate entries and declutter the Docker UI when you prefer to manage Compose stacks separately. This is enabled in the settings under "Hide Compose Containers from Docker UI via Patch" - -Notes and version limitations: - -- Unraid 6.12 and later integrate the basic "ignore" behavior natively; the legacy patch is not required for that functionality. -- The explicit "hide" patches are provided only for **6.12–7.2**. If you are on a version outside this range, check compatibility before enabling the hide option. -- Patch files are stored under `source/compose.manager/patches/` and are applied only when a matching Unraid version is detected. - ## Debug Logging Enable debug logging to troubleshoot issues. Logs are written to the Unraid syslog. diff --git a/docs/getting-started.md b/docs/getting-started.md index b738824d..b2ca7f0d 100644 --- a/docs/getting-started.md +++ b/docs/getting-started.md @@ -2,7 +2,7 @@ ## Requirements -- Unraid 6.9.0 or later +- Unraid 6.12.0 or later (PHP 8 is required; first included in Unraid 6.12.0) - Docker service enabled ## Installation diff --git a/docs/user-guide.md b/docs/user-guide.md index a670c1b8..bfe690f6 100644 --- a/docs/user-guide.md +++ b/docs/user-guide.md @@ -81,7 +81,7 @@ Compose Manager provides a Backup & Restore interface under **Settings → Compo ### Hiding and Filtering Compose Containers -You can optionally hide compose-managed containers from the native Docker manager and Dashboard. Use the setting to toggle patching of the Docker page and enable server-side filters so only the desired containers/stacks are shown. +Use **Hide Compose Containers from Docker Page** to hide Compose-managed containers from the native Docker Containers table when **Show Compose in Header Menu** is disabled (inline mode). Use **Hide Compose Containers from Docker Dashboard Tile** to hide them from the native Dashboard tile when **Show Dashboard Tile** is enabled. These display options avoid duplicate entries without changing how containers run; no legacy WebUI patch is required. ### Display Options @@ -119,7 +119,3 @@ services: ``` The **Web UI** tab in the editor provides a visual interface for adding these labels. - -### Patching the Native UI - -Enable "Patch Web UI" in settings to show compose containers in the native Docker manager with stack grouping. diff --git a/source/compose.manager/compose.manager.settings.page b/source/compose.manager/compose.manager.settings.page index 30103498..f2033cf1 100755 --- a/source/compose.manager/compose.manager.settings.page +++ b/source/compose.manager/compose.manager.settings.page @@ -9,12 +9,6 @@ Lock="true" include "/usr/local/emhttp/plugins/dynamix.docker.manager/include/DockerClient.php"; require_once("/usr/local/emhttp/plugins/compose.manager/include/Defines.php"); $cfg = parse_plugin_cfg($sName); -$ui_patch_button_type = (!$option_patch_ui && strcmp($cfg['PATCH_UI'], "true") == 0 && !isset($composemanDockerClientPatch)) ? "Button" : "Hidden"; -$ui_unpatch_button_type = (!$option_patch_ui && strcmp($cfg['PATCH_UI'], "false") == 0 && isset($composemanDockerClientPatch)) ? "Button" : "Hidden"; - -$option_patch_ui = version_compare(parse_ini_file('/etc/unraid-version')['version'], '6.12.0-RC0', '>='); -$ui_patch_help_class = $option_patch_ui ? "hidden" : "inline_help"; // hide help on 6.12+ (integration built-in) // - $projects_exist = intval(shell_exec("ls -l " . $compose_root . " | grep ^d | wc -l")) != 0; @@ -658,7 +652,6 @@ $acePath = file_exists('/usr/local/emhttp/plugins/dynamix/javascript/ace/ace.js'