From a70abac38fce16e60f4578bc211c26ebc107f223 Mon Sep 17 00:00:00 2001 From: trent Date: Mon, 3 Aug 2026 21:28:15 +0000 Subject: [PATCH 1/7] fall back to network checkout when the mirror sticky disk stalls Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- dist/index.js | 214 +++++++++++++++++++++++++++++++++++-- src/blacksmith-cache.ts | 131 +++++++++++++++++++++++ src/git-command-manager.ts | 74 ++++++++++++- src/git-source-provider.ts | 74 +++++++++++-- 4 files changed, 474 insertions(+), 19 deletions(-) diff --git a/dist/index.js b/dist/index.js index 5b2beff..9a38cb2 100644 --- a/dist/index.js +++ b/dist/index.js @@ -39,6 +39,7 @@ var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, ge }); }; Object.defineProperty(exports, "__esModule", ({ value: true })); +exports.MIRROR_STALL_TIMEOUT_SECS = void 0; exports.getMountPoint = getMountPoint; exports.isBlacksmithEnvironment = isBlacksmithEnvironment; exports.getAgentAddr = getAgentAddr; @@ -49,6 +50,7 @@ exports.getMirrorPath = getMirrorPath; exports.setupCache = setupCache; exports.ensureMirror = ensureMirror; exports.refreshMirror = refreshMirror; +exports.removeAlternates = removeAlternates; exports.writeAlternates = writeAlternates; exports.dissociate = dissociate; exports.cleanup = cleanup; @@ -75,6 +77,18 @@ const UMOUNT_INITIAL_DELAY_MS = 1000; // Initial delay between retries (1 second const UMOUNT_BACKOFF_MULTIPLIER = 2; // Exponential backoff multiplier // Exit code returned by the `timeout` command when the child is killed. const TIMEOUT_EXIT_CODE = 124; +// Read-health probe of the mirror sticky disk, run right after mount. A +// degraded Ceph-backed disk can serve reads at pathological latencies +// (observed: hundreds of seconds of read wait during a 5-minute step), which +// stalls fetch/checkout until the customer's step timeout kills the job. +// If the probe cannot read a few MB quickly, skip the mirror entirely. +const READ_PROBE_TIMEOUT_SECS = 10; +const READ_PROBE_MB = 8; +// Deadline applied to mirror-assisted `git fetch`/`git checkout` in the main +// step. If either stalls past this (e.g. alternates reads hitting a degraded +// sticky disk), the checkout falls back to a network-only fetch/checkout +// instead of failing the job. +exports.MIRROR_STALL_TIMEOUT_SECS = 120; /** * Get the mount point for a specific repository. * Each repository gets its own mount point to support multiple checkouts. @@ -311,6 +325,22 @@ function setupCache(owner, repo) { // the device (uninitialized inode tables), which is unnecessary here. yield exec.exec('sudo', ['mount', '-o', 'noinit_itable', device, mountPoint]); core.info(`[git-mirror] Mounted ${device} at ${mountPoint}`); + if (!(yield probeDeviceReadHealth(device))) { + core.warning(`[git-mirror] Sticky disk ${device} failed the read-health probe; unmounting and falling back to network-only checkout`); + yield exec.getExecOutput('timeout', [String(UMOUNT_TIMEOUT_SECS), 'sudo', 'umount', mountPoint], { ignoreReturnCode: true }); + yield releaseStickyDisk(exposeId, stickyDiskKey, repoName); + return { + exposeId: '', + stickyDiskKey, + repoName, + device: '', + mountPoint: '', + mirrorPath: '', + hydrationInProgress: false, + performedHydration: false, + readProbeFailed: true + }; + } return { exposeId, stickyDiskKey, @@ -323,6 +353,62 @@ function setupCache(owner, repo) { }; }); } +/** + * Check that the sticky disk can serve reads at a sane latency by reading a + * few MB directly from the block device (O_DIRECT, bypassing the page cache). + * Returns false if the read fails or takes longer than the probe timeout. + */ +function probeDeviceReadHealth(device) { + return __awaiter(this, void 0, void 0, function* () { + const start = Date.now(); + const result = yield exec.getExecOutput('timeout', [ + String(READ_PROBE_TIMEOUT_SECS), + 'sudo', + 'dd', + `if=${device}`, + 'of=/dev/null', + 'bs=1M', + `count=${READ_PROBE_MB}`, + 'iflag=direct' + ], { ignoreReturnCode: true, silent: true }); + const durationMs = Date.now() - start; + if (result.exitCode === TIMEOUT_EXIT_CODE) { + core.warning(`[git-mirror] Read-health probe timed out after ${READ_PROBE_TIMEOUT_SECS}s reading ${READ_PROBE_MB}MB from ${device}`); + return false; + } + if (result.exitCode !== 0) { + core.warning(`[git-mirror] Read-health probe failed with exit code ${result.exitCode}: ${result.stderr.trim()}`); + return false; + } + core.info(`[git-mirror] Read-health probe: read ${READ_PROBE_MB}MB from ${device} in ${durationMs}ms`); + return true; + }); +} +/** + * Release a sticky disk without persisting any changes. Used when the disk + * is unusable (e.g. failed the read-health probe) so the backend can detach + * it instead of leaving the exposure dangling. + */ +function releaseStickyDisk(exposeId, stickyDiskKey, repoName) { + return __awaiter(this, void 0, void 0, function* () { + try { + const client = createBlacksmithClient(); + yield client.commitStickyDisk({ + exposeId, + stickyDiskKey, + vmId: process.env.BLACKSMITH_VM_ID || '', + shouldCommit: false, + repoName, + stickyDiskToken: process.env.BLACKSMITH_STICKYDISK_TOKEN || '', + vmHydratedGitMirror: false + }); + core.info('[git-mirror] Released sticky disk without committing'); + } + catch (error) { + core.warning(`[git-mirror] Failed to release sticky disk: ${error.message}`); + } + }); +} /** * Get the extraheader config value for git authentication. * Uses the same format as upstream actions/checkout: @@ -491,6 +577,23 @@ function refreshMirror(mirrorPath_1, repoUrl_1, authToken_1) { * This allows the workspace git repo to use objects from the mirror * without copying them */ +/** + * Remove the alternates file so the workspace repo no longer borrows objects + * from the mirror. Any objects that were only reachable via the mirror must + * be re-fetched from the network afterwards. + */ +function removeAlternates(workspacePath) { + return __awaiter(this, void 0, void 0, function* () { + const alternatesFile = path.join(workspacePath, '.git', 'objects', 'info', 'alternates'); + try { + yield fs.promises.unlink(alternatesFile); + core.info('[git-mirror] Removed alternates file'); + } + catch (_a) { + // File may not exist, that's fine + } + }); +} function writeAlternates(workspacePath, mirrorPath) { return __awaiter(this, void 0, void 0, function* () { const alternatesDir = path.join(workspacePath, '.git', 'objects', 'info'); @@ -1525,7 +1628,7 @@ var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, ge }); }; Object.defineProperty(exports, "__esModule", ({ value: true })); -exports.MinimumGitSparseCheckoutVersion = exports.MinimumGitVersion = void 0; +exports.GitStallTimeoutError = exports.MinimumGitSparseCheckoutVersion = exports.MinimumGitVersion = void 0; exports.createCommandManager = createCommandManager; const core = __importStar(__nccwpck_require__(2186)); const exec = __importStar(__nccwpck_require__(1514)); @@ -1542,6 +1645,22 @@ const git_version_1 = __nccwpck_require__(3142); // sparse-checkout not [well-]supported before 2.28 (see https://github.com/actions/checkout/issues/1386) exports.MinimumGitVersion = new git_version_1.GitVersion('2.18'); exports.MinimumGitSparseCheckoutVersion = new git_version_1.GitVersion('2.28'); +// Exit code returned by the `timeout` command when the child is killed. +const TIMEOUT_EXIT_CODE = 124; +// Grace period before `timeout` escalates from SIGTERM to SIGKILL. +const STALL_KILL_AFTER_SECS = 10; +/** + * Thrown when a git command wrapped with a stall timeout is killed because it + * exceeded the deadline. Callers use this to distinguish a stalled command + * (e.g. mirror sticky-disk reads hanging) from a genuine git failure. + */ +class GitStallTimeoutError extends Error { + constructor(command, timeoutSecs) { + super(`git ${command} stalled and was killed after ${timeoutSecs}s`); + this.name = 'GitStallTimeoutError'; + } +} +exports.GitStallTimeoutError = GitStallTimeoutError; function createCommandManager(workingDirectory, lfs, doSparseCheckout) { return __awaiter(this, void 0, void 0, function* () { return yield GitCommandManager.createCommandManager(workingDirectory, lfs, doSparseCheckout); @@ -1661,12 +1780,16 @@ class GitCommandManager { yield fs.promises.appendFile(sparseCheckoutPath, `\n${sparseCheckout.join('\n')}\n`); }); } - checkout(ref, startPoint) { + checkout(ref, startPoint, stallTimeoutSecs) { return __awaiter(this, void 0, void 0, function* () { const args = ['checkout', '--progress', '--force']; if (startPoint) { args.push('-B', ref, startPoint); } + if (stallTimeoutSecs) { + yield this.execGit(args, false, false, {}, stallTimeoutSecs); + return; + } else { args.push(ref); } @@ -1731,6 +1854,12 @@ class GitCommandManager { for (const arg of refSpec) { args.push(arg); } + if (options.stallTimeoutSecs) { + // Single attempt: on a stall the caller falls back to a network-only + // fetch, so retrying with the mirror still attached only burns time. + yield this.execGit(args, false, false, {}, options.stallTimeoutSecs); + return; + } const that = this; yield retryHelper.execute(() => __awaiter(this, void 0, void 0, function* () { yield that.execGit(args); @@ -1996,7 +2125,7 @@ class GitCommandManager { }); } execGit(args_1) { - return __awaiter(this, arguments, void 0, function* (args, allowAllExitCodes = false, silent = false, customListeners = {}) { + return __awaiter(this, arguments, void 0, function* (args, allowAllExitCodes = false, silent = false, customListeners = {}, stallTimeoutSecs) { fshelper.directoryExistsSync(this.workingDirectory, true); const result = new GitOutput(); const env = {}; @@ -2020,6 +2149,29 @@ class GitCommandManager { ignoreReturnCode: allowAllExitCodes, listeners: mergedListeners }; + if (stallTimeoutSecs) { + // Run git under `timeout` so a command stalled on unresponsive storage + // (e.g. a degraded mirror sticky disk) is killed instead of running + // until the job-level timeout. Only used on Blacksmith Linux runners. + const timeoutArgs = [ + '-k', + String(STALL_KILL_AFTER_SECS), + String(stallTimeoutSecs), + this.gitPath, + ...args + ]; + result.exitCode = yield exec.exec('timeout', timeoutArgs, Object.assign(Object.assign({}, options), { ignoreReturnCode: true })); + result.stdout = stdout.join(''); + if (result.exitCode === TIMEOUT_EXIT_CODE) { + throw new GitStallTimeoutError(args.join(' '), stallTimeoutSecs); + } + if (result.exitCode !== 0 && !allowAllExitCodes) { + throw new Error(`The process '${this.gitPath}' failed with exit code ${result.exitCode}`); + } + core.debug(result.exitCode.toString()); + core.debug(result.stdout); + return result; + } result.exitCode = yield exec.exec(`"${this.gitPath}"`, args, options); result.stdout = stdout.join(''); core.debug(result.exitCode.toString()); @@ -2366,8 +2518,10 @@ function getSource(settings) { try { core.startGroup('Setting up Blacksmith git mirror cache'); cacheInfo = yield blacksmithCache.setupCache(settings.repositoryOwner, settings.repositoryName); - // Check if hydration is in progress - another job is doing the initial git clone --mirror - if (cacheInfo.hydrationInProgress) { + // Fall back to standard checkout if hydration is in progress (another + // job is doing the initial git clone --mirror) or the disk failed the + // read-health probe (already unmounted and released) + if (cacheInfo.hydrationInProgress || cacheInfo.readProbeFailed) { // Warning already logged by setupCache, just fall back to standard checkout cacheInfo = null; core.endGroup(); @@ -2435,6 +2589,15 @@ function getSource(settings) { if (settings.lfs) { yield git.lfsInstall(); } + // When the mirror is attached via alternates, a degraded sticky disk can + // stall fetch/checkout on object reads until the customer's step timeout + // kills the job. Cap mirror-assisted fetch/checkout and, on a stall, + // detach the mirror and fall back to the network instead of failing. + const disableMirrorAfterStall = (operation) => __awaiter(this, void 0, void 0, function* () { + core.warning(`[git-mirror] ${operation} stalled after ${blacksmithCache.MIRROR_STALL_TIMEOUT_SECS}s with the mirror attached; detaching mirror and falling back to network-only checkout`); + yield blacksmithCache.removeAlternates(settings.repositoryPath); + cacheInfo = null; + }); // Fetch core.startGroup('Fetching the repository'); const fetchOptions = {}; @@ -2444,22 +2607,38 @@ function getSource(settings) { else if (settings.sparseCheckout) { fetchOptions.filter = 'blob:none'; } + const fetchWithMirrorFallback = (refSpec) => __awaiter(this, void 0, void 0, function* () { + if (!cacheInfo) { + yield git.fetch(refSpec, fetchOptions); + return; + } + try { + yield git.fetch(refSpec, Object.assign(Object.assign({}, fetchOptions), { stallTimeoutSecs: blacksmithCache.MIRROR_STALL_TIMEOUT_SECS })); + } + catch (error) { + if (!(error instanceof git_command_manager_1.GitStallTimeoutError)) { + throw error; + } + yield disableMirrorAfterStall('git fetch'); + yield git.fetch(refSpec, fetchOptions); + } + }); if (settings.fetchDepth <= 0) { // Fetch all branches and tags let refSpec = refHelper.getRefSpecForAllHistory(settings.ref, settings.commit); - yield git.fetch(refSpec, fetchOptions); + yield fetchWithMirrorFallback(refSpec); // When all history is fetched, the ref we're interested in may have moved to a different // commit (push or force push). If so, fetch again with a targeted refspec. if (!(yield refHelper.testRef(git, settings.ref, settings.commit))) { refSpec = refHelper.getRefSpec(settings.ref, settings.commit); - yield git.fetch(refSpec, fetchOptions); + yield fetchWithMirrorFallback(refSpec); } } else { fetchOptions.fetchDepth = settings.fetchDepth; fetchOptions.fetchTags = settings.fetchTags; const refSpec = refHelper.getRefSpec(settings.ref, settings.commit); - yield git.fetch(refSpec, fetchOptions); + yield fetchWithMirrorFallback(refSpec); } core.endGroup(); // Checkout info @@ -2495,7 +2674,24 @@ function getSource(settings) { } // Checkout core.startGroup('Checking out the ref'); - yield git.checkout(checkoutInfo.ref, checkoutInfo.startPoint); + if (cacheInfo) { + try { + yield git.checkout(checkoutInfo.ref, checkoutInfo.startPoint, blacksmithCache.MIRROR_STALL_TIMEOUT_SECS); + } + catch (error) { + if (!(error instanceof git_command_manager_1.GitStallTimeoutError)) { + throw error; + } + yield disableMirrorAfterStall('git checkout'); + // Objects previously borrowed from the mirror are gone with the + // alternates file; re-fetch so they exist locally, then retry. + yield git.fetch(refHelper.getRefSpec(settings.ref, settings.commit), fetchOptions); + yield git.checkout(checkoutInfo.ref, checkoutInfo.startPoint); + } + } + else { + yield git.checkout(checkoutInfo.ref, checkoutInfo.startPoint); + } core.endGroup(); // Dissociate from Blacksmith mirror if requested // This copies all objects from alternates into the local repo so it's independent diff --git a/src/blacksmith-cache.ts b/src/blacksmith-cache.ts index 2d4296d..799d522 100644 --- a/src/blacksmith-cache.ts +++ b/src/blacksmith-cache.ts @@ -25,6 +25,20 @@ const UMOUNT_BACKOFF_MULTIPLIER = 2 // Exponential backoff multiplier // Exit code returned by the `timeout` command when the child is killed. const TIMEOUT_EXIT_CODE = 124 +// Read-health probe of the mirror sticky disk, run right after mount. A +// degraded Ceph-backed disk can serve reads at pathological latencies +// (observed: hundreds of seconds of read wait during a 5-minute step), which +// stalls fetch/checkout until the customer's step timeout kills the job. +// If the probe cannot read a few MB quickly, skip the mirror entirely. +const READ_PROBE_TIMEOUT_SECS = 10 +const READ_PROBE_MB = 8 + +// Deadline applied to mirror-assisted `git fetch`/`git checkout` in the main +// step. If either stalls past this (e.g. alternates reads hitting a degraded +// sticky disk), the checkout falls back to a network-only fetch/checkout +// instead of failing the job. +export const MIRROR_STALL_TIMEOUT_SECS = 120 + /** * Result of a git mirror operation that may fail or time out. */ @@ -65,6 +79,10 @@ export interface CacheInfo { // performedHydration indicates that this job performed the initial git mirror clone. // Used to notify the backend on commit so it can mark hydration as complete. performedHydration: boolean + // readProbeFailed indicates the mounted sticky disk failed the read-health + // probe. The disk has already been unmounted and released; the caller + // should fall back to regular checkout without using the cache. + readProbeFailed?: boolean } /** @@ -343,6 +361,29 @@ export async function setupCache( await exec.exec('sudo', ['mount', '-o', 'noinit_itable', device, mountPoint]) core.info(`[git-mirror] Mounted ${device} at ${mountPoint}`) + if (!(await probeDeviceReadHealth(device))) { + core.warning( + `[git-mirror] Sticky disk ${device} failed the read-health probe; unmounting and falling back to network-only checkout` + ) + await exec.getExecOutput( + 'timeout', + [String(UMOUNT_TIMEOUT_SECS), 'sudo', 'umount', mountPoint], + {ignoreReturnCode: true} + ) + await releaseStickyDisk(exposeId, stickyDiskKey, repoName) + return { + exposeId: '', + stickyDiskKey, + repoName, + device: '', + mountPoint: '', + mirrorPath: '', + hydrationInProgress: false, + performedHydration: false, + readProbeFailed: true + } + } + return { exposeId, stickyDiskKey, @@ -355,6 +396,75 @@ export async function setupCache( } } +/** + * Check that the sticky disk can serve reads at a sane latency by reading a + * few MB directly from the block device (O_DIRECT, bypassing the page cache). + * Returns false if the read fails or takes longer than the probe timeout. + */ +async function probeDeviceReadHealth(device: string): Promise { + const start = Date.now() + const result = await exec.getExecOutput( + 'timeout', + [ + String(READ_PROBE_TIMEOUT_SECS), + 'sudo', + 'dd', + `if=${device}`, + 'of=/dev/null', + 'bs=1M', + `count=${READ_PROBE_MB}`, + 'iflag=direct' + ], + {ignoreReturnCode: true, silent: true} + ) + const durationMs = Date.now() - start + if (result.exitCode === TIMEOUT_EXIT_CODE) { + core.warning( + `[git-mirror] Read-health probe timed out after ${READ_PROBE_TIMEOUT_SECS}s reading ${READ_PROBE_MB}MB from ${device}` + ) + return false + } + if (result.exitCode !== 0) { + core.warning( + `[git-mirror] Read-health probe failed with exit code ${result.exitCode}: ${result.stderr.trim()}` + ) + return false + } + core.info( + `[git-mirror] Read-health probe: read ${READ_PROBE_MB}MB from ${device} in ${durationMs}ms` + ) + return true +} + +/** + * Release a sticky disk without persisting any changes. Used when the disk + * is unusable (e.g. failed the read-health probe) so the backend can detach + * it instead of leaving the exposure dangling. + */ +async function releaseStickyDisk( + exposeId: string, + stickyDiskKey: string, + repoName: string +): Promise { + try { + const client = createBlacksmithClient() + await client.commitStickyDisk({ + exposeId, + stickyDiskKey, + vmId: process.env.BLACKSMITH_VM_ID || '', + shouldCommit: false, + repoName, + stickyDiskToken: process.env.BLACKSMITH_STICKYDISK_TOKEN || '', + vmHydratedGitMirror: false + }) + core.info('[git-mirror] Released sticky disk without committing') + } catch (error) { + core.warning( + `[git-mirror] Failed to release sticky disk: ${(error as Error).message}` + ) + } +} + /** * Get the extraheader config value for git authentication. * Uses the same format as upstream actions/checkout: @@ -563,6 +673,27 @@ export async function refreshMirror( * This allows the workspace git repo to use objects from the mirror * without copying them */ +/** + * Remove the alternates file so the workspace repo no longer borrows objects + * from the mirror. Any objects that were only reachable via the mirror must + * be re-fetched from the network afterwards. + */ +export async function removeAlternates(workspacePath: string): Promise { + const alternatesFile = path.join( + workspacePath, + '.git', + 'objects', + 'info', + 'alternates' + ) + try { + await fs.promises.unlink(alternatesFile) + core.info('[git-mirror] Removed alternates file') + } catch { + // File may not exist, that's fine + } +} + export async function writeAlternates( workspacePath: string, mirrorPath: string diff --git a/src/git-command-manager.ts b/src/git-command-manager.ts index a45e15a..cd598f6 100644 --- a/src/git-command-manager.ts +++ b/src/git-command-manager.ts @@ -15,6 +15,23 @@ import {GitVersion} from './git-version' export const MinimumGitVersion = new GitVersion('2.18') export const MinimumGitSparseCheckoutVersion = new GitVersion('2.28') +// Exit code returned by the `timeout` command when the child is killed. +const TIMEOUT_EXIT_CODE = 124 +// Grace period before `timeout` escalates from SIGTERM to SIGKILL. +const STALL_KILL_AFTER_SECS = 10 + +/** + * Thrown when a git command wrapped with a stall timeout is killed because it + * exceeded the deadline. Callers use this to distinguish a stalled command + * (e.g. mirror sticky-disk reads hanging) from a genuine git failure. + */ +export class GitStallTimeoutError extends Error { + constructor(command: string, timeoutSecs: number) { + super(`git ${command} stalled and was killed after ${timeoutSecs}s`) + this.name = 'GitStallTimeoutError' + } +} + export interface IGitCommandManager { branchDelete(remote: boolean, branch: string): Promise branchExists(remote: boolean, pattern: string): Promise @@ -22,7 +39,11 @@ export interface IGitCommandManager { disableSparseCheckout(): Promise sparseCheckout(sparseCheckout: string[]): Promise sparseCheckoutNonConeMode(sparseCheckout: string[]): Promise - checkout(ref: string, startPoint: string): Promise + checkout( + ref: string, + startPoint: string, + stallTimeoutSecs?: number + ): Promise checkoutDetach(): Promise config( configKey: string, @@ -39,6 +60,7 @@ export interface IGitCommandManager { fetchDepth?: number fetchTags?: boolean showProgress?: boolean + stallTimeoutSecs?: number } ): Promise getDefaultBranch(repositoryUrl: string): Promise @@ -221,10 +243,18 @@ class GitCommandManager { ) } - async checkout(ref: string, startPoint: string): Promise { + async checkout( + ref: string, + startPoint: string, + stallTimeoutSecs?: number + ): Promise { const args = ['checkout', '--progress', '--force'] if (startPoint) { args.push('-B', ref, startPoint) + } + if (stallTimeoutSecs) { + await this.execGit(args, false, false, {}, stallTimeoutSecs) + return } else { args.push(ref) } @@ -282,6 +312,7 @@ class GitCommandManager { fetchDepth?: number fetchTags?: boolean showProgress?: boolean + stallTimeoutSecs?: number } ): Promise { const args = ['-c', 'protocol.version=2', 'fetch'] @@ -313,6 +344,13 @@ class GitCommandManager { args.push(arg) } + if (options.stallTimeoutSecs) { + // Single attempt: on a stall the caller falls back to a network-only + // fetch, so retrying with the mirror still attached only burns time. + await this.execGit(args, false, false, {}, options.stallTimeoutSecs) + return + } + const that = this await retryHelper.execute(async () => { await that.execGit(args) @@ -615,7 +653,8 @@ class GitCommandManager { args: string[], allowAllExitCodes = false, silent = false, - customListeners = {} + customListeners = {}, + stallTimeoutSecs?: number ): Promise { fshelper.directoryExistsSync(this.workingDirectory, true) @@ -646,6 +685,35 @@ class GitCommandManager { listeners: mergedListeners } + if (stallTimeoutSecs) { + // Run git under `timeout` so a command stalled on unresponsive storage + // (e.g. a degraded mirror sticky disk) is killed instead of running + // until the job-level timeout. Only used on Blacksmith Linux runners. + const timeoutArgs = [ + '-k', + String(STALL_KILL_AFTER_SECS), + String(stallTimeoutSecs), + this.gitPath, + ...args + ] + result.exitCode = await exec.exec('timeout', timeoutArgs, { + ...options, + ignoreReturnCode: true + }) + result.stdout = stdout.join('') + if (result.exitCode === TIMEOUT_EXIT_CODE) { + throw new GitStallTimeoutError(args.join(' '), stallTimeoutSecs) + } + if (result.exitCode !== 0 && !allowAllExitCodes) { + throw new Error( + `The process '${this.gitPath}' failed with exit code ${result.exitCode}` + ) + } + core.debug(result.exitCode.toString()) + core.debug(result.stdout) + return result + } + result.exitCode = await exec.exec(`"${this.gitPath}"`, args, options) result.stdout = stdout.join('') diff --git a/src/git-source-provider.ts b/src/git-source-provider.ts index d7b1422..360c0a7 100644 --- a/src/git-source-provider.ts +++ b/src/git-source-provider.ts @@ -12,7 +12,8 @@ import * as urlHelper from './url-helper' import * as blacksmithCache from './blacksmith-cache' import { MinimumGitSparseCheckoutVersion, - IGitCommandManager + IGitCommandManager, + GitStallTimeoutError } from './git-command-manager' import {IGitSourceSettings} from './git-source-settings' @@ -120,8 +121,10 @@ export async function getSource(settings: IGitSourceSettings): Promise { settings.repositoryName ) - // Check if hydration is in progress - another job is doing the initial git clone --mirror - if (cacheInfo.hydrationInProgress) { + // Fall back to standard checkout if hydration is in progress (another + // job is doing the initial git clone --mirror) or the disk failed the + // read-health probe (already unmounted and released) + if (cacheInfo.hydrationInProgress || cacheInfo.readProbeFailed) { // Warning already logged by setupCache, just fall back to standard checkout cacheInfo = null core.endGroup() @@ -212,6 +215,20 @@ export async function getSource(settings: IGitSourceSettings): Promise { await git.lfsInstall() } + // When the mirror is attached via alternates, a degraded sticky disk can + // stall fetch/checkout on object reads until the customer's step timeout + // kills the job. Cap mirror-assisted fetch/checkout and, on a stall, + // detach the mirror and fall back to the network instead of failing. + const disableMirrorAfterStall = async ( + operation: string + ): Promise => { + core.warning( + `[git-mirror] ${operation} stalled after ${blacksmithCache.MIRROR_STALL_TIMEOUT_SECS}s with the mirror attached; detaching mirror and falling back to network-only checkout` + ) + await blacksmithCache.removeAlternates(settings.repositoryPath) + cacheInfo = null + } + // Fetch core.startGroup('Fetching the repository') const fetchOptions: { @@ -227,25 +244,46 @@ export async function getSource(settings: IGitSourceSettings): Promise { fetchOptions.filter = 'blob:none' } + const fetchWithMirrorFallback = async ( + refSpec: string[] + ): Promise => { + if (!cacheInfo) { + await git.fetch(refSpec, fetchOptions) + return + } + try { + await git.fetch(refSpec, { + ...fetchOptions, + stallTimeoutSecs: blacksmithCache.MIRROR_STALL_TIMEOUT_SECS + }) + } catch (error) { + if (!(error instanceof GitStallTimeoutError)) { + throw error + } + await disableMirrorAfterStall('git fetch') + await git.fetch(refSpec, fetchOptions) + } + } + if (settings.fetchDepth <= 0) { // Fetch all branches and tags let refSpec = refHelper.getRefSpecForAllHistory( settings.ref, settings.commit ) - await git.fetch(refSpec, fetchOptions) + await fetchWithMirrorFallback(refSpec) // When all history is fetched, the ref we're interested in may have moved to a different // commit (push or force push). If so, fetch again with a targeted refspec. if (!(await refHelper.testRef(git, settings.ref, settings.commit))) { refSpec = refHelper.getRefSpec(settings.ref, settings.commit) - await git.fetch(refSpec, fetchOptions) + await fetchWithMirrorFallback(refSpec) } } else { fetchOptions.fetchDepth = settings.fetchDepth fetchOptions.fetchTags = settings.fetchTags const refSpec = refHelper.getRefSpec(settings.ref, settings.commit) - await git.fetch(refSpec, fetchOptions) + await fetchWithMirrorFallback(refSpec) } core.endGroup() @@ -287,7 +325,29 @@ export async function getSource(settings: IGitSourceSettings): Promise { // Checkout core.startGroup('Checking out the ref') - await git.checkout(checkoutInfo.ref, checkoutInfo.startPoint) + if (cacheInfo) { + try { + await git.checkout( + checkoutInfo.ref, + checkoutInfo.startPoint, + blacksmithCache.MIRROR_STALL_TIMEOUT_SECS + ) + } catch (error) { + if (!(error instanceof GitStallTimeoutError)) { + throw error + } + await disableMirrorAfterStall('git checkout') + // Objects previously borrowed from the mirror are gone with the + // alternates file; re-fetch so they exist locally, then retry. + await git.fetch( + refHelper.getRefSpec(settings.ref, settings.commit), + fetchOptions + ) + await git.checkout(checkoutInfo.ref, checkoutInfo.startPoint) + } + } else { + await git.checkout(checkoutInfo.ref, checkoutInfo.startPoint) + } core.endGroup() // Dissociate from Blacksmith mirror if requested From 09888c774b546932614ad23e3a471e3d78c4d388 Mon Sep 17 00:00:00 2001 From: trent Date: Mon, 3 Aug 2026 21:30:13 +0000 Subject: [PATCH 2/7] fix checkout ref args when startPoint is set Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- dist/index.js | 6 +++--- src/git-command-manager.ts | 5 +++-- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/dist/index.js b/dist/index.js index 9a38cb2..ea05283 100644 --- a/dist/index.js +++ b/dist/index.js @@ -1786,13 +1786,13 @@ class GitCommandManager { if (startPoint) { args.push('-B', ref, startPoint); } + else { + args.push(ref); + } if (stallTimeoutSecs) { yield this.execGit(args, false, false, {}, stallTimeoutSecs); return; } - else { - args.push(ref); - } yield this.execGit(args); }); } diff --git a/src/git-command-manager.ts b/src/git-command-manager.ts index cd598f6..0bbefc6 100644 --- a/src/git-command-manager.ts +++ b/src/git-command-manager.ts @@ -251,12 +251,13 @@ class GitCommandManager { const args = ['checkout', '--progress', '--force'] if (startPoint) { args.push('-B', ref, startPoint) + } else { + args.push(ref) } + if (stallTimeoutSecs) { await this.execGit(args, false, false, {}, stallTimeoutSecs) return - } else { - args.push(ref) } await this.execGit(args) From 3a7d2767f545acda9acde3c3b4fe3f9a11572b8c Mon Sep 17 00:00:00 2001 From: trent Date: Mon, 3 Aug 2026 21:40:06 +0000 Subject: [PATCH 3/7] address review: probe SIGKILL escalation, stale lock cleanup, --refetch on fallback Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- dist/index.js | 44 ++++++++++++++++++++++++++++++++++++-- src/blacksmith-cache.ts | 24 +++++++++++++++++++++ src/git-command-manager.ts | 8 +++++++ src/git-source-provider.ts | 23 +++++++++++++++----- 4 files changed, 92 insertions(+), 7 deletions(-) diff --git a/dist/index.js b/dist/index.js index ea05283..80c199a 100644 --- a/dist/index.js +++ b/dist/index.js @@ -51,6 +51,7 @@ exports.setupCache = setupCache; exports.ensureMirror = ensureMirror; exports.refreshMirror = refreshMirror; exports.removeAlternates = removeAlternates; +exports.removeStaleGitLocks = removeStaleGitLocks; exports.writeAlternates = writeAlternates; exports.dissociate = dissociate; exports.cleanup = cleanup; @@ -83,6 +84,7 @@ const TIMEOUT_EXIT_CODE = 124; // stalls fetch/checkout until the customer's step timeout kills the job. // If the probe cannot read a few MB quickly, skip the mirror entirely. const READ_PROBE_TIMEOUT_SECS = 10; +const READ_PROBE_KILL_AFTER_SECS = 5; const READ_PROBE_MB = 8; // Deadline applied to mirror-assisted `git fetch`/`git checkout` in the main // step. If either stalls past this (e.g. alternates reads hitting a degraded @@ -362,6 +364,8 @@ function probeDeviceReadHealth(device) { return __awaiter(this, void 0, void 0, function* () { const start = Date.now(); const result = yield exec.getExecOutput('timeout', [ + '-k', + String(READ_PROBE_KILL_AFTER_SECS), String(READ_PROBE_TIMEOUT_SECS), 'sudo', 'dd', @@ -594,6 +598,27 @@ function removeAlternates(workspacePath) { } }); } +/** + * Remove stale git lock files left behind when a git process is killed + * (e.g. by the stall timeout) without a chance to clean up. + */ +function removeStaleGitLocks(workspacePath) { + return __awaiter(this, void 0, void 0, function* () { + const lockFiles = [ + path.join(workspacePath, '.git', 'index.lock'), + path.join(workspacePath, '.git', 'shallow.lock') + ]; + for (const lockFile of lockFiles) { + try { + yield fs.promises.unlink(lockFile); + core.info(`[git-mirror] Removed stale lock file ${lockFile}`); + } + catch (_a) { + // File may not exist, that's fine + } + } + }); +} function writeAlternates(workspacePath, mirrorPath) { return __awaiter(this, void 0, void 0, function* () { const alternatesDir = path.join(workspacePath, '.git', 'objects', 'info'); @@ -1834,6 +1859,12 @@ class GitCommandManager { fetch(refSpec, options) { return __awaiter(this, void 0, void 0, function* () { const args = ['-c', 'protocol.version=2', 'fetch']; + if (options.refetch) { + // Re-download all objects without negotiating with the server, so + // objects previously borrowed from a (now detached) mirror via + // alternates are fetched again instead of being assumed present. + args.push('--refetch'); + } if (!refSpec.some(x => x === refHelper.tagsRefSpec) && !options.fetchTags) { args.push('--no-tags'); } @@ -2453,6 +2484,9 @@ const stateHelper = __importStar(__nccwpck_require__(4866)); const urlHelper = __importStar(__nccwpck_require__(9437)); const blacksmithCache = __importStar(__nccwpck_require__(9242)); const git_command_manager_1 = __nccwpck_require__(738); +const git_version_1 = __nccwpck_require__(3142); +// `git fetch --refetch` requires Git 2.36+. +const MinimumGitRefetchVersion = new git_version_1.GitVersion('2.36'); function getSource(settings) { return __awaiter(this, void 0, void 0, function* () { // Repository URL @@ -2596,8 +2630,14 @@ function getSource(settings) { const disableMirrorAfterStall = (operation) => __awaiter(this, void 0, void 0, function* () { core.warning(`[git-mirror] ${operation} stalled after ${blacksmithCache.MIRROR_STALL_TIMEOUT_SECS}s with the mirror attached; detaching mirror and falling back to network-only checkout`); yield blacksmithCache.removeAlternates(settings.repositoryPath); + // The killed git process may have left stale lock files behind + yield blacksmithCache.removeStaleGitLocks(settings.repositoryPath); cacheInfo = null; }); + // After the mirror is detached, objects previously borrowed via + // alternates are missing locally even though refs may point at them. + // --refetch skips negotiation so those objects are downloaded again. + const canRefetch = (yield git.version()).checkMinimum(MinimumGitRefetchVersion); // Fetch core.startGroup('Fetching the repository'); const fetchOptions = {}; @@ -2620,7 +2660,7 @@ function getSource(settings) { throw error; } yield disableMirrorAfterStall('git fetch'); - yield git.fetch(refSpec, fetchOptions); + yield git.fetch(refSpec, Object.assign(Object.assign({}, fetchOptions), { refetch: canRefetch })); } }); if (settings.fetchDepth <= 0) { @@ -2685,7 +2725,7 @@ function getSource(settings) { yield disableMirrorAfterStall('git checkout'); // Objects previously borrowed from the mirror are gone with the // alternates file; re-fetch so they exist locally, then retry. - yield git.fetch(refHelper.getRefSpec(settings.ref, settings.commit), fetchOptions); + yield git.fetch(refHelper.getRefSpec(settings.ref, settings.commit), Object.assign(Object.assign({}, fetchOptions), { refetch: canRefetch })); yield git.checkout(checkoutInfo.ref, checkoutInfo.startPoint); } } diff --git a/src/blacksmith-cache.ts b/src/blacksmith-cache.ts index 799d522..1d6c97e 100644 --- a/src/blacksmith-cache.ts +++ b/src/blacksmith-cache.ts @@ -31,6 +31,7 @@ const TIMEOUT_EXIT_CODE = 124 // stalls fetch/checkout until the customer's step timeout kills the job. // If the probe cannot read a few MB quickly, skip the mirror entirely. const READ_PROBE_TIMEOUT_SECS = 10 +const READ_PROBE_KILL_AFTER_SECS = 5 const READ_PROBE_MB = 8 // Deadline applied to mirror-assisted `git fetch`/`git checkout` in the main @@ -406,6 +407,8 @@ async function probeDeviceReadHealth(device: string): Promise { const result = await exec.getExecOutput( 'timeout', [ + '-k', + String(READ_PROBE_KILL_AFTER_SECS), String(READ_PROBE_TIMEOUT_SECS), 'sudo', 'dd', @@ -694,6 +697,27 @@ export async function removeAlternates(workspacePath: string): Promise { } } +/** + * Remove stale git lock files left behind when a git process is killed + * (e.g. by the stall timeout) without a chance to clean up. + */ +export async function removeStaleGitLocks( + workspacePath: string +): Promise { + const lockFiles = [ + path.join(workspacePath, '.git', 'index.lock'), + path.join(workspacePath, '.git', 'shallow.lock') + ] + for (const lockFile of lockFiles) { + try { + await fs.promises.unlink(lockFile) + core.info(`[git-mirror] Removed stale lock file ${lockFile}`) + } catch { + // File may not exist, that's fine + } + } +} + export async function writeAlternates( workspacePath: string, mirrorPath: string diff --git a/src/git-command-manager.ts b/src/git-command-manager.ts index 0bbefc6..0753cec 100644 --- a/src/git-command-manager.ts +++ b/src/git-command-manager.ts @@ -61,6 +61,7 @@ export interface IGitCommandManager { fetchTags?: boolean showProgress?: boolean stallTimeoutSecs?: number + refetch?: boolean } ): Promise getDefaultBranch(repositoryUrl: string): Promise @@ -314,9 +315,16 @@ class GitCommandManager { fetchTags?: boolean showProgress?: boolean stallTimeoutSecs?: number + refetch?: boolean } ): Promise { const args = ['-c', 'protocol.version=2', 'fetch'] + if (options.refetch) { + // Re-download all objects without negotiating with the server, so + // objects previously borrowed from a (now detached) mirror via + // alternates are fetched again instead of being assumed present. + args.push('--refetch') + } if (!refSpec.some(x => x === refHelper.tagsRefSpec) && !options.fetchTags) { args.push('--no-tags') } diff --git a/src/git-source-provider.ts b/src/git-source-provider.ts index 360c0a7..f9e256f 100644 --- a/src/git-source-provider.ts +++ b/src/git-source-provider.ts @@ -15,8 +15,12 @@ import { IGitCommandManager, GitStallTimeoutError } from './git-command-manager' +import {GitVersion} from './git-version' import {IGitSourceSettings} from './git-source-settings' +// `git fetch --refetch` requires Git 2.36+. +const MinimumGitRefetchVersion = new GitVersion('2.36') + export async function getSource(settings: IGitSourceSettings): Promise { // Repository URL core.info( @@ -226,9 +230,18 @@ export async function getSource(settings: IGitSourceSettings): Promise { `[git-mirror] ${operation} stalled after ${blacksmithCache.MIRROR_STALL_TIMEOUT_SECS}s with the mirror attached; detaching mirror and falling back to network-only checkout` ) await blacksmithCache.removeAlternates(settings.repositoryPath) + // The killed git process may have left stale lock files behind + await blacksmithCache.removeStaleGitLocks(settings.repositoryPath) cacheInfo = null } + // After the mirror is detached, objects previously borrowed via + // alternates are missing locally even though refs may point at them. + // --refetch skips negotiation so those objects are downloaded again. + const canRefetch = (await git.version()).checkMinimum( + MinimumGitRefetchVersion + ) + // Fetch core.startGroup('Fetching the repository') const fetchOptions: { @@ -261,7 +274,7 @@ export async function getSource(settings: IGitSourceSettings): Promise { throw error } await disableMirrorAfterStall('git fetch') - await git.fetch(refSpec, fetchOptions) + await git.fetch(refSpec, {...fetchOptions, refetch: canRefetch}) } } @@ -339,10 +352,10 @@ export async function getSource(settings: IGitSourceSettings): Promise { await disableMirrorAfterStall('git checkout') // Objects previously borrowed from the mirror are gone with the // alternates file; re-fetch so they exist locally, then retry. - await git.fetch( - refHelper.getRefSpec(settings.ref, settings.commit), - fetchOptions - ) + await git.fetch(refHelper.getRefSpec(settings.ref, settings.commit), { + ...fetchOptions, + refetch: canRefetch + }) await git.checkout(checkoutInfo.ref, checkoutInfo.startPoint) } } else { From 05b404591b585977423261c64566b0c27d12e4df Mon Sep 17 00:00:00 2001 From: trent Date: Mon, 3 Aug 2026 21:46:26 +0000 Subject: [PATCH 4/7] retry sticky disk release in post step when probe-failure release fails Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- dist/index.js | 36 ++++++++++++++++++++++++++++++++---- src/blacksmith-cache.ts | 8 +++++--- src/git-source-provider.ts | 9 +++++++++ src/main.ts | 5 ++++- src/state-helper.ts | 15 +++++++++++++++ 5 files changed, 65 insertions(+), 8 deletions(-) diff --git a/dist/index.js b/dist/index.js index 80c199a..c58f9d9 100644 --- a/dist/index.js +++ b/dist/index.js @@ -330,9 +330,9 @@ function setupCache(owner, repo) { if (!(yield probeDeviceReadHealth(device))) { core.warning(`[git-mirror] Sticky disk ${device} failed the read-health probe; unmounting and falling back to network-only checkout`); yield exec.getExecOutput('timeout', [String(UMOUNT_TIMEOUT_SECS), 'sudo', 'umount', mountPoint], { ignoreReturnCode: true }); - yield releaseStickyDisk(exposeId, stickyDiskKey, repoName); + const released = yield releaseStickyDisk(exposeId, stickyDiskKey, repoName); return { - exposeId: '', + exposeId: released ? '' : exposeId, stickyDiskKey, repoName, device: '', @@ -407,9 +407,11 @@ function releaseStickyDisk(exposeId, stickyDiskKey, repoName) { vmHydratedGitMirror: false }); core.info('[git-mirror] Released sticky disk without committing'); + return true; } catch (error) { core.warning(`[git-mirror] Failed to release sticky disk: ${error.message}`); + return false; } }); } @@ -2557,6 +2559,15 @@ function getSource(settings) { // read-health probe (already unmounted and released) if (cacheInfo.hydrationInProgress || cacheInfo.readProbeFailed) { // Warning already logged by setupCache, just fall back to standard checkout + if (cacheInfo.readProbeFailed && cacheInfo.exposeId) { + // The immediate release failed; save state so the post step can + // retry releasing the disk, and mark it unhealthy so the post + // step never commits it. + stateHelper.setBlacksmithCacheExposeId(cacheInfo.exposeId); + stateHelper.setBlacksmithCacheStickyDiskKey(cacheInfo.stickyDiskKey); + stateHelper.setBlacksmithCacheRepoName(cacheInfo.repoName); + stateHelper.setBlacksmithCacheDiskUnhealthy(); + } cacheInfo = null; core.endGroup(); } @@ -3470,7 +3481,11 @@ function cleanup() { const failureCheck = yield (0, step_checker_1.checkPreviousStepFailures)(); let shouldCommit = true; let skipReason = ''; - if (failureCheck.error) { + if (stateHelper.BlacksmithCacheDiskUnhealthy) { + shouldCommit = false; + skipReason = 'Sticky disk was marked unhealthy during the main step'; + } + else if (failureCheck.error) { // If we can't determine failure status, skip commit to be safe shouldCommit = false; skipReason = `Unable to check for step failures: ${failureCheck.error}`; @@ -3957,7 +3972,7 @@ var __importStar = (this && this.__importStar) || function (mod) { return result; }; Object.defineProperty(exports, "__esModule", ({ value: true })); -exports.BlacksmithCacheVerbose = exports.BlacksmithCacheRepoUrl = exports.BlacksmithCachePerformedHydration = exports.BlacksmithCacheStickyDiskKey = exports.BlacksmithCacheRepoName = exports.BlacksmithCacheMountPoint = exports.BlacksmithCacheMirrorPath = exports.BlacksmithCacheExposeId = exports.SshKnownHostsPath = exports.SshKeyPath = exports.PostSetSafeDirectory = exports.RepositoryPath = exports.IsPost = void 0; +exports.BlacksmithCacheVerbose = exports.BlacksmithCacheRepoUrl = exports.BlacksmithCacheDiskUnhealthy = exports.BlacksmithCachePerformedHydration = exports.BlacksmithCacheStickyDiskKey = exports.BlacksmithCacheRepoName = exports.BlacksmithCacheMountPoint = exports.BlacksmithCacheMirrorPath = exports.BlacksmithCacheExposeId = exports.SshKnownHostsPath = exports.SshKeyPath = exports.PostSetSafeDirectory = exports.RepositoryPath = exports.IsPost = void 0; exports.setRepositoryPath = setRepositoryPath; exports.setSshKeyPath = setSshKeyPath; exports.setSshKnownHostsPath = setSshKnownHostsPath; @@ -3968,6 +3983,7 @@ exports.setBlacksmithCacheMountPoint = setBlacksmithCacheMountPoint; exports.setBlacksmithCacheRepoName = setBlacksmithCacheRepoName; exports.setBlacksmithCacheStickyDiskKey = setBlacksmithCacheStickyDiskKey; exports.setBlacksmithCachePerformedHydration = setBlacksmithCachePerformedHydration; +exports.setBlacksmithCacheDiskUnhealthy = setBlacksmithCacheDiskUnhealthy; exports.setBlacksmithCacheRepoUrl = setBlacksmithCacheRepoUrl; exports.setBlacksmithCacheVerbose = setBlacksmithCacheVerbose; const core = __importStar(__nccwpck_require__(2186)); @@ -4016,6 +4032,11 @@ exports.BlacksmithCacheStickyDiskKey = core.getState('blacksmithCacheStickyDiskK * Used to notify the backend on commit so it can mark hydration as complete. */ exports.BlacksmithCachePerformedHydration = core.getState('blacksmithCachePerformedHydration') === 'true'; +/** + * Indicates the sticky disk was deemed unhealthy (e.g. failed the read-health + * probe) and must never be committed by the POST action. + */ +exports.BlacksmithCacheDiskUnhealthy = core.getState('blacksmithCacheDiskUnhealthy') === 'true'; /** * The repository URL for refreshing the git mirror in the POST action. */ @@ -4085,6 +4106,13 @@ function setBlacksmithCacheStickyDiskKey(stickyDiskKey) { function setBlacksmithCachePerformedHydration(performed) { core.saveState('blacksmithCachePerformedHydration', performed ? 'true' : 'false'); } +/** + * Save that the sticky disk is unhealthy so the POST action releases it + * without committing. + */ +function setBlacksmithCacheDiskUnhealthy() { + core.saveState('blacksmithCacheDiskUnhealthy', 'true'); +} /** * Save the repository URL so the POST action can refresh the git mirror. */ diff --git a/src/blacksmith-cache.ts b/src/blacksmith-cache.ts index 1d6c97e..5147a45 100644 --- a/src/blacksmith-cache.ts +++ b/src/blacksmith-cache.ts @@ -371,9 +371,9 @@ export async function setupCache( [String(UMOUNT_TIMEOUT_SECS), 'sudo', 'umount', mountPoint], {ignoreReturnCode: true} ) - await releaseStickyDisk(exposeId, stickyDiskKey, repoName) + const released = await releaseStickyDisk(exposeId, stickyDiskKey, repoName) return { - exposeId: '', + exposeId: released ? '' : exposeId, stickyDiskKey, repoName, device: '', @@ -448,7 +448,7 @@ async function releaseStickyDisk( exposeId: string, stickyDiskKey: string, repoName: string -): Promise { +): Promise { try { const client = createBlacksmithClient() await client.commitStickyDisk({ @@ -461,10 +461,12 @@ async function releaseStickyDisk( vmHydratedGitMirror: false }) core.info('[git-mirror] Released sticky disk without committing') + return true } catch (error) { core.warning( `[git-mirror] Failed to release sticky disk: ${(error as Error).message}` ) + return false } } diff --git a/src/git-source-provider.ts b/src/git-source-provider.ts index f9e256f..065dc63 100644 --- a/src/git-source-provider.ts +++ b/src/git-source-provider.ts @@ -130,6 +130,15 @@ export async function getSource(settings: IGitSourceSettings): Promise { // read-health probe (already unmounted and released) if (cacheInfo.hydrationInProgress || cacheInfo.readProbeFailed) { // Warning already logged by setupCache, just fall back to standard checkout + if (cacheInfo.readProbeFailed && cacheInfo.exposeId) { + // The immediate release failed; save state so the post step can + // retry releasing the disk, and mark it unhealthy so the post + // step never commits it. + stateHelper.setBlacksmithCacheExposeId(cacheInfo.exposeId) + stateHelper.setBlacksmithCacheStickyDiskKey(cacheInfo.stickyDiskKey) + stateHelper.setBlacksmithCacheRepoName(cacheInfo.repoName) + stateHelper.setBlacksmithCacheDiskUnhealthy() + } cacheInfo = null core.endGroup() } else { diff --git a/src/main.ts b/src/main.ts index 71f0480..f7fd24e 100644 --- a/src/main.ts +++ b/src/main.ts @@ -89,7 +89,10 @@ async function cleanup(): Promise { let shouldCommit = true let skipReason = '' - if (failureCheck.error) { + if (stateHelper.BlacksmithCacheDiskUnhealthy) { + shouldCommit = false + skipReason = 'Sticky disk was marked unhealthy during the main step' + } else if (failureCheck.error) { // If we can't determine failure status, skip commit to be safe shouldCommit = false skipReason = `Unable to check for step failures: ${failureCheck.error}` diff --git a/src/state-helper.ts b/src/state-helper.ts index 5b136af..a87bcc7 100644 --- a/src/state-helper.ts +++ b/src/state-helper.ts @@ -63,6 +63,13 @@ export const BlacksmithCacheStickyDiskKey = core.getState( export const BlacksmithCachePerformedHydration = core.getState('blacksmithCachePerformedHydration') === 'true' +/** + * Indicates the sticky disk was deemed unhealthy (e.g. failed the read-health + * probe) and must never be committed by the POST action. + */ +export const BlacksmithCacheDiskUnhealthy = + core.getState('blacksmithCacheDiskUnhealthy') === 'true' + /** * The repository URL for refreshing the git mirror in the POST action. */ @@ -148,6 +155,14 @@ export function setBlacksmithCachePerformedHydration(performed: boolean) { ) } +/** + * Save that the sticky disk is unhealthy so the POST action releases it + * without committing. + */ +export function setBlacksmithCacheDiskUnhealthy() { + core.saveState('blacksmithCacheDiskUnhealthy', 'true') +} + /** * Save the repository URL so the POST action can refresh the git mirror. */ From 8bc33447f702a00013931d0bd88d8340709a1b4b Mon Sep 17 00:00:00 2001 From: trent Date: Mon, 3 Aug 2026 21:53:49 +0000 Subject: [PATCH 5/7] only enable stall fallback when git supports fetch --refetch Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- dist/index.js | 15 +++++++++------ src/git-source-provider.ts | 18 ++++++++++-------- 2 files changed, 19 insertions(+), 14 deletions(-) diff --git a/dist/index.js b/dist/index.js index c58f9d9..a747d58 100644 --- a/dist/index.js +++ b/dist/index.js @@ -2647,8 +2647,11 @@ function getSource(settings) { }); // After the mirror is detached, objects previously borrowed via // alternates are missing locally even though refs may point at them. - // --refetch skips negotiation so those objects are downloaded again. - const canRefetch = (yield git.version()).checkMinimum(MinimumGitRefetchVersion); + // Recovering requires `git fetch --refetch` (skips negotiation so those + // objects are downloaded again), so the stall fallback is only enabled + // when the installed git supports it. + const stallFallbackEnabled = cacheInfo !== null && + (yield git.version()).checkMinimum(MinimumGitRefetchVersion); // Fetch core.startGroup('Fetching the repository'); const fetchOptions = {}; @@ -2659,7 +2662,7 @@ function getSource(settings) { fetchOptions.filter = 'blob:none'; } const fetchWithMirrorFallback = (refSpec) => __awaiter(this, void 0, void 0, function* () { - if (!cacheInfo) { + if (!cacheInfo || !stallFallbackEnabled) { yield git.fetch(refSpec, fetchOptions); return; } @@ -2671,7 +2674,7 @@ function getSource(settings) { throw error; } yield disableMirrorAfterStall('git fetch'); - yield git.fetch(refSpec, Object.assign(Object.assign({}, fetchOptions), { refetch: canRefetch })); + yield git.fetch(refSpec, Object.assign(Object.assign({}, fetchOptions), { refetch: true })); } }); if (settings.fetchDepth <= 0) { @@ -2725,7 +2728,7 @@ function getSource(settings) { } // Checkout core.startGroup('Checking out the ref'); - if (cacheInfo) { + if (cacheInfo && stallFallbackEnabled) { try { yield git.checkout(checkoutInfo.ref, checkoutInfo.startPoint, blacksmithCache.MIRROR_STALL_TIMEOUT_SECS); } @@ -2736,7 +2739,7 @@ function getSource(settings) { yield disableMirrorAfterStall('git checkout'); // Objects previously borrowed from the mirror are gone with the // alternates file; re-fetch so they exist locally, then retry. - yield git.fetch(refHelper.getRefSpec(settings.ref, settings.commit), Object.assign(Object.assign({}, fetchOptions), { refetch: canRefetch })); + yield git.fetch(refHelper.getRefSpec(settings.ref, settings.commit), Object.assign(Object.assign({}, fetchOptions), { refetch: true })); yield git.checkout(checkoutInfo.ref, checkoutInfo.startPoint); } } diff --git a/src/git-source-provider.ts b/src/git-source-provider.ts index 065dc63..705aa1b 100644 --- a/src/git-source-provider.ts +++ b/src/git-source-provider.ts @@ -246,10 +246,12 @@ export async function getSource(settings: IGitSourceSettings): Promise { // After the mirror is detached, objects previously borrowed via // alternates are missing locally even though refs may point at them. - // --refetch skips negotiation so those objects are downloaded again. - const canRefetch = (await git.version()).checkMinimum( - MinimumGitRefetchVersion - ) + // Recovering requires `git fetch --refetch` (skips negotiation so those + // objects are downloaded again), so the stall fallback is only enabled + // when the installed git supports it. + const stallFallbackEnabled = + cacheInfo !== null && + (await git.version()).checkMinimum(MinimumGitRefetchVersion) // Fetch core.startGroup('Fetching the repository') @@ -269,7 +271,7 @@ export async function getSource(settings: IGitSourceSettings): Promise { const fetchWithMirrorFallback = async ( refSpec: string[] ): Promise => { - if (!cacheInfo) { + if (!cacheInfo || !stallFallbackEnabled) { await git.fetch(refSpec, fetchOptions) return } @@ -283,7 +285,7 @@ export async function getSource(settings: IGitSourceSettings): Promise { throw error } await disableMirrorAfterStall('git fetch') - await git.fetch(refSpec, {...fetchOptions, refetch: canRefetch}) + await git.fetch(refSpec, {...fetchOptions, refetch: true}) } } @@ -347,7 +349,7 @@ export async function getSource(settings: IGitSourceSettings): Promise { // Checkout core.startGroup('Checking out the ref') - if (cacheInfo) { + if (cacheInfo && stallFallbackEnabled) { try { await git.checkout( checkoutInfo.ref, @@ -363,7 +365,7 @@ export async function getSource(settings: IGitSourceSettings): Promise { // alternates file; re-fetch so they exist locally, then retry. await git.fetch(refHelper.getRefSpec(settings.ref, settings.commit), { ...fetchOptions, - refetch: canRefetch + refetch: true }) await git.checkout(checkoutInfo.ref, checkoutInfo.startPoint) } From 7e5bccdecd56aae896133c81ce8cb5e6752009c1 Mon Sep 17 00:00:00 2001 From: trent Date: Mon, 3 Aug 2026 22:01:24 +0000 Subject: [PATCH 6/7] keep transient-error retries on mirror-assisted fetch, abort only on stall Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- dist/index.js | 24 ++++++++++++++++++++---- src/git-command-manager.ts | 23 +++++++++++++++++++---- 2 files changed, 39 insertions(+), 8 deletions(-) diff --git a/dist/index.js b/dist/index.js index a747d58..bc21dad 100644 --- a/dist/index.js +++ b/dist/index.js @@ -1887,13 +1887,29 @@ class GitCommandManager { for (const arg of refSpec) { args.push(arg); } + const that = this; if (options.stallTimeoutSecs) { - // Single attempt: on a stall the caller falls back to a network-only - // fetch, so retrying with the mirror still attached only burns time. - yield this.execGit(args, false, false, {}, options.stallTimeoutSecs); + // Transient failures are still retried, but a stall aborts immediately: + // the caller falls back to a network-only fetch, so retrying with the + // mirror still attached only burns time. + let stallError; + yield retryHelper.execute(() => __awaiter(this, void 0, void 0, function* () { + try { + yield that.execGit(args, false, false, {}, options.stallTimeoutSecs); + } + catch (err) { + if (err instanceof GitStallTimeoutError) { + stallError = err; + return; + } + throw err; + } + })); + if (stallError) { + throw stallError; + } return; } - const that = this; yield retryHelper.execute(() => __awaiter(this, void 0, void 0, function* () { yield that.execGit(args); })); diff --git a/src/git-command-manager.ts b/src/git-command-manager.ts index 0753cec..c08d6b4 100644 --- a/src/git-command-manager.ts +++ b/src/git-command-manager.ts @@ -353,14 +353,29 @@ class GitCommandManager { args.push(arg) } + const that = this if (options.stallTimeoutSecs) { - // Single attempt: on a stall the caller falls back to a network-only - // fetch, so retrying with the mirror still attached only burns time. - await this.execGit(args, false, false, {}, options.stallTimeoutSecs) + // Transient failures are still retried, but a stall aborts immediately: + // the caller falls back to a network-only fetch, so retrying with the + // mirror still attached only burns time. + let stallError: GitStallTimeoutError | undefined + await retryHelper.execute(async () => { + try { + await that.execGit(args, false, false, {}, options.stallTimeoutSecs) + } catch (err) { + if (err instanceof GitStallTimeoutError) { + stallError = err + return + } + throw err + } + }) + if (stallError) { + throw stallError + } return } - const that = this await retryHelper.execute(async () => { await that.execGit(args) }) From f16ec5bb7cd6345646872bc5b0e75ec514b821f3 Mon Sep 17 00:00:00 2001 From: trent Date: Mon, 3 Aug 2026 22:07:51 +0000 Subject: [PATCH 7/7] add watchdog so a D-state git stall cannot outlive the stall timeout Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- dist/index.js | 21 ++++++++++++++++++++- src/git-command-manager.ts | 27 ++++++++++++++++++++++++--- 2 files changed, 44 insertions(+), 4 deletions(-) diff --git a/dist/index.js b/dist/index.js index bc21dad..3937792 100644 --- a/dist/index.js +++ b/dist/index.js @@ -2209,7 +2209,26 @@ class GitCommandManager { this.gitPath, ...args ]; - result.exitCode = yield exec.exec('timeout', timeoutArgs, Object.assign(Object.assign({}, options), { ignoreReturnCode: true })); + // If git is stuck in uninterruptible sleep (D-state) on a dead storage + // backend, even SIGKILL is deferred and `timeout` never returns. Race + // the exec against a watchdog so we can abandon the wait and fall back + // to the network path regardless. + const watchdogSecs = stallTimeoutSecs + STALL_KILL_AFTER_SECS * 2; + let watchdogTimer; + const watchdog = new Promise(resolve => { + watchdogTimer = setTimeout(() => resolve(TIMEOUT_EXIT_CODE), watchdogSecs * 1000); + }); + try { + result.exitCode = yield Promise.race([ + exec.exec('timeout', timeoutArgs, Object.assign(Object.assign({}, options), { ignoreReturnCode: true })), + watchdog + ]); + } + finally { + if (watchdogTimer) { + clearTimeout(watchdogTimer); + } + } result.stdout = stdout.join(''); if (result.exitCode === TIMEOUT_EXIT_CODE) { throw new GitStallTimeoutError(args.join(' '), stallTimeoutSecs); diff --git a/src/git-command-manager.ts b/src/git-command-manager.ts index c08d6b4..bb0b620 100644 --- a/src/git-command-manager.ts +++ b/src/git-command-manager.ts @@ -720,10 +720,31 @@ class GitCommandManager { this.gitPath, ...args ] - result.exitCode = await exec.exec('timeout', timeoutArgs, { - ...options, - ignoreReturnCode: true + // If git is stuck in uninterruptible sleep (D-state) on a dead storage + // backend, even SIGKILL is deferred and `timeout` never returns. Race + // the exec against a watchdog so we can abandon the wait and fall back + // to the network path regardless. + const watchdogSecs = stallTimeoutSecs + STALL_KILL_AFTER_SECS * 2 + let watchdogTimer: NodeJS.Timeout | undefined + const watchdog = new Promise(resolve => { + watchdogTimer = setTimeout( + () => resolve(TIMEOUT_EXIT_CODE), + watchdogSecs * 1000 + ) }) + try { + result.exitCode = await Promise.race([ + exec.exec('timeout', timeoutArgs, { + ...options, + ignoreReturnCode: true + }), + watchdog + ]) + } finally { + if (watchdogTimer) { + clearTimeout(watchdogTimer) + } + } result.stdout = stdout.join('') if (result.exitCode === TIMEOUT_EXIT_CODE) { throw new GitStallTimeoutError(args.join(' '), stallTimeoutSecs)