fall back to network checkout when the mirror sticky disk stalls

src/blacksmith-cache.ts

@@ -25,6 +25,21 @@ const UMOUNT_BACKOFF_MULTIPLIER = 2 // Exponential backoff multiplier
 // Exit code returned by the `timeout` command when the child is killed.
 const TIMEOUT_EXIT_CODE = 124
 
+// Read-health probe of the mirror sticky disk, run right after mount. A
+// degraded Ceph-backed disk can serve reads at pathological latencies
+// (observed: hundreds of seconds of read wait during a 5-minute step), which
+// stalls fetch/checkout until the customer's step timeout kills the job.
+// If the probe cannot read a few MB quickly, skip the mirror entirely.
+const READ_PROBE_TIMEOUT_SECS = 10
+const READ_PROBE_KILL_AFTER_SECS = 5
+const READ_PROBE_MB = 8
+
+// Deadline applied to mirror-assisted `git fetch`/`git checkout` in the main
+// step. If either stalls past this (e.g. alternates reads hitting a degraded
+// sticky disk), the checkout falls back to a network-only fetch/checkout
+// instead of failing the job.
+export const MIRROR_STALL_TIMEOUT_SECS = 120
+
 /**
  * Result of a git mirror operation that may fail or time out.
  */
@@ -65,6 +80,10 @@ export interface CacheInfo {
   // performedHydration indicates that this job performed the initial git mirror clone.
   // Used to notify the backend on commit so it can mark hydration as complete.
   performedHydration: boolean
+  // readProbeFailed indicates the mounted sticky disk failed the read-health
+  // probe. The disk has already been unmounted and released; the caller
+  // should fall back to regular checkout without using the cache.
+  readProbeFailed?: boolean
 }
 
 /**
@@ -343,6 +362,29 @@ export async function setupCache(
   await exec.exec('sudo', ['mount', '-o', 'noinit_itable', device, mountPoint])
   core.info(`[git-mirror] Mounted ${device} at ${mountPoint}`)
 
+  if (!(await probeDeviceReadHealth(device))) {
+    core.warning(
+      `[git-mirror] Sticky disk ${device} failed the read-health probe; unmounting and falling back to network-only checkout`
+    )
+    await exec.getExecOutput(
+      'timeout',
+      [String(UMOUNT_TIMEOUT_SECS), 'sudo', 'umount', mountPoint],
+      {ignoreReturnCode: true}
+    )
+    const released = await releaseStickyDisk(exposeId, stickyDiskKey, repoName)
+    return {
+      exposeId: released ? '' : exposeId,
+      stickyDiskKey,
+      repoName,
+      device: '',
+      mountPoint: '',
+      mirrorPath: '',
+      hydrationInProgress: false,
+      performedHydration: false,
+      readProbeFailed: true
+    }
+  }
+
   return {
     exposeId,
     stickyDiskKey,
@@ -355,6 +397,79 @@ export async function setupCache(
   }
 }
 
+/**
+ * Check that the sticky disk can serve reads at a sane latency by reading a
+ * few MB directly from the block device (O_DIRECT, bypassing the page cache).
+ * Returns false if the read fails or takes longer than the probe timeout.
+ */
+async function probeDeviceReadHealth(device: string): Promise<boolean> {
+  const start = Date.now()
+  const result = await exec.getExecOutput(
+    'timeout',
+    [
+      '-k',
+      String(READ_PROBE_KILL_AFTER_SECS),
+      String(READ_PROBE_TIMEOUT_SECS),
+      'sudo',
+      'dd',
+      `if=${device}`,
+      'of=/dev/null',
+      'bs=1M',
+      `count=${READ_PROBE_MB}`,
+      'iflag=direct'
+    ],
+    {ignoreReturnCode: true, silent: true}
+  )
+  const durationMs = Date.now() - start
+  if (result.exitCode === TIMEOUT_EXIT_CODE) {
+    core.warning(
+      `[git-mirror] Read-health probe timed out after ${READ_PROBE_TIMEOUT_SECS}s reading ${READ_PROBE_MB}MB from ${device}`
+    )
+    return false
+  }
+  if (result.exitCode !== 0) {
+    core.warning(
+      `[git-mirror] Read-health probe failed with exit code ${result.exitCode}: ${result.stderr.trim()}`
+    )
+    return false
+  }
+  core.info(
+    `[git-mirror] Read-health probe: read ${READ_PROBE_MB}MB from ${device} in ${durationMs}ms`
+  )
+  return true
+}
+
+/**
+ * Release a sticky disk without persisting any changes. Used when the disk
+ * is unusable (e.g. failed the read-health probe) so the backend can detach
+ * it instead of leaving the exposure dangling.
+ */
+async function releaseStickyDisk(
+  exposeId: string,
+  stickyDiskKey: string,
+  repoName: string
+): Promise<boolean> {
+  try {
+    const client = createBlacksmithClient()
+    await client.commitStickyDisk({
+      exposeId,
+      stickyDiskKey,
+      vmId: process.env.BLACKSMITH_VM_ID || '',
+      shouldCommit: false,
+      repoName,
+      stickyDiskToken: process.env.BLACKSMITH_STICKYDISK_TOKEN || '',
+      vmHydratedGitMirror: false
+    })
+    core.info('[git-mirror] Released sticky disk without committing')
+    return true
+  } catch (error) {
+    core.warning(
+      `[git-mirror] Failed to release sticky disk: ${(error as Error).message}`
+    )
+    return false
+  }
+}
+
 /**
  * Get the extraheader config value for git authentication.
  * Uses the same format as upstream actions/checkout:
@@ -563,6 +678,48 @@ export async function refreshMirror(
  * This allows the workspace git repo to use objects from the mirror
  * without copying them
  */
+/**
+ * Remove the alternates file so the workspace repo no longer borrows objects
+ * from the mirror. Any objects that were only reachable via the mirror must
+ * be re-fetched from the network afterwards.
+ */
+export async function removeAlternates(workspacePath: string): Promise<void> {
+  const alternatesFile = path.join(
+    workspacePath,
+    '.git',
+    'objects',
+    'info',
+    'alternates'
+  )
+  try {
+    await fs.promises.unlink(alternatesFile)
+    core.info('[git-mirror] Removed alternates file')
+  } catch {
+    // File may not exist, that's fine
+  }
+}
+
+/**
+ * Remove stale git lock files left behind when a git process is killed
+ * (e.g. by the stall timeout) without a chance to clean up.
+ */
+export async function removeStaleGitLocks(
+  workspacePath: string
+): Promise<void> {
+  const lockFiles = [
+    path.join(workspacePath, '.git', 'index.lock'),
+    path.join(workspacePath, '.git', 'shallow.lock')
+  ]
+  for (const lockFile of lockFiles) {
+    try {
+      await fs.promises.unlink(lockFile)
+      core.info(`[git-mirror] Removed stale lock file ${lockFile}`)
+    } catch {
+      // File may not exist, that's fine
+    }
+  }
+}
+
 export async function writeAlternates(
   workspacePath: string,
   mirrorPath: string

src/git-command-manager.ts

@@ -15,14 +15,35 @@ import {GitVersion} from './git-version'
 export const MinimumGitVersion = new GitVersion('2.18')
 export const MinimumGitSparseCheckoutVersion = new GitVersion('2.28')
 
+// Exit code returned by the `timeout` command when the child is killed.
+const TIMEOUT_EXIT_CODE = 124
+// Grace period before `timeout` escalates from SIGTERM to SIGKILL.
+const STALL_KILL_AFTER_SECS = 10
+
+/**
+ * Thrown when a git command wrapped with a stall timeout is killed because it
+ * exceeded the deadline. Callers use this to distinguish a stalled command
+ * (e.g. mirror sticky-disk reads hanging) from a genuine git failure.
+ */
+export class GitStallTimeoutError extends Error {
+  constructor(command: string, timeoutSecs: number) {
+    super(`git ${command} stalled and was killed after ${timeoutSecs}s`)
+    this.name = 'GitStallTimeoutError'
+  }
+}
+
 export interface IGitCommandManager {
   branchDelete(remote: boolean, branch: string): Promise<void>
   branchExists(remote: boolean, pattern: string): Promise<boolean>
   branchList(remote: boolean): Promise<string[]>
   disableSparseCheckout(): Promise<void>
   sparseCheckout(sparseCheckout: string[]): Promise<void>
   sparseCheckoutNonConeMode(sparseCheckout: string[]): Promise<void>
-  checkout(ref: string, startPoint: string): Promise<void>
+  checkout(
+    ref: string,
+    startPoint: string,
+    stallTimeoutSecs?: number
+  ): Promise<void>
   checkoutDetach(): Promise<void>
   config(
     configKey: string,
@@ -39,6 +60,8 @@ export interface IGitCommandManager {
       fetchDepth?: number
       fetchTags?: boolean
       showProgress?: boolean
+      stallTimeoutSecs?: number
+      refetch?: boolean
     }
   ): Promise<void>
   getDefaultBranch(repositoryUrl: string): Promise<string>
@@ -221,14 +244,23 @@ class GitCommandManager {
     )
   }
 
-  async checkout(ref: string, startPoint: string): Promise<void> {
+  async checkout(
+    ref: string,
+    startPoint: string,
+    stallTimeoutSecs?: number
+  ): Promise<void> {
     const args = ['checkout', '--progress', '--force']
     if (startPoint) {
       args.push('-B', ref, startPoint)
     } else {
       args.push(ref)
     }
 
+    if (stallTimeoutSecs) {
+      await this.execGit(args, false, false, {}, stallTimeoutSecs)
+      return
+    }
+
     await this.execGit(args)
   }
 
@@ -282,9 +314,17 @@ class GitCommandManager {
       fetchDepth?: number
       fetchTags?: boolean
       showProgress?: boolean
+      stallTimeoutSecs?: number
+      refetch?: boolean
     }
   ): Promise<void> {
     const args = ['-c', 'protocol.version=2', 'fetch']
+    if (options.refetch) {
+      // Re-download all objects without negotiating with the server, so
+      // objects previously borrowed from a (now detached) mirror via
+      // alternates are fetched again instead of being assumed present.
+      args.push('--refetch')
+    }
     if (!refSpec.some(x => x === refHelper.tagsRefSpec) && !options.fetchTags) {
       args.push('--no-tags')
     }
@@ -314,6 +354,28 @@ class GitCommandManager {
     }
 
     const that = this
+    if (options.stallTimeoutSecs) {
+      // Transient failures are still retried, but a stall aborts immediately:
+      // the caller falls back to a network-only fetch, so retrying with the
+      // mirror still attached only burns time.
+      let stallError: GitStallTimeoutError | undefined
+      await retryHelper.execute(async () => {
+        try {
+          await that.execGit(args, false, false, {}, options.stallTimeoutSecs)
+        } catch (err) {
+          if (err instanceof GitStallTimeoutError) {
+            stallError = err
+            return
+          }
+          throw err
+        }
+      })
+      if (stallError) {
+        throw stallError
+      }
+      return
+    }
+
     await retryHelper.execute(async () => {
       await that.execGit(args)
     })
@@ -615,7 +677,8 @@ class GitCommandManager {
     args: string[],
     allowAllExitCodes = false,
     silent = false,
-    customListeners = {}
+    customListeners = {},
+    stallTimeoutSecs?: number
   ): Promise<GitOutput> {
     fshelper.directoryExistsSync(this.workingDirectory, true)
 
@@ -646,6 +709,56 @@ class GitCommandManager {
       listeners: mergedListeners
     }
 
+    if (stallTimeoutSecs) {
+      // Run git under `timeout` so a command stalled on unresponsive storage
+      // (e.g. a degraded mirror sticky disk) is killed instead of running
+      // until the job-level timeout. Only used on Blacksmith Linux runners.
+      const timeoutArgs = [
+        '-k',
+        String(STALL_KILL_AFTER_SECS),
+        String(stallTimeoutSecs),
+        this.gitPath,
+        ...args
+      ]
+      // If git is stuck in uninterruptible sleep (D-state) on a dead storage
+      // backend, even SIGKILL is deferred and `timeout` never returns. Race
+      // the exec against a watchdog so we can abandon the wait and fall back
+      // to the network path regardless.
+      const watchdogSecs = stallTimeoutSecs + STALL_KILL_AFTER_SECS * 2
+      let watchdogTimer: NodeJS.Timeout | undefined
+      const watchdog = new Promise<number>(resolve => {
+        watchdogTimer = setTimeout(
+          () => resolve(TIMEOUT_EXIT_CODE),
+          watchdogSecs * 1000
+        )
+      })
+      try {
+        result.exitCode = await Promise.race([
+          exec.exec('timeout', timeoutArgs, {
+            ...options,
+            ignoreReturnCode: true
+          }),
+          watchdog
+        ])
+      } finally {
+        if (watchdogTimer) {
+          clearTimeout(watchdogTimer)
+        }
+      }
+      result.stdout = stdout.join('')
+      if (result.exitCode === TIMEOUT_EXIT_CODE) {
+        throw new GitStallTimeoutError(args.join(' '), stallTimeoutSecs)
+      }
+      if (result.exitCode !== 0 && !allowAllExitCodes) {
+        throw new Error(
+          `The process '${this.gitPath}' failed with exit code ${result.exitCode}`
+        )
+      }
+      core.debug(result.exitCode.toString())
+      core.debug(result.stdout)
+      return result
+    }
+
     result.exitCode = await exec.exec(`"${this.gitPath}"`, args, options)
     result.stdout = stdout.join('')