From ba99891d3820d6baa6d2602ad5b1ee0bc8f3b400 Mon Sep 17 00:00:00 2001 From: unohee Date: Thu, 22 Oct 2026 02:16:47 +0900 Subject: [PATCH] fix(daemon): pause dispatch on low disk and carry workerSandbox to the runner (AGT-4785) The heartbeat now measures free space on every volume the daemon writes to and holds dispatch below autonomous.minFreeDiskGiB (default 10), after the worktree sweep and before any fetch or claim. One notice per change of state. service.ts also never passed autonomous.workerSandbox to the runner, so the worker bash ran unfenced while the boot log said the sandbox was on. --- CHANGELOG.md | 3 + config.example.yaml | 6 + .../autonomousRunner.hostHealth.test.ts | 104 ++++++++++ src/automation/autonomousRunner.ts | 19 ++ src/automation/runnerTypes.ts | 2 + src/core/config.test.ts | 16 ++ src/core/config.ts | 3 + src/core/service.test.ts | 13 ++ src/core/service.ts | 6 + src/core/types.ts | 6 + src/support/hostHealth.test.ts | 180 ++++++++++++++++++ src/support/hostHealth.ts | 152 +++++++++++++++ 12 files changed, 510 insertions(+) create mode 100644 src/automation/autonomousRunner.hostHealth.test.ts create mode 100644 src/support/hostHealth.test.ts create mode 100644 src/support/hostHealth.ts diff --git a/CHANGELOG.md b/CHANGELOG.md index 325cc7b5..fdd0a861 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -18,6 +18,8 @@ ### Added +- **The daemon stops dispatching when the disk is nearly full (AGT-4785).** On 2026-10-20 the host ran out of disk: tester and worker stages failed with `ENOSPC` copying git packs into `$TMPDIR/.openswarm-verify-head-*`, most of the night's runs ended `infra_error`, and every new heartbeat claimed more work that could only fail the same way. `src/support/hostHealth.ts` measures free bytes (`statfs`, `bavail` — the root-reserved blocks are not ours) on every volume the daemon writes to (each allowed project, the temp dir) and the heartbeat pauses dispatch when the scarcest is under `autonomous.minFreeDiskGiB` (default 10; 0 switches the check off). The check sits after the worktree sweep, which is what frees space, so a full disk stops new work and never the cleanup that would end the condition. One notice on the way down and one on recovery — not one per heartbeat. Reads fail open: a path that cannot be measured is skipped, and with nothing measurable dispatch continues. `isHostResourceError` recognises `ENOSPC` by code or by the text git and node print. **Not done:** an explicit dispatch (`/api/work`, `exec --issue`) is not held by this gate, and a run that already failed with `ENOSPC` is still classified as a plain `infra_error`. + - **`projectAgents[].setup` — dependency install commands that run in the worktree before the worker (AGT-4798).** They run with the network open but inside the sandbox and without credentials, in order, stopping at the first failure; a failed setup ends the run as `infra_error` before any model call. Most projects need none. - **`openswarm exec --issue INT-123` deploys the worker pipeline for one Linear issue asynchronously and hands the run to an external orchestrator.** It resolves the issue, checks the repo's Linear project mapping and the deployable state, records an **exec hold**, and posts to the daemon's `/api/work` (falling back to a detached `work` child only when the daemon is unreachable — never on an HTTP refusal, so a 403 from `autonomous.allowedProjects` cannot be routed around), then prints `{runId, identifier, transport, deployed}` and returns. `exec status `, `exec await --timeout ` and `exec answer [--correlation hq-…] [--text …]` are the observation half: `await` blocks until the run elevates or finishes and prints the structured elevation JSON (state, park reason, branch/worktree/PR, pending questions with their class, and the exact next command), and `answer` writes the reply to the coordination board as `actorRole: 'orchestrator'` — the same delegated authority OpenSwarm's own supervisor answers with — resumes the ledger row through the exact-correlation park contract first, re-marks the hold, and redeploys. All four take `--json`. Reads use a read-only handle (`{ readonly: true }`, the `ledgerOverviewTool` pattern) so `status`/`await` never create a database, migrate a schema, or take a write lock on a live daemon's file. @@ -32,6 +34,7 @@ ### Fixed +- **`autonomous.workerSandbox` never reached the runner, so the worker's bash ran unfenced while the boot log said "Worker sandbox: on" (found while wiring AGT-4785).** The schema defaults it to `on` and the boot log reads it from the config object, but `core/service.ts` builds the runner's config by hand-picked mapping and did not carry the field; the runner then saw `undefined`, and the bash tool fences only on an explicit `on` (`tools.ts`, `execOptions?.sandbox === 'on'`; `pairPipelineTypes.ts`: "Unset = off"). The setup step defaults an unset value to `on`, so it was fenced while the worker's own shell was not. Every control that rides on the sandbox wrapper for the worker's shell — the loopback-only network (AGT-4798) and the write fence — was inactive in a daemon started through `service.ts`; the credential-free environment (AGT-4783) does not depend on it and was in effect. The same class as AGT-4122 and AGT-4404. The field is now mapped, and `service.test.ts` asserts it (removing the mapping makes that test fail). Not checked: whether the CLI entry points that build their own runner config carry the field. - **The test suite no longer writes into the operator's live `~/.openswarm/task-state.json`.** `vitest.setup.ts` redirected a dozen home-dir state paths per worker but missed `OPENSWARM_TASK_STATE_FILE`, so suites that built a real task-state store fell back to the default path and appended fixture rows (`issue-uuid`/"Serial task", `INT-1`) to the live file. It is now redirected per worker like the rest. ## 0.24.4 — 2026-09-29 diff --git a/config.example.yaml b/config.example.yaml index c9398acd..fb2788dc 100644 --- a/config.example.yaml +++ b/config.example.yaml @@ -272,6 +272,12 @@ autonomous: # that need the network go in projectAgents[].setup below. # workerNetwork: loopback + # Free-space floor, in GiB, for every volume the daemon writes to (each + # allowed project, the temp dir). Below it the heartbeat stops fetching and + # claiming work — worktree cleanup keeps running — and resumes once space is + # freed; one notice each way. 0 switches the check off. (AGT-4785) + # minFreeDiskGiB: 10 + # Delegated CLI adapters (claude, codex, cursor) own their tool loop, so # OpenSwarm's publication fence, protected files, role tool limits and the # credential-free sandboxed shell do not reach them (claude even runs with diff --git a/src/automation/autonomousRunner.hostHealth.test.ts b/src/automation/autonomousRunner.hostHealth.test.ts new file mode 100644 index 00000000..8f45945f --- /dev/null +++ b/src/automation/autonomousRunner.hostHealth.test.ts @@ -0,0 +1,104 @@ +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; +import { mkdtempSync, rmSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import type { TaskItem } from '../orchestration/decisionEngine.js'; +import type { ITaskSource } from './taskSource.js'; +import type { DurableRunCoordinator } from './durableRunCoordinator.js'; + +// AGT-4785: on 2026-10-20 the host ran out of disk and the daemon kept claiming work that could +// only fail with ENOSPC. HostDiskGate is unit-tested in support/hostHealth.test.ts; this proves +// the heartbeat consults it with the configured floor, before anything is fetched. + +vi.mock('../core/providerOverride.js', () => ({ writeProviderOverride: vi.fn() })); +vi.mock('../agents/stageModelResolver.js', () => ({ resolveAdapterDefaultModel: vi.fn(async () => 'model') })); +vi.mock('../memory/repoKnowledge.js', () => ({ + recordTaskOutcome: vi.fn(async () => {}), + promoteStagedMemories: vi.fn(async () => 0), +})); +vi.mock('../linear/projectUpdater.js', () => ({ updateProjectAfterTask: vi.fn(async () => {}) })); + +type InternalRunner = { + heartbeatParallel: ReturnType; + refreshKnowledgeGraphs(): void; + durableRuns: DurableRunCoordinator; +}; + +// No real volume has this much free space, so a floor this high is always "low". +const UNREACHABLE_FLOOR_GIB = 1e9; + +describe('heartbeat holds dispatch when the disk is nearly full (AGT-4785)', () => { + let root: string; + + beforeEach(() => { + vi.useFakeTimers(); + vi.setSystemTime(new Date('2026-07-21T16:00:00Z')); // 01:00 KST: work window allowed + root = mkdtempSync(join(tmpdir(), 'openswarm-runner-hosthealth-')); + vi.stubEnv('OPENSWARM_TASK_STATE_FILE', join(root, 'task-state.json')); + vi.stubEnv('OPENSWARM_RUNNER_TASK_STATE_FILE', join(root, 'runner-state.json')); + vi.stubEnv('OPENSWARM_RUNNER_REJECTION_STATE_FILE', join(root, 'rejections.json')); + vi.stubEnv('OPENSWARM_RUNNER_PIPELINE_HISTORY_FILE', join(root, 'history.json')); + vi.stubEnv('OPENSWARM_RUNNER_DECOMPOSITION_STATE_FILE', join(root, 'decomposition.json')); + }); + + afterEach(() => { + vi.useRealTimers(); + vi.unstubAllEnvs(); + vi.restoreAllMocks(); + rmSync(root, { recursive: true, force: true }); + vi.resetModules(); + }); + + async function setup(minFreeDiskGiB: number | undefined) { + const [{ AutonomousRunner }, execution] = await Promise.all([ + import('./autonomousRunner.js'), + import('./runnerExecution.js'), + ]); + const fetchTasks = vi.fn(async () => [] as TaskItem[]); + const source = { + kind: 'linear', fetchTasks, + lookupIssueState: vi.fn(async () => ({ ok: true as const, issue: { state: 'Todo', stateType: 'unstarted' } })), + updateState: vi.fn(async () => true), addComment: vi.fn(async () => {}), + createTask: vi.fn(), createSubIssue: vi.fn(), logPairStart: vi.fn(), + logPairComplete: vi.fn(), logBlocked: vi.fn(), logStuck: vi.fn(), unstick: vi.fn(), + logHalt: vi.fn(), markAsDecomposed: vi.fn(), + } as unknown as ITaskSource; + execution.setTaskSource(source); + const runner = new AutonomousRunner({ + linearTeamId: 'team', allowedProjects: [root], heartbeatSchedule: '0 * * * *', + autoExecute: true, dryRun: true, pairMode: true, maxConcurrentTasks: 4, + automationLedgerMode: 'primary', automationDbPath: join(root, 'hosthealth.db'), + ...(minFreeDiskGiB === undefined ? {} : { minFreeDiskGiB }), + }); + const internal = runner as unknown as InternalRunner; + internal.refreshKnowledgeGraphs = vi.fn(); + internal.heartbeatParallel = vi.fn(async () => {}); + return { runner, internal, fetchTasks }; + } + + it('fetches and claims nothing while free space is under the configured floor', async () => { + const { runner, internal, fetchTasks } = await setup(UNREACHABLE_FLOOR_GIB); + await runner.heartbeat(); + internal.durableRuns.close(); + expect(fetchTasks).not.toHaveBeenCalled(); + expect(internal.heartbeatParallel).not.toHaveBeenCalled(); + }, 60_000); + + it('says so once per change of state, not once per heartbeat', async () => { + const warn = vi.spyOn(console, 'warn').mockImplementation(() => {}); + const { runner, internal } = await setup(UNREACHABLE_FLOOR_GIB); + await runner.heartbeat(); + await runner.heartbeat(); + await runner.heartbeat(); + internal.durableRuns.close(); + const pauses = warn.mock.calls.filter((c) => String(c[0]).includes('Dispatch paused')); + expect(pauses).toHaveLength(1); + }, 60_000); + + it('a floor of 0 switches the check off and the heartbeat proceeds to fetch', async () => { + const { runner, internal, fetchTasks } = await setup(0); + await runner.heartbeat(); + internal.durableRuns.close(); + expect(fetchTasks).toHaveBeenCalled(); + }, 60_000); +}); diff --git a/src/automation/autonomousRunner.ts b/src/automation/autonomousRunner.ts index 39f6cacd..e67978f6 100644 --- a/src/automation/autonomousRunner.ts +++ b/src/automation/autonomousRunner.ts @@ -70,6 +70,7 @@ import { SANDBOX_OUTCOME_UNKNOWN_PARK_REASON } from '../sandboxExecutor/protocol import { INFRA_CIRCUIT_PARK_REASON } from './infraFailureCircuit.js'; import { listHeldIssueIds } from './execHold.js'; import { inputFingerprint, mayRedispatch } from './dispatchFingerprint.js'; +import { HostDiskGate, dispatchWritePaths } from '../support/hostHealth.js'; import { VERIFIED_WORKER_BLOCKER_PARK_REASON } from '../agents/pairPipelineTypes.js'; import { decideExplicitReadmission, OPERATOR_QUESTION_PARK_MARKER } from './explicitDispatchReadmission.js'; import { broadcastEvent, type SwarmStats } from '../core/eventHub.js'; @@ -533,6 +534,11 @@ export class AutonomousRunner { // Rate-limit hold: epoch ms until which all task execution is paused. // Set when any adapter returns a 429 / usage_limit_reached response (INT-1906). private rateLimitUntil = 0; + // Disk-space hold: new claims wait while the volumes we write to are nearly full (AGT-4785). + private readonly diskGate = new HostDiskGate( + () => dispatchWritePaths(this.config.allowedProjects.map((p) => normalizeProjectPath(p))), + () => (this.config.minFreeDiskGiB ?? 10) * 1024 ** 3, + ); // Issues whose Linear project can't be mapped to a local repo path. Recorded on // the first resolve failure so they aren't re-picked every heartbeat (which @@ -2838,6 +2844,19 @@ export class AutonomousRunner { } this.syslog('✓ Time window: allowed'); + // 1.1 Disk-space hold — after the worktree sweep above, which is what frees + // space, and before anything is fetched or claimed. One notice per change of + // state, not one per heartbeat. (AGT-4785) + const disk = this.diskGate.evaluate(); + if (disk.notice) { + console.warn(`[AutonomousRunner] ${disk.notice}`); + await reportToDiscord(`${disk.pause ? '⚠️' : '✅'} ${disk.notice}`); + } + if (disk.pause) { + this.syslog(`⏸ Disk space hold: ${disk.line}`); + return; + } + // 1.2 Rate-limit hold — skip the heartbeat while a 429 pause is still active. // Cleared implicitly once the clock passes rateLimitUntil. (INT-1906) if (Date.now() < this.rateLimitUntil) { diff --git a/src/automation/runnerTypes.ts b/src/automation/runnerTypes.ts index b70cbb8f..fe465128 100644 --- a/src/automation/runnerTypes.ts +++ b/src/automation/runnerTypes.ts @@ -86,6 +86,8 @@ export interface AutonomousConfig { adapterRouting?: { primary?: import('../adapters/types.js').AdapterName; fallbacks?: Array<'cc-router' | 'cursor' | 'codex' | 'codex-responses'>; allowReasons?: Array<'quota' | 'infra' | 'capability'> }; /** OS fence for the worker's bash tool (AGT-4387). */ workerSandbox?: 'on' | 'off'; + /** Free-space floor (GiB) below which dispatch pauses; 0 switches the check off (AGT-4785). */ + minFreeDiskGiB?: number; /** Periodic read-only repository review jobs. */ periodicReviews?: Array<{ profile: 'permissions' | 'hygiene' | 'security' | 'review'; schedule: string; adapter?: 'codex' | 'cc-router' | 'cursor' }>; /** Explicit high-capability project supervisor. */ diff --git a/src/core/config.test.ts b/src/core/config.test.ts index 42098f54..4d2cd425 100644 --- a/src/core/config.test.ts +++ b/src/core/config.test.ts @@ -371,6 +371,22 @@ agents: expect(agents?.[1].goal).toBeUndefined(); }); + it('carries minFreeDiskGiB to the runtime config and defaults the floor to 10 GiB (AGT-4785)', () => { + vi.mocked(existsSync).mockReturnValue(true); + const load = (autonomous: Record) => { + vi.mocked(readFileSync).mockReturnValue(JSON.stringify({ + language: 'en', + linear: { apiKey: 'k', teamId: 't' }, + agents: [{ name: 'main', projectPath: '/p', enabled: true, paused: false }], + autonomous: { enabled: true, ...autonomous }, + })); + return loadConfig('/tmp/config.json').autonomous; + }; + expect(load({})?.minFreeDiskGiB).toBe(10); + expect(load({ minFreeDiskGiB: 25 })?.minFreeDiskGiB).toBe(25); + expect(load({ minFreeDiskGiB: 0 })?.minFreeDiskGiB).toBe(0); + }); + it('carries workerNetwork and a project setup list to the runtime config (AGT-4798)', () => { vi.mocked(existsSync).mockReturnValue(true); vi.mocked(readFileSync).mockReturnValue(JSON.stringify({ diff --git a/src/core/config.ts b/src/core/config.ts index 8a30ecb0..e4522e4f 100644 --- a/src/core/config.ts +++ b/src/core/config.ts @@ -481,6 +481,8 @@ const AutonomousConfigSchema = z.object({ workerSandbox: z.enum(['on', 'off']).default('on'), /** Accept delegated CLI adapters (claude/codex/cursor) for unattended work despite the missing fences (AGT-4444) */ allowDelegatedAdapters: z.boolean().default(false), + /** Free disk space (GiB) below which dispatch pauses; 0 switches the check off (AGT-4785) */ + minFreeDiskGiB: z.number().min(0).default(10), /** Network for the worker's shell: loopback only (default) or open (AGT-4798) */ workerNetwork: z.enum(['loopback', 'open']).default('loopback'), periodicReviews: z.array(z.object({ @@ -890,6 +892,7 @@ function transformConfig(raw: RawConfig): SwarmConfig { adapterRouting: raw.autonomous.adapterRouting, workerSandbox: raw.autonomous.workerSandbox, workerNetwork: raw.autonomous.workerNetwork, + minFreeDiskGiB: raw.autonomous.minFreeDiskGiB, allowDelegatedAdapters: raw.autonomous.allowDelegatedAdapters, periodicReviews: raw.autonomous.periodicReviews, orchestrator: raw.autonomous.orchestrator, diff --git a/src/core/service.test.ts b/src/core/service.test.ts index 68ec232e..232b02b3 100644 --- a/src/core/service.test.ts +++ b/src/core/service.test.ts @@ -299,6 +299,19 @@ describe('service', () => { ); }); + it('forwards workerSandbox and minFreeDiskGiB to the runner — the mapping dropped workerSandbox, so the fence was off while the log said on (AGT-4785)', async () => { + const { startAutonomous } = await import('../automation/autonomousRunner.js'); + + await startService({ + ...mockAutonomousConfig, + autonomous: { ...mockAutonomousConfig.autonomous, workerSandbox: 'on', minFreeDiskGiB: 7 }, + } as SwarmConfig); + + expect(vi.mocked(startAutonomous)).toHaveBeenCalledWith( + expect.objectContaining({ workerSandbox: 'on', minFreeDiskGiB: 7 }), + ); + }); + it('reapplies a persisted provider override on boot when it differs from config', async () => { const { readProviderOverride } = await import('./providerOverride.js'); const { setDefaultAdapter } = await import('../adapters/index.js'); diff --git a/src/core/service.ts b/src/core/service.ts index defe4940..e604cabd 100644 --- a/src/core/service.ts +++ b/src/core/service.ts @@ -394,6 +394,12 @@ async function startServiceLocked(config: SwarmConfig): Promise { coordinationBoardIssueId: config.autonomous.coordinationBoardIssueId, mcpPolicies: config.autonomous.mcpPolicies, adapterRouting: config.autonomous.adapterRouting, + // Same trap as AGT-4122/AGT-4404: the schema, the boot log below and the + // pipeline all read workerSandbox, but this hand-picked mapping did not + // pass it, so the runner saw undefined and the bash tool (which fences only + // on an explicit 'on') ran unfenced while the log said "sandbox: on". + workerSandbox: config.autonomous.workerSandbox, + minFreeDiskGiB: config.autonomous.minFreeDiskGiB, // AGT-4785 periodicReviews: config.autonomous.periodicReviews, orchestrator: config.autonomous.orchestrator, orchestratorSchedule: config.autonomous.orchestratorSchedule, diff --git a/src/core/types.ts b/src/core/types.ts index 49284e2a..c9036135 100644 --- a/src/core/types.ts +++ b/src/core/types.ts @@ -717,6 +717,12 @@ export type AutonomousStartupConfig = { * Dependency installs that need the network go in projectAgents[].setup. */ workerNetwork?: 'loopback' | 'open'; + /** + * Free disk space, in GiB, below which the heartbeat stops claiming new work + * (AGT-4785). Worktree cleanup keeps running. 0 switches the check off. + * Default 10. + */ + minFreeDiskGiB?: number; /** * Run unattended work on delegated CLI adapters (claude/codex/cursor) even * though OpenSwarm's publication fence, protected files and credential-free diff --git a/src/support/hostHealth.test.ts b/src/support/hostHealth.test.ts new file mode 100644 index 00000000..3257ad04 --- /dev/null +++ b/src/support/hostHealth.test.ts @@ -0,0 +1,180 @@ +// ============================================ +// OpenSwarm - Host health: stop dispatching when the disk is nearly full (AGT-4785) +// ============================================ + +import { describe, expect, it } from 'vitest'; +import { mkdtempSync, rmSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import path from 'node:path'; +import { + DEFAULT_MIN_FREE_BYTES, HostDiskGate, checkHostHealth, dispatchWritePaths, formatGiB, freeBytesFor, hostDiskVerdict, + isHostResourceError, lowestFreeBytes, +} from './hostHealth.js'; + +const GiB = 1024 ** 3; + +describe('hostDiskVerdict', () => { + it('is low strictly below the floor and ok at or above it', () => { + expect(hostDiskVerdict({ freeBytes: 2 * GiB, minFreeBytes: 10 * GiB })).toBe('low'); + expect(hostDiskVerdict({ freeBytes: 10 * GiB - 1, minFreeBytes: 10 * GiB })).toBe('low'); + expect(hostDiskVerdict({ freeBytes: 10 * GiB, minFreeBytes: 10 * GiB })).toBe('ok'); + expect(hostDiskVerdict({ freeBytes: 50 * GiB, minFreeBytes: 10 * GiB })).toBe('ok'); + }); + + it('a floor of zero never pauses, which is how the check is turned off', () => { + expect(hostDiskVerdict({ freeBytes: 0, minFreeBytes: 0 })).toBe('ok'); + }); +}); + +describe('freeBytesFor / lowestFreeBytes', () => { + it('measures a real directory', () => { + expect(freeBytesFor(tmpdir())).toBeGreaterThan(0); + }); + + it('throws for a path that does not exist, and the scan skips it instead', () => { + const missing = path.join(tmpdir(), 'osw-host-health-missing', 'nested'); + expect(() => freeBytesFor(missing)).toThrow(); + expect(lowestFreeBytes([missing])).toBeUndefined(); + }); + + it('returns the scarcest of several and names it, ignoring unmeasurable ones', () => { + const dir = mkdtempSync(path.join(tmpdir(), 'osw-host-health-')); + try { + const result = lowestFreeBytes([path.join(dir, 'nope'), dir, tmpdir(), dir]); + expect(result).toBeDefined(); + expect([dir, tmpdir()]).toContain(result!.dir); + // Live volumes change between two statfs calls (other processes write while the + // suite runs), so an exact match flakes; both paths are one volume here, so the + // reading must be within a small band of a fresh one. + expect(Math.abs(result!.freeBytes - freeBytesFor(dir))).toBeLessThan(1024 ** 3); + } finally { rmSync(dir, { recursive: true, force: true }); } + }); +}); + +describe('checkHostHealth', () => { + it('is low when the measured volume is under the floor', () => { + // No real volume has this much free space, so the real measurement is always under it. + const health = checkHostHealth([tmpdir()], Number.MAX_SAFE_INTEGER); + expect(health.verdict).toBe('low'); + expect(health.scarcest?.dir).toBe(tmpdir()); + }); + + it('is ok when the measured volume clears the floor', () => { + expect(checkHostHealth([tmpdir()], 1).verdict).toBe('ok'); + }); + + it('fails open: nothing measurable is never a reason to stop', () => { + const health = checkHostHealth([path.join(tmpdir(), 'osw-host-health-missing')], Number.MAX_SAFE_INTEGER); + expect(health).toEqual({ verdict: 'ok', minFreeBytes: Number.MAX_SAFE_INTEGER }); + }); + + it('defaults to a 10 GiB floor', () => { + expect(DEFAULT_MIN_FREE_BYTES).toBe(10 * GiB); + expect(checkHostHealth([tmpdir()]).minFreeBytes).toBe(10 * GiB); + }); +}); + +describe('isHostResourceError', () => { + it('recognises ENOSPC by code', () => { + expect(isHostResourceError(Object.assign(new Error('write failed'), { code: 'ENOSPC' }))).toBe(true); + expect(isHostResourceError({ code: 'ENOSPC' })).toBe(true); + }); + + it.each([ + 'ENOSPC: no space left on device, open \'/Users/x/.claude/pipeline-history.json.lock\'', + 'tester: git exited with code 128: error: copy-fd: write returned: No space left on device', + 'fatal: write error: No space left on device', + ])('recognises the text the incident printed: %s', (message) => { + expect(isHostResourceError(new Error(message))).toBe(true); + expect(isHostResourceError(message)).toBe(true); + }); + + it.each([ + 'Could not resolve host: api.github.com', + 'tests failed: 3 failed, 10 passed', + 'there is no space left for the label in the header', + 'EACCES: permission denied', + '', + ])('does not mistake another failure for a full disk: %s', (message) => { + expect(isHostResourceError(new Error(message))).toBe(false); + }); + + it('handles a non-error input', () => { + expect(isHostResourceError(undefined)).toBe(false); + expect(isHostResourceError(null)).toBe(false); + expect(isHostResourceError(42)).toBe(false); + }); +}); + +describe('dispatchWritePaths / formatGiB', () => { + it('always includes the temp dir the verification clones are made in', () => { + expect(dispatchWritePaths(['/p'], ['/state'])).toEqual(['/p', tmpdir(), '/state']); + expect(dispatchWritePaths([])).toEqual([tmpdir()]); + }); + + it('formats for the log line', () => { + expect(formatGiB(2 * GiB)).toBe('2.0 GiB'); + expect(formatGiB(1.25 * GiB)).toBe('1.3 GiB'); + }); +}); + +describe('HostDiskGate', () => { + /** A gate whose measured free space the test controls, through the floor: the real volume is fixed, the floor moves. */ + const gate = (floorBytes: { value: number }) => new HostDiskGate(() => [tmpdir()], () => floorBytes.value); + const OVER = Number.MAX_SAFE_INTEGER; // no volume has this much free: always low + const UNDER = 1; // any volume has this much: always ok + + it('passes while the disk is fine and says nothing', () => { + expect(gate({ value: UNDER }).evaluate()).toEqual({ pause: false }); + }); + + it('pauses on a low disk and speaks once, then stays quiet while it stays low', () => { + const floor = { value: OVER }; + const g = gate(floor); + const first = g.evaluate(); + expect(first).toMatchObject({ pause: true, notice: expect.stringContaining('Dispatch paused') }); + for (let i = 0; i < 10; i += 1) { + const next = g.evaluate(); + expect(next.pause).toBe(true); + expect(next.notice).toBeUndefined(); + } + }); + + it('says once that it recovered, and not again', () => { + const floor = { value: OVER }; + const g = gate(floor); + g.evaluate(); + floor.value = UNDER; + expect(g.evaluate()).toMatchObject({ pause: false, notice: expect.stringContaining('recovered') }); + expect(g.evaluate()).toEqual({ pause: false }); + }); + + it('announces a second episode after a recovery', () => { + const floor = { value: OVER }; + const g = gate(floor); + g.evaluate(); + floor.value = UNDER; g.evaluate(); + floor.value = OVER; + expect(g.evaluate().notice).toContain('Dispatch paused'); + }); + + it('a floor of zero switches it off, and says so if it was paused', () => { + const floor = { value: OVER }; + const g = gate(floor); + g.evaluate(); + floor.value = 0; + expect(g.evaluate()).toMatchObject({ pause: false, notice: expect.stringContaining('switched off') }); + expect(g.evaluate()).toEqual({ pause: false }); + }); + + it('never pauses when nothing can be measured (fails open)', () => { + const g = new HostDiskGate(() => [path.join(tmpdir(), 'osw-host-health-missing')], () => OVER); + expect(g.evaluate()).toEqual({ pause: false }); + }); + + it('names the scarcest volume and the floor in the pause line', () => { + const decision = gate({ value: OVER }).evaluate(); + expect(decision.pause && decision.line).toContain(tmpdir()); + expect(decision.pause && decision.line).toContain('below the'); + }); +}); diff --git a/src/support/hostHealth.ts b/src/support/hostHealth.ts new file mode 100644 index 00000000..f17ae289 --- /dev/null +++ b/src/support/hostHealth.ts @@ -0,0 +1,152 @@ +// ============================================ +// OpenSwarm - Host health: stop dispatching when the disk is nearly full (AGT-4785) +// ============================================ +// +// On 2026-10-20 the host ran out of disk. Tester and worker stages failed with +// ENOSPC while copying git packs into `$TMPDIR/.openswarm-verify-head-*`, most of +// the night's runs ended `infra_error`, and the daemon kept dispatching: every +// new run failed the same way. Nothing in the tree measured free space — +// `errorClassification` only labels ENOSPC "an environment condition, back off +// and retry", which is right for one run and wrong for a whole queue. +// +// This is the measurement and the verdict. The runner reads it once per +// heartbeat, after the worktree sweep (which is what frees space) and before any +// task is fetched or claimed, so a full disk stops new work and never stops the +// cleanup that would end the condition. +// +// Reads fail OPEN: a path that cannot be measured is skipped, and with nothing +// measurable the verdict is `ok`. A health probe that cannot run must not stop +// a daemon that was working. + +import { statfsSync } from 'node:fs'; +import { tmpdir } from 'node:os'; + +const GiB = 1024 ** 3; + +/** + * Free space below which dispatch pauses. Sized from the night it failed: one + * verify head clone is about 1 GB (measured on cgf-portal), each worker + * worktree about 0.5 GB, and verification runs on two slots next to up to eight + * workers — several GB can be wanted at once, and the incident showed trouble + * at 2 GB free. + */ +export const DEFAULT_MIN_FREE_BYTES = 10 * GiB; + +export type DiskVerdict = 'ok' | 'low'; + +/** `low` strictly below the floor, `ok` at or above it. */ +export function hostDiskVerdict(input: { freeBytes: number; minFreeBytes: number }): DiskVerdict { + return input.freeBytes < input.minFreeBytes ? 'low' : 'ok'; +} + +/** + * Bytes available to an unprivileged process on the volume holding `dir` + * (`bavail`, not `bfree`: the root-reserved blocks are not ours to use). + * Throws when the path cannot be measured; {@link lowestFreeBytes} handles that. + */ +export function freeBytesFor(dir: string): number { + const stats = statfsSync(dir); + return Number(stats.bavail) * Number(stats.bsize); +} + +/** + * The scarcest of several volumes, with the path that is scarcest. Paths that + * cannot be measured are skipped; `undefined` when none can. + */ +export function lowestFreeBytes(dirs: readonly string[]): { dir: string; freeBytes: number } | undefined { + let lowest: { dir: string; freeBytes: number } | undefined; + for (const dir of new Set(dirs)) { + try { + const freeBytes = freeBytesFor(dir); + if (!lowest || freeBytes < lowest.freeBytes) lowest = { dir, freeBytes }; + } catch { + // Not measurable (missing, no permission): not evidence of a full disk. + } + } + return lowest; +} + +/** The directories the daemon writes under: each project's tree, the temp dir verification clones into, and its own state. */ +export function dispatchWritePaths(projectPaths: readonly string[], extra: readonly string[] = []): string[] { + return [...projectPaths, tmpdir(), ...extra]; +} + +/** + * True for a failure that means the host is out of a resource rather than that + * the task was wrong: ENOSPC, by code or by the text git and node print. + * Deliberately narrow — "no space left for the label" in prose must not match, + * and neither must a network failure. + */ +export function isHostResourceError(error: unknown): boolean { + if (error && typeof error === 'object' && (error as { code?: unknown }).code === 'ENOSPC') return true; + const message = error instanceof Error ? error.message : typeof error === 'string' ? error : ''; + return /\bENOSPC\b|no space left on device/i.test(message); +} + +export interface HostHealth { + verdict: DiskVerdict; + /** The scarcest measured volume, when any could be measured. */ + scarcest?: { dir: string; freeBytes: number }; + minFreeBytes: number; +} + +/** Measure the volumes the daemon writes to and judge them against the floor. */ +export function checkHostHealth( + dirs: readonly string[], + minFreeBytes: number = DEFAULT_MIN_FREE_BYTES, +): HostHealth { + const scarcest = lowestFreeBytes(dirs); + if (!scarcest) return { verdict: 'ok', minFreeBytes }; + return { verdict: hostDiskVerdict({ freeBytes: scarcest.freeBytes, minFreeBytes }), scarcest, minFreeBytes }; +} + +/** `12.3 GiB`, for the one log line and the one alert. */ +export function formatGiB(bytes: number): string { + return `${(bytes / GiB).toFixed(1)} GiB`; +} + +export type DispatchGateDecision = + /** Disk is fine: carry on. */ + | { pause: false; notice?: string } + /** Disk is low: skip this heartbeat's claims. `notice` is set only on the first low reading. */ + | { pause: true; line: string; notice?: string }; + +/** + * One heartbeat's disk check, remembering the previous one so the operator hears + * about a *change*, once, instead of every five minutes while the disk is full. + * On 2026-10-20 the daemon produced a night of near-identical failures; a night + * of near-identical alerts would be the same noise at a different address. + */ +export class HostDiskGate { + private low = false; + + constructor( + private readonly paths: () => readonly string[], + private readonly minFreeBytes: () => number = () => DEFAULT_MIN_FREE_BYTES, + ) {} + + /** Measure now and decide. A floor of zero or less switches the check off. */ + evaluate(): DispatchGateDecision { + const floor = this.minFreeBytes(); + if (floor <= 0) { + const recovered = this.low; + this.low = false; + return recovered ? { pause: false, notice: 'Disk-space check switched off; dispatch resumes.' } : { pause: false }; + } + const health = checkHostHealth(this.paths(), floor); + if (health.verdict === 'ok') { + const recovered = this.low; + this.low = false; + if (!recovered) return { pause: false }; + const free = health.scarcest ? ` (${formatGiB(health.scarcest.freeBytes)} free on ${health.scarcest.dir})` : ''; + return { pause: false, notice: `Disk space recovered${free}; dispatch resumes.` }; + } + const scarcest = health.scarcest!; + const line = `${formatGiB(scarcest.freeBytes)} free on ${scarcest.dir}, below the ${formatGiB(floor)} floor`; + const first = !this.low; + this.low = true; + return first + ? { pause: true, line, notice: `Dispatch paused: ${line}. Worktree cleanup keeps running; new work waits until space is freed.` } + : { pause: true, line }; + } +}