diff --git a/CHANGELOG.md b/CHANGELOG.md index 325cc7b5..fdd0a861 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -18,6 +18,8 @@ ### Added +- **The daemon stops dispatching when the disk is nearly full (AGT-4785).** On 2026-10-20 the host ran out of disk: tester and worker stages failed with `ENOSPC` copying git packs into `$TMPDIR/.openswarm-verify-head-*`, most of the night's runs ended `infra_error`, and every new heartbeat claimed more work that could only fail the same way. `src/support/hostHealth.ts` measures free bytes (`statfs`, `bavail` — the root-reserved blocks are not ours) on every volume the daemon writes to (each allowed project, the temp dir) and the heartbeat pauses dispatch when the scarcest is under `autonomous.minFreeDiskGiB` (default 10; 0 switches the check off). The check sits after the worktree sweep, which is what frees space, so a full disk stops new work and never the cleanup that would end the condition. One notice on the way down and one on recovery — not one per heartbeat. Reads fail open: a path that cannot be measured is skipped, and with nothing measurable dispatch continues. `isHostResourceError` recognises `ENOSPC` by code or by the text git and node print. **Not done:** an explicit dispatch (`/api/work`, `exec --issue`) is not held by this gate, and a run that already failed with `ENOSPC` is still classified as a plain `infra_error`. + - **`projectAgents[].setup` — dependency install commands that run in the worktree before the worker (AGT-4798).** They run with the network open but inside the sandbox and without credentials, in order, stopping at the first failure; a failed setup ends the run as `infra_error` before any model call. Most projects need none. - **`openswarm exec --issue INT-123` deploys the worker pipeline for one Linear issue asynchronously and hands the run to an external orchestrator.** It resolves the issue, checks the repo's Linear project mapping and the deployable state, records an **exec hold**, and posts to the daemon's `/api/work` (falling back to a detached `work` child only when the daemon is unreachable — never on an HTTP refusal, so a 403 from `autonomous.allowedProjects` cannot be routed around), then prints `{runId, identifier, transport, deployed}` and returns. `exec status `, `exec await --timeout ` and `exec answer [--correlation hq-…] [--text …]` are the observation half: `await` blocks until the run elevates or finishes and prints the structured elevation JSON (state, park reason, branch/worktree/PR, pending questions with their class, and the exact next command), and `answer` writes the reply to the coordination board as `actorRole: 'orchestrator'` — the same delegated authority OpenSwarm's own supervisor answers with — resumes the ledger row through the exact-correlation park contract first, re-marks the hold, and redeploys. All four take `--json`. Reads use a read-only handle (`{ readonly: true }`, the `ledgerOverviewTool` pattern) so `status`/`await` never create a database, migrate a schema, or take a write lock on a live daemon's file. @@ -32,6 +34,7 @@ ### Fixed +- **`autonomous.workerSandbox` never reached the runner, so the worker's bash ran unfenced while the boot log said "Worker sandbox: on" (found while wiring AGT-4785).** The schema defaults it to `on` and the boot log reads it from the config object, but `core/service.ts` builds the runner's config by hand-picked mapping and did not carry the field; the runner then saw `undefined`, and the bash tool fences only on an explicit `on` (`tools.ts`, `execOptions?.sandbox === 'on'`; `pairPipelineTypes.ts`: "Unset = off"). The setup step defaults an unset value to `on`, so it was fenced while the worker's own shell was not. Every control that rides on the sandbox wrapper for the worker's shell — the loopback-only network (AGT-4798) and the write fence — was inactive in a daemon started through `service.ts`; the credential-free environment (AGT-4783) does not depend on it and was in effect. The same class as AGT-4122 and AGT-4404. The field is now mapped, and `service.test.ts` asserts it (removing the mapping makes that test fail). Not checked: whether the CLI entry points that build their own runner config carry the field. - **The test suite no longer writes into the operator's live `~/.openswarm/task-state.json`.** `vitest.setup.ts` redirected a dozen home-dir state paths per worker but missed `OPENSWARM_TASK_STATE_FILE`, so suites that built a real task-state store fell back to the default path and appended fixture rows (`issue-uuid`/"Serial task", `INT-1`) to the live file. It is now redirected per worker like the rest. ## 0.24.4 — 2026-09-29 diff --git a/config.example.yaml b/config.example.yaml index c9398acd..fb2788dc 100644 --- a/config.example.yaml +++ b/config.example.yaml @@ -272,6 +272,12 @@ autonomous: # that need the network go in projectAgents[].setup below. # workerNetwork: loopback + # Free-space floor, in GiB, for every volume the daemon writes to (each + # allowed project, the temp dir). Below it the heartbeat stops fetching and + # claiming work — worktree cleanup keeps running — and resumes once space is + # freed; one notice each way. 0 switches the check off. (AGT-4785) + # minFreeDiskGiB: 10 + # Delegated CLI adapters (claude, codex, cursor) own their tool loop, so # OpenSwarm's publication fence, protected files, role tool limits and the # credential-free sandboxed shell do not reach them (claude even runs with diff --git a/src/automation/autonomousRunner.hostHealth.test.ts b/src/automation/autonomousRunner.hostHealth.test.ts new file mode 100644 index 00000000..8f45945f --- /dev/null +++ b/src/automation/autonomousRunner.hostHealth.test.ts @@ -0,0 +1,104 @@ +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; +import { mkdtempSync, rmSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import type { TaskItem } from '../orchestration/decisionEngine.js'; +import type { ITaskSource } from './taskSource.js'; +import type { DurableRunCoordinator } from './durableRunCoordinator.js'; + +// AGT-4785: on 2026-10-20 the host ran out of disk and the daemon kept claiming work that could +// only fail with ENOSPC. HostDiskGate is unit-tested in support/hostHealth.test.ts; this proves +// the heartbeat consults it with the configured floor, before anything is fetched. + +vi.mock('../core/providerOverride.js', () => ({ writeProviderOverride: vi.fn() })); +vi.mock('../agents/stageModelResolver.js', () => ({ resolveAdapterDefaultModel: vi.fn(async () => 'model') })); +vi.mock('../memory/repoKnowledge.js', () => ({ + recordTaskOutcome: vi.fn(async () => {}), + promoteStagedMemories: vi.fn(async () => 0), +})); +vi.mock('../linear/projectUpdater.js', () => ({ updateProjectAfterTask: vi.fn(async () => {}) })); + +type InternalRunner = { + heartbeatParallel: ReturnType; + refreshKnowledgeGraphs(): void; + durableRuns: DurableRunCoordinator; +}; + +// No real volume has this much free space, so a floor this high is always "low". +const UNREACHABLE_FLOOR_GIB = 1e9; + +describe('heartbeat holds dispatch when the disk is nearly full (AGT-4785)', () => { + let root: string; + + beforeEach(() => { + vi.useFakeTimers(); + vi.setSystemTime(new Date('2026-07-21T16:00:00Z')); // 01:00 KST: work window allowed + root = mkdtempSync(join(tmpdir(), 'openswarm-runner-hosthealth-')); + vi.stubEnv('OPENSWARM_TASK_STATE_FILE', join(root, 'task-state.json')); + vi.stubEnv('OPENSWARM_RUNNER_TASK_STATE_FILE', join(root, 'runner-state.json')); + vi.stubEnv('OPENSWARM_RUNNER_REJECTION_STATE_FILE', join(root, 'rejections.json')); + vi.stubEnv('OPENSWARM_RUNNER_PIPELINE_HISTORY_FILE', join(root, 'history.json')); + vi.stubEnv('OPENSWARM_RUNNER_DECOMPOSITION_STATE_FILE', join(root, 'decomposition.json')); + }); + + afterEach(() => { + vi.useRealTimers(); + vi.unstubAllEnvs(); + vi.restoreAllMocks(); + rmSync(root, { recursive: true, force: true }); + vi.resetModules(); + }); + + async function setup(minFreeDiskGiB: number | undefined) { + const [{ AutonomousRunner }, execution] = await Promise.all([ + import('./autonomousRunner.js'), + import('./runnerExecution.js'), + ]); + const fetchTasks = vi.fn(async () => [] as TaskItem[]); + const source = { + kind: 'linear', fetchTasks, + lookupIssueState: vi.fn(async () => ({ ok: true as const, issue: { state: 'Todo', stateType: 'unstarted' } })), + updateState: vi.fn(async () => true), addComment: vi.fn(async () => {}), + createTask: vi.fn(), createSubIssue: vi.fn(), logPairStart: vi.fn(), + logPairComplete: vi.fn(), logBlocked: vi.fn(), logStuck: vi.fn(), unstick: vi.fn(), + logHalt: vi.fn(), markAsDecomposed: vi.fn(), + } as unknown as ITaskSource; + execution.setTaskSource(source); + const runner = new AutonomousRunner({ + linearTeamId: 'team', allowedProjects: [root], heartbeatSchedule: '0 * * * *', + autoExecute: true, dryRun: true, pairMode: true, maxConcurrentTasks: 4, + automationLedgerMode: 'primary', automationDbPath: join(root, 'hosthealth.db'), + ...(minFreeDiskGiB === undefined ? {} : { minFreeDiskGiB }), + }); + const internal = runner as unknown as InternalRunner; + internal.refreshKnowledgeGraphs = vi.fn(); + internal.heartbeatParallel = vi.fn(async () => {}); + return { runner, internal, fetchTasks }; + } + + it('fetches and claims nothing while free space is under the configured floor', async () => { + const { runner, internal, fetchTasks } = await setup(UNREACHABLE_FLOOR_GIB); + await runner.heartbeat(); + internal.durableRuns.close(); + expect(fetchTasks).not.toHaveBeenCalled(); + expect(internal.heartbeatParallel).not.toHaveBeenCalled(); + }, 60_000); + + it('says so once per change of state, not once per heartbeat', async () => { + const warn = vi.spyOn(console, 'warn').mockImplementation(() => {}); + const { runner, internal } = await setup(UNREACHABLE_FLOOR_GIB); + await runner.heartbeat(); + await runner.heartbeat(); + await runner.heartbeat(); + internal.durableRuns.close(); + const pauses = warn.mock.calls.filter((c) => String(c[0]).includes('Dispatch paused')); + expect(pauses).toHaveLength(1); + }, 60_000); + + it('a floor of 0 switches the check off and the heartbeat proceeds to fetch', async () => { + const { runner, internal, fetchTasks } = await setup(0); + await runner.heartbeat(); + internal.durableRuns.close(); + expect(fetchTasks).toHaveBeenCalled(); + }, 60_000); +}); diff --git a/src/automation/autonomousRunner.ts b/src/automation/autonomousRunner.ts index 39f6cacd..e67978f6 100644 --- a/src/automation/autonomousRunner.ts +++ b/src/automation/autonomousRunner.ts @@ -70,6 +70,7 @@ import { SANDBOX_OUTCOME_UNKNOWN_PARK_REASON } from '../sandboxExecutor/protocol import { INFRA_CIRCUIT_PARK_REASON } from './infraFailureCircuit.js'; import { listHeldIssueIds } from './execHold.js'; import { inputFingerprint, mayRedispatch } from './dispatchFingerprint.js'; +import { HostDiskGate, dispatchWritePaths } from '../support/hostHealth.js'; import { VERIFIED_WORKER_BLOCKER_PARK_REASON } from '../agents/pairPipelineTypes.js'; import { decideExplicitReadmission, OPERATOR_QUESTION_PARK_MARKER } from './explicitDispatchReadmission.js'; import { broadcastEvent, type SwarmStats } from '../core/eventHub.js'; @@ -533,6 +534,11 @@ export class AutonomousRunner { // Rate-limit hold: epoch ms until which all task execution is paused. // Set when any adapter returns a 429 / usage_limit_reached response (INT-1906). private rateLimitUntil = 0; + // Disk-space hold: new claims wait while the volumes we write to are nearly full (AGT-4785). + private readonly diskGate = new HostDiskGate( + () => dispatchWritePaths(this.config.allowedProjects.map((p) => normalizeProjectPath(p))), + () => (this.config.minFreeDiskGiB ?? 10) * 1024 ** 3, + ); // Issues whose Linear project can't be mapped to a local repo path. Recorded on // the first resolve failure so they aren't re-picked every heartbeat (which @@ -2838,6 +2844,19 @@ export class AutonomousRunner { } this.syslog('✓ Time window: allowed'); + // 1.1 Disk-space hold — after the worktree sweep above, which is what frees + // space, and before anything is fetched or claimed. One notice per change of + // state, not one per heartbeat. (AGT-4785) + const disk = this.diskGate.evaluate(); + if (disk.notice) { + console.warn(`[AutonomousRunner] ${disk.notice}`); + await reportToDiscord(`${disk.pause ? '⚠️' : '✅'} ${disk.notice}`); + } + if (disk.pause) { + this.syslog(`⏸ Disk space hold: ${disk.line}`); + return; + } + // 1.2 Rate-limit hold — skip the heartbeat while a 429 pause is still active. // Cleared implicitly once the clock passes rateLimitUntil. (INT-1906) if (Date.now() < this.rateLimitUntil) { diff --git a/src/automation/runnerTypes.ts b/src/automation/runnerTypes.ts index b70cbb8f..fe465128 100644 --- a/src/automation/runnerTypes.ts +++ b/src/automation/runnerTypes.ts @@ -86,6 +86,8 @@ export interface AutonomousConfig { adapterRouting?: { primary?: import('../adapters/types.js').AdapterName; fallbacks?: Array<'cc-router' | 'cursor' | 'codex' | 'codex-responses'>; allowReasons?: Array<'quota' | 'infra' | 'capability'> }; /** OS fence for the worker's bash tool (AGT-4387). */ workerSandbox?: 'on' | 'off'; + /** Free-space floor (GiB) below which dispatch pauses; 0 switches the check off (AGT-4785). */ + minFreeDiskGiB?: number; /** Periodic read-only repository review jobs. */ periodicReviews?: Array<{ profile: 'permissions' | 'hygiene' | 'security' | 'review'; schedule: string; adapter?: 'codex' | 'cc-router' | 'cursor' }>; /** Explicit high-capability project supervisor. */ diff --git a/src/core/config.test.ts b/src/core/config.test.ts index 42098f54..4d2cd425 100644 --- a/src/core/config.test.ts +++ b/src/core/config.test.ts @@ -371,6 +371,22 @@ agents: expect(agents?.[1].goal).toBeUndefined(); }); + it('carries minFreeDiskGiB to the runtime config and defaults the floor to 10 GiB (AGT-4785)', () => { + vi.mocked(existsSync).mockReturnValue(true); + const load = (autonomous: Record) => { + vi.mocked(readFileSync).mockReturnValue(JSON.stringify({ + language: 'en', + linear: { apiKey: 'k', teamId: 't' }, + agents: [{ name: 'main', projectPath: '/p', enabled: true, paused: false }], + autonomous: { enabled: true, ...autonomous }, + })); + return loadConfig('/tmp/config.json').autonomous; + }; + expect(load({})?.minFreeDiskGiB).toBe(10); + expect(load({ minFreeDiskGiB: 25 })?.minFreeDiskGiB).toBe(25); + expect(load({ minFreeDiskGiB: 0 })?.minFreeDiskGiB).toBe(0); + }); + it('carries workerNetwork and a project setup list to the runtime config (AGT-4798)', () => { vi.mocked(existsSync).mockReturnValue(true); vi.mocked(readFileSync).mockReturnValue(JSON.stringify({ diff --git a/src/core/config.ts b/src/core/config.ts index 8a30ecb0..e4522e4f 100644 --- a/src/core/config.ts +++ b/src/core/config.ts @@ -481,6 +481,8 @@ const AutonomousConfigSchema = z.object({ workerSandbox: z.enum(['on', 'off']).default('on'), /** Accept delegated CLI adapters (claude/codex/cursor) for unattended work despite the missing fences (AGT-4444) */ allowDelegatedAdapters: z.boolean().default(false), + /** Free disk space (GiB) below which dispatch pauses; 0 switches the check off (AGT-4785) */ + minFreeDiskGiB: z.number().min(0).default(10), /** Network for the worker's shell: loopback only (default) or open (AGT-4798) */ workerNetwork: z.enum(['loopback', 'open']).default('loopback'), periodicReviews: z.array(z.object({ @@ -890,6 +892,7 @@ function transformConfig(raw: RawConfig): SwarmConfig { adapterRouting: raw.autonomous.adapterRouting, workerSandbox: raw.autonomous.workerSandbox, workerNetwork: raw.autonomous.workerNetwork, + minFreeDiskGiB: raw.autonomous.minFreeDiskGiB, allowDelegatedAdapters: raw.autonomous.allowDelegatedAdapters, periodicReviews: raw.autonomous.periodicReviews, orchestrator: raw.autonomous.orchestrator, diff --git a/src/core/service.test.ts b/src/core/service.test.ts index 68ec232e..232b02b3 100644 --- a/src/core/service.test.ts +++ b/src/core/service.test.ts @@ -299,6 +299,19 @@ describe('service', () => { ); }); + it('forwards workerSandbox and minFreeDiskGiB to the runner — the mapping dropped workerSandbox, so the fence was off while the log said on (AGT-4785)', async () => { + const { startAutonomous } = await import('../automation/autonomousRunner.js'); + + await startService({ + ...mockAutonomousConfig, + autonomous: { ...mockAutonomousConfig.autonomous, workerSandbox: 'on', minFreeDiskGiB: 7 }, + } as SwarmConfig); + + expect(vi.mocked(startAutonomous)).toHaveBeenCalledWith( + expect.objectContaining({ workerSandbox: 'on', minFreeDiskGiB: 7 }), + ); + }); + it('reapplies a persisted provider override on boot when it differs from config', async () => { const { readProviderOverride } = await import('./providerOverride.js'); const { setDefaultAdapter } = await import('../adapters/index.js'); diff --git a/src/core/service.ts b/src/core/service.ts index defe4940..e604cabd 100644 --- a/src/core/service.ts +++ b/src/core/service.ts @@ -394,6 +394,12 @@ async function startServiceLocked(config: SwarmConfig): Promise { coordinationBoardIssueId: config.autonomous.coordinationBoardIssueId, mcpPolicies: config.autonomous.mcpPolicies, adapterRouting: config.autonomous.adapterRouting, + // Same trap as AGT-4122/AGT-4404: the schema, the boot log below and the + // pipeline all read workerSandbox, but this hand-picked mapping did not + // pass it, so the runner saw undefined and the bash tool (which fences only + // on an explicit 'on') ran unfenced while the log said "sandbox: on". + workerSandbox: config.autonomous.workerSandbox, + minFreeDiskGiB: config.autonomous.minFreeDiskGiB, // AGT-4785 periodicReviews: config.autonomous.periodicReviews, orchestrator: config.autonomous.orchestrator, orchestratorSchedule: config.autonomous.orchestratorSchedule, diff --git a/src/core/types.ts b/src/core/types.ts index 49284e2a..c9036135 100644 --- a/src/core/types.ts +++ b/src/core/types.ts @@ -717,6 +717,12 @@ export type AutonomousStartupConfig = { * Dependency installs that need the network go in projectAgents[].setup. */ workerNetwork?: 'loopback' | 'open'; + /** + * Free disk space, in GiB, below which the heartbeat stops claiming new work + * (AGT-4785). Worktree cleanup keeps running. 0 switches the check off. + * Default 10. + */ + minFreeDiskGiB?: number; /** * Run unattended work on delegated CLI adapters (claude/codex/cursor) even * though OpenSwarm's publication fence, protected files and credential-free diff --git a/src/support/hostHealth.test.ts b/src/support/hostHealth.test.ts new file mode 100644 index 00000000..3257ad04 --- /dev/null +++ b/src/support/hostHealth.test.ts @@ -0,0 +1,180 @@ +// ============================================ +// OpenSwarm - Host health: stop dispatching when the disk is nearly full (AGT-4785) +// ============================================ + +import { describe, expect, it } from 'vitest'; +import { mkdtempSync, rmSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import path from 'node:path'; +import { + DEFAULT_MIN_FREE_BYTES, HostDiskGate, checkHostHealth, dispatchWritePaths, formatGiB, freeBytesFor, hostDiskVerdict, + isHostResourceError, lowestFreeBytes, +} from './hostHealth.js'; + +const GiB = 1024 ** 3; + +describe('hostDiskVerdict', () => { + it('is low strictly below the floor and ok at or above it', () => { + expect(hostDiskVerdict({ freeBytes: 2 * GiB, minFreeBytes: 10 * GiB })).toBe('low'); + expect(hostDiskVerdict({ freeBytes: 10 * GiB - 1, minFreeBytes: 10 * GiB })).toBe('low'); + expect(hostDiskVerdict({ freeBytes: 10 * GiB, minFreeBytes: 10 * GiB })).toBe('ok'); + expect(hostDiskVerdict({ freeBytes: 50 * GiB, minFreeBytes: 10 * GiB })).toBe('ok'); + }); + + it('a floor of zero never pauses, which is how the check is turned off', () => { + expect(hostDiskVerdict({ freeBytes: 0, minFreeBytes: 0 })).toBe('ok'); + }); +}); + +describe('freeBytesFor / lowestFreeBytes', () => { + it('measures a real directory', () => { + expect(freeBytesFor(tmpdir())).toBeGreaterThan(0); + }); + + it('throws for a path that does not exist, and the scan skips it instead', () => { + const missing = path.join(tmpdir(), 'osw-host-health-missing', 'nested'); + expect(() => freeBytesFor(missing)).toThrow(); + expect(lowestFreeBytes([missing])).toBeUndefined(); + }); + + it('returns the scarcest of several and names it, ignoring unmeasurable ones', () => { + const dir = mkdtempSync(path.join(tmpdir(), 'osw-host-health-')); + try { + const result = lowestFreeBytes([path.join(dir, 'nope'), dir, tmpdir(), dir]); + expect(result).toBeDefined(); + expect([dir, tmpdir()]).toContain(result!.dir); + // Live volumes change between two statfs calls (other processes write while the + // suite runs), so an exact match flakes; both paths are one volume here, so the + // reading must be within a small band of a fresh one. + expect(Math.abs(result!.freeBytes - freeBytesFor(dir))).toBeLessThan(1024 ** 3); + } finally { rmSync(dir, { recursive: true, force: true }); } + }); +}); + +describe('checkHostHealth', () => { + it('is low when the measured volume is under the floor', () => { + // No real volume has this much free space, so the real measurement is always under it. + const health = checkHostHealth([tmpdir()], Number.MAX_SAFE_INTEGER); + expect(health.verdict).toBe('low'); + expect(health.scarcest?.dir).toBe(tmpdir()); + }); + + it('is ok when the measured volume clears the floor', () => { + expect(checkHostHealth([tmpdir()], 1).verdict).toBe('ok'); + }); + + it('fails open: nothing measurable is never a reason to stop', () => { + const health = checkHostHealth([path.join(tmpdir(), 'osw-host-health-missing')], Number.MAX_SAFE_INTEGER); + expect(health).toEqual({ verdict: 'ok', minFreeBytes: Number.MAX_SAFE_INTEGER }); + }); + + it('defaults to a 10 GiB floor', () => { + expect(DEFAULT_MIN_FREE_BYTES).toBe(10 * GiB); + expect(checkHostHealth([tmpdir()]).minFreeBytes).toBe(10 * GiB); + }); +}); + +describe('isHostResourceError', () => { + it('recognises ENOSPC by code', () => { + expect(isHostResourceError(Object.assign(new Error('write failed'), { code: 'ENOSPC' }))).toBe(true); + expect(isHostResourceError({ code: 'ENOSPC' })).toBe(true); + }); + + it.each([ + 'ENOSPC: no space left on device, open \'/Users/x/.claude/pipeline-history.json.lock\'', + 'tester: git exited with code 128: error: copy-fd: write returned: No space left on device', + 'fatal: write error: No space left on device', + ])('recognises the text the incident printed: %s', (message) => { + expect(isHostResourceError(new Error(message))).toBe(true); + expect(isHostResourceError(message)).toBe(true); + }); + + it.each([ + 'Could not resolve host: api.github.com', + 'tests failed: 3 failed, 10 passed', + 'there is no space left for the label in the header', + 'EACCES: permission denied', + '', + ])('does not mistake another failure for a full disk: %s', (message) => { + expect(isHostResourceError(new Error(message))).toBe(false); + }); + + it('handles a non-error input', () => { + expect(isHostResourceError(undefined)).toBe(false); + expect(isHostResourceError(null)).toBe(false); + expect(isHostResourceError(42)).toBe(false); + }); +}); + +describe('dispatchWritePaths / formatGiB', () => { + it('always includes the temp dir the verification clones are made in', () => { + expect(dispatchWritePaths(['/p'], ['/state'])).toEqual(['/p', tmpdir(), '/state']); + expect(dispatchWritePaths([])).toEqual([tmpdir()]); + }); + + it('formats for the log line', () => { + expect(formatGiB(2 * GiB)).toBe('2.0 GiB'); + expect(formatGiB(1.25 * GiB)).toBe('1.3 GiB'); + }); +}); + +describe('HostDiskGate', () => { + /** A gate whose measured free space the test controls, through the floor: the real volume is fixed, the floor moves. */ + const gate = (floorBytes: { value: number }) => new HostDiskGate(() => [tmpdir()], () => floorBytes.value); + const OVER = Number.MAX_SAFE_INTEGER; // no volume has this much free: always low + const UNDER = 1; // any volume has this much: always ok + + it('passes while the disk is fine and says nothing', () => { + expect(gate({ value: UNDER }).evaluate()).toEqual({ pause: false }); + }); + + it('pauses on a low disk and speaks once, then stays quiet while it stays low', () => { + const floor = { value: OVER }; + const g = gate(floor); + const first = g.evaluate(); + expect(first).toMatchObject({ pause: true, notice: expect.stringContaining('Dispatch paused') }); + for (let i = 0; i < 10; i += 1) { + const next = g.evaluate(); + expect(next.pause).toBe(true); + expect(next.notice).toBeUndefined(); + } + }); + + it('says once that it recovered, and not again', () => { + const floor = { value: OVER }; + const g = gate(floor); + g.evaluate(); + floor.value = UNDER; + expect(g.evaluate()).toMatchObject({ pause: false, notice: expect.stringContaining('recovered') }); + expect(g.evaluate()).toEqual({ pause: false }); + }); + + it('announces a second episode after a recovery', () => { + const floor = { value: OVER }; + const g = gate(floor); + g.evaluate(); + floor.value = UNDER; g.evaluate(); + floor.value = OVER; + expect(g.evaluate().notice).toContain('Dispatch paused'); + }); + + it('a floor of zero switches it off, and says so if it was paused', () => { + const floor = { value: OVER }; + const g = gate(floor); + g.evaluate(); + floor.value = 0; + expect(g.evaluate()).toMatchObject({ pause: false, notice: expect.stringContaining('switched off') }); + expect(g.evaluate()).toEqual({ pause: false }); + }); + + it('never pauses when nothing can be measured (fails open)', () => { + const g = new HostDiskGate(() => [path.join(tmpdir(), 'osw-host-health-missing')], () => OVER); + expect(g.evaluate()).toEqual({ pause: false }); + }); + + it('names the scarcest volume and the floor in the pause line', () => { + const decision = gate({ value: OVER }).evaluate(); + expect(decision.pause && decision.line).toContain(tmpdir()); + expect(decision.pause && decision.line).toContain('below the'); + }); +}); diff --git a/src/support/hostHealth.ts b/src/support/hostHealth.ts new file mode 100644 index 00000000..f17ae289 --- /dev/null +++ b/src/support/hostHealth.ts @@ -0,0 +1,152 @@ +// ============================================ +// OpenSwarm - Host health: stop dispatching when the disk is nearly full (AGT-4785) +// ============================================ +// +// On 2026-10-20 the host ran out of disk. Tester and worker stages failed with +// ENOSPC while copying git packs into `$TMPDIR/.openswarm-verify-head-*`, most of +// the night's runs ended `infra_error`, and the daemon kept dispatching: every +// new run failed the same way. Nothing in the tree measured free space — +// `errorClassification` only labels ENOSPC "an environment condition, back off +// and retry", which is right for one run and wrong for a whole queue. +// +// This is the measurement and the verdict. The runner reads it once per +// heartbeat, after the worktree sweep (which is what frees space) and before any +// task is fetched or claimed, so a full disk stops new work and never stops the +// cleanup that would end the condition. +// +// Reads fail OPEN: a path that cannot be measured is skipped, and with nothing +// measurable the verdict is `ok`. A health probe that cannot run must not stop +// a daemon that was working. + +import { statfsSync } from 'node:fs'; +import { tmpdir } from 'node:os'; + +const GiB = 1024 ** 3; + +/** + * Free space below which dispatch pauses. Sized from the night it failed: one + * verify head clone is about 1 GB (measured on cgf-portal), each worker + * worktree about 0.5 GB, and verification runs on two slots next to up to eight + * workers — several GB can be wanted at once, and the incident showed trouble + * at 2 GB free. + */ +export const DEFAULT_MIN_FREE_BYTES = 10 * GiB; + +export type DiskVerdict = 'ok' | 'low'; + +/** `low` strictly below the floor, `ok` at or above it. */ +export function hostDiskVerdict(input: { freeBytes: number; minFreeBytes: number }): DiskVerdict { + return input.freeBytes < input.minFreeBytes ? 'low' : 'ok'; +} + +/** + * Bytes available to an unprivileged process on the volume holding `dir` + * (`bavail`, not `bfree`: the root-reserved blocks are not ours to use). + * Throws when the path cannot be measured; {@link lowestFreeBytes} handles that. + */ +export function freeBytesFor(dir: string): number { + const stats = statfsSync(dir); + return Number(stats.bavail) * Number(stats.bsize); +} + +/** + * The scarcest of several volumes, with the path that is scarcest. Paths that + * cannot be measured are skipped; `undefined` when none can. + */ +export function lowestFreeBytes(dirs: readonly string[]): { dir: string; freeBytes: number } | undefined { + let lowest: { dir: string; freeBytes: number } | undefined; + for (const dir of new Set(dirs)) { + try { + const freeBytes = freeBytesFor(dir); + if (!lowest || freeBytes < lowest.freeBytes) lowest = { dir, freeBytes }; + } catch { + // Not measurable (missing, no permission): not evidence of a full disk. + } + } + return lowest; +} + +/** The directories the daemon writes under: each project's tree, the temp dir verification clones into, and its own state. */ +export function dispatchWritePaths(projectPaths: readonly string[], extra: readonly string[] = []): string[] { + return [...projectPaths, tmpdir(), ...extra]; +} + +/** + * True for a failure that means the host is out of a resource rather than that + * the task was wrong: ENOSPC, by code or by the text git and node print. + * Deliberately narrow — "no space left for the label" in prose must not match, + * and neither must a network failure. + */ +export function isHostResourceError(error: unknown): boolean { + if (error && typeof error === 'object' && (error as { code?: unknown }).code === 'ENOSPC') return true; + const message = error instanceof Error ? error.message : typeof error === 'string' ? error : ''; + return /\bENOSPC\b|no space left on device/i.test(message); +} + +export interface HostHealth { + verdict: DiskVerdict; + /** The scarcest measured volume, when any could be measured. */ + scarcest?: { dir: string; freeBytes: number }; + minFreeBytes: number; +} + +/** Measure the volumes the daemon writes to and judge them against the floor. */ +export function checkHostHealth( + dirs: readonly string[], + minFreeBytes: number = DEFAULT_MIN_FREE_BYTES, +): HostHealth { + const scarcest = lowestFreeBytes(dirs); + if (!scarcest) return { verdict: 'ok', minFreeBytes }; + return { verdict: hostDiskVerdict({ freeBytes: scarcest.freeBytes, minFreeBytes }), scarcest, minFreeBytes }; +} + +/** `12.3 GiB`, for the one log line and the one alert. */ +export function formatGiB(bytes: number): string { + return `${(bytes / GiB).toFixed(1)} GiB`; +} + +export type DispatchGateDecision = + /** Disk is fine: carry on. */ + | { pause: false; notice?: string } + /** Disk is low: skip this heartbeat's claims. `notice` is set only on the first low reading. */ + | { pause: true; line: string; notice?: string }; + +/** + * One heartbeat's disk check, remembering the previous one so the operator hears + * about a *change*, once, instead of every five minutes while the disk is full. + * On 2026-10-20 the daemon produced a night of near-identical failures; a night + * of near-identical alerts would be the same noise at a different address. + */ +export class HostDiskGate { + private low = false; + + constructor( + private readonly paths: () => readonly string[], + private readonly minFreeBytes: () => number = () => DEFAULT_MIN_FREE_BYTES, + ) {} + + /** Measure now and decide. A floor of zero or less switches the check off. */ + evaluate(): DispatchGateDecision { + const floor = this.minFreeBytes(); + if (floor <= 0) { + const recovered = this.low; + this.low = false; + return recovered ? { pause: false, notice: 'Disk-space check switched off; dispatch resumes.' } : { pause: false }; + } + const health = checkHostHealth(this.paths(), floor); + if (health.verdict === 'ok') { + const recovered = this.low; + this.low = false; + if (!recovered) return { pause: false }; + const free = health.scarcest ? ` (${formatGiB(health.scarcest.freeBytes)} free on ${health.scarcest.dir})` : ''; + return { pause: false, notice: `Disk space recovered${free}; dispatch resumes.` }; + } + const scarcest = health.scarcest!; + const line = `${formatGiB(scarcest.freeBytes)} free on ${scarcest.dir}, below the ${formatGiB(floor)} floor`; + const first = !this.low; + this.low = true; + return first + ? { pause: true, line, notice: `Dispatch paused: ${line}. Worktree cleanup keeps running; new work waits until space is freed.` } + : { pause: true, line }; + } +}