From 46036a06342de452751b1a2c6c5a71d2b02da1ba Mon Sep 17 00:00:00 2001 From: GaganSD Date: Mon, 2 Nov 2026 18:23:21 +0530 Subject: [PATCH 01/15] insights: transactional bounded scans with cache-first UI Replace whole-store clones with a precise per-file mutation journal, add bounded foreground slices with plain-file offsets, incremental worker checkpoints, generation fencing, and a committed presentation cache. --- herdr-insights/mods/insights/collector.mjs | 670 +++++++++++++++--- herdr-insights/mods/insights/core/cache.mjs | 71 ++ herdr-insights/mods/insights/core/fence.mjs | 139 ++++ .../mods/insights/core/indexer-control.mjs | 139 ++++ .../mods/insights/core/progress.mjs | 34 + herdr-insights/mods/insights/core/scan.mjs | 610 +++++++++++----- herdr-insights/mods/insights/core/store.mjs | 324 ++++++++- herdr-insights/mods/insights/core/util.mjs | 13 +- herdr-insights/mods/insights/indexer.mjs | 278 ++++++++ herdr-insights/mods/insights/main.js | 57 +- herdr-insights/mods/insights/presenter.mjs | 33 +- .../herdr-insights/collector-bounded.test.mjs | 253 +++++++ .../collector-transactions.test.mjs | 191 +++++ 13 files changed, 2512 insertions(+), 300 deletions(-) create mode 100644 herdr-insights/mods/insights/core/cache.mjs create mode 100644 herdr-insights/mods/insights/core/fence.mjs create mode 100644 herdr-insights/mods/insights/core/indexer-control.mjs create mode 100644 herdr-insights/mods/insights/core/progress.mjs create mode 100644 herdr-insights/mods/insights/indexer.mjs create mode 100644 tests/herdr-insights/collector-bounded.test.mjs create mode 100644 tests/herdr-insights/collector-transactions.test.mjs diff --git a/herdr-insights/mods/insights/collector.mjs b/herdr-insights/mods/insights/collector.mjs index 2d003bc8..e29f1882 100644 --- a/herdr-insights/mods/insights/collector.mjs +++ b/herdr-insights/mods/insights/collector.mjs @@ -1,24 +1,54 @@ #!/usr/bin/env node -import { randomUUID } from "node:crypto"; -import { realpathSync, renameSync, unlinkSync, writeFileSync } from "node:fs"; +import { realpathSync } from "node:fs"; +import { stat } from "node:fs/promises"; import os from "node:os"; import path from "node:path"; import { fileURLToPath, pathToFileURL } from "node:url"; +import { readPresentationCache } from "./core/cache.mjs"; import { migrateLoadedConfig, normalizeConfig, sourcesSignature } from "./core/config.mjs"; +import { + bumpFence, + readEffectiveFence, + restoreFence, +} from "./core/fence.mjs"; +import { + DEFAULT_WORKER_ACK_MS, + ensureIndexer, + waitForIndexerAck, +} from "./core/indexer-control.mjs"; import { scanSources } from "./core/scan.mjs"; +import { progressWriter } from "./core/progress.mjs"; import { clearCollectedData, + emptyScanState, + ensureStoreMeta, loadStore, saveStore, snapshotRows, + storeGeneration, } from "./core/store.mjs"; -import { withStoreLock } from "./core/util.mjs"; +import { + isLockBusyError, + readStoreLockInfo, + sleep, + withStoreLock, +} from "./core/util.mjs"; + +export { progressWriter }; + +// Leave room for startup, store load, save, and presentation derivation inside +// Herdr's fixed 30s exec budget. A partial slice returns committed data and the +// UI continues automatically. +export const DEFAULT_SCAN_BUDGET_MS = 15_000; +const FOREGROUND_LOCK_WAIT_MS = 2_000; +const CONFIGURE_LOCK_WAIT_MS = 20_000; function usage() { return [ "Usage:", " collector.mjs snapshot --store [--home ] [--rebuild]", " collector.mjs configure --store [--home ] --config [--rebuild]", + " collector.mjs status --store ", ].join("\n"); } @@ -49,77 +79,465 @@ function parseArgs(argv) { return { command, options }; } -async function buildSnapshot(store, scan, nowIso, loadWarnings = [], loadErrors = 0) { - const warnings = [...loadWarnings, ...scan.warnings]; +function emptyStats() { + return { + files: 0, + filesRead: 0, + bytesRead: 0, + unchangedFiles: 0, + errors: 0, + decompressedBytes: 0, + }; +} + +function scanView(scan = {}) { + const stats = scan.stats && typeof scan.stats === "object" ? scan.stats : emptyStats(); + return { + status: typeof scan.status === "string" ? scan.status : "idle", + generation: Number.isInteger(scan.generation) ? scan.generation : 0, + startedAt: scan.startedAt ?? null, + updatedAt: scan.updatedAt ?? null, + completedAt: scan.completedAt ?? null, + processed: Number.isInteger(scan.processed) ? scan.processed : 0, + total: Number.isInteger(scan.total) ? scan.total : 0, + files: stats.files ?? 0, + filesRead: stats.filesRead ?? 0, + bytesRead: stats.bytesRead ?? 0, + decompressedBytes: stats.decompressedBytes ?? 0, + errors: stats.errors ?? 0, + currentFile: scan.currentFile ?? null, + pendingFile: scan.pendingFile ?? null, + workerPid: Number.isInteger(scan.workerPid) ? scan.workerPid : null, + error: scan.error ?? null, + }; +} + +function scanResultFromStore(store) { + ensureStoreMeta(store); + return { + stats: store.scan.stats ?? emptyStats(), + sourceStats: store.scan.sourceStats ?? {}, + warnings: Array.isArray(store.scan.warnings) ? store.scan.warnings : [], + }; +} + +function buildSnapshot(store, scan, nowIso, loadWarnings = [], loadErrors = 0) { + ensureStoreMeta(store); + const warnings = [...loadWarnings, ...(scan?.warnings ?? [])]; const stats = { - ...scan.stats, - errors: scan.stats.errors + loadErrors, + ...emptyStats(), + ...(scan?.stats ?? {}), + errors: (scan?.stats?.errors ?? 0) + loadErrors, }; return { version: 1, generatedAt: nowIso, - lastScanAt: nowIso, + lastScanAt: store.lastScanAt ?? null, + lastCompletedScanAt: store.lastCompletedScanAt ?? null, + savedAt: store.lastScanAt ?? null, config: store.config, rows: snapshotRows(store), stats, - sourceStats: scan.sourceStats ?? undefined, + sourceStats: scan?.sourceStats ?? undefined, + scan: scanView(store.scan), warnings, }; } +function buildSnapshotFromCache(cached, nowIso) { + const scan = cached.scan ?? {}; + const stats = { ...emptyStats(), ...(scan.stats ?? {}) }; + return { + version: 1, + generatedAt: nowIso, + lastScanAt: cached.lastScanAt ?? null, + lastCompletedScanAt: cached.lastCompletedScanAt ?? null, + savedAt: cached.savedAt ?? cached.lastScanAt ?? null, + config: cached.config ?? {}, + rows: Array.isArray(cached.rows) ? cached.rows : [], + stats, + sourceStats: scan.sourceStats ?? undefined, + scan: scanView(scan), + warnings: Array.isArray(scan.warnings) ? scan.warnings : [], + }; +} + +function busyOverrides(owner) { + if (!owner) return {}; + const worker = owner?.role === "worker"; + return { + status: worker ? "running" : "partial", + workerPid: worker && Number.isInteger(owner.pid) ? owner.pid : null, + }; +} + /** - * Best-effort scan progress sidecar next to the store. The native view polls - * it only while a refresh is running; it is removed when the scan settles. + * Committed presentation for a reader. Never takes the writer lock, so a long + * indexer cannot make status/prepare wait for a scan. The cache is used only + * when it is at least as new as the committed store. */ -export function progressWriter(storePath) { - const progressPath = `${storePath}.progress.json`; - const tempPath = `${progressPath}.${process.pid}-${randomUUID()}.tmp`; - let lastWrite = 0; - return { - update(state) { - const now = Date.now(); - if (now - lastWrite < 250 && state.processed < state.total) return; - lastWrite = now; - try { - // Atomic replacement also avoids following an existing sidecar link. - writeFileSync(tempPath, `${JSON.stringify(state)}\n`, { mode: 0o600, flag: "wx" }); - renameSync(tempPath, progressPath); - } catch { /* progress is best-effort */ } - finally { - try { unlinkSync(tempPath); } catch { /* renamed or not created */ } +export async function readCommittedSnapshot(storePath, { now = Date.now } = {}) { + const nowIso = new Date(now()).toISOString(); + const cached = await readPresentationCache(storePath); + if (cached) return buildSnapshotFromCache(cached, nowIso); + const loaded = await loadStore(storePath, { readOnly: true }); + return buildSnapshot( + loaded.store, + scanResultFromStore(loaded.store), + nowIso, + loaded.warnings, + loaded.errors, + ); +} + +async function busySnapshot(storePath, owner, now) { + const snapshot = await readCommittedSnapshot(storePath, { now }); + const overrides = busyOverrides(owner); + snapshot.scan = { ...snapshot.scan, ...overrides }; + return snapshot; +} + +async function generationBaseline(storePath) { + try { + const cached = await readPresentationCache(storePath); + if (cached) return cached.generation; + } catch { /* fall through to a committed read */ } + try { + const loaded = await loadStore(storePath, { readOnly: true }); + return storeGeneration(loaded.store); + } catch { + return 0; + } +} + +/** + * Attempt the writer lock without a 20s wait. A live long-lived worker makes a + * collector return committed data immediately; ordinary foreground slices get + * a short coalescing wait. Configure/rebuild mode fences and signals a worker. + */ +async function runWithStoreLock( + storePath, + callback, + { + waitMs = 0, + meta = {}, + signalWorker = false, + fenceGeneration = null, + } = {}, +) { + const started = Date.now(); + let signaled = false; + for (;;) { + try { + const value = await withStoreLock(storePath, callback, { timeoutMs: 0, meta }); + return { acquired: true, value }; + } catch (error) { + if (!isLockBusyError(error)) throw error; + const owner = await readStoreLockInfo(storePath); + if (owner?.role === "worker") { + if ( + signalWorker + && !signaled + && Number.isInteger(owner.pid) + && owner.pid > 0 + && (fenceGeneration === null || Number(owner.generation ?? 0) < fenceGeneration) + ) { + // Only the pid recorded by the current writer lease is signalled. + try { process.kill(owner.pid, "SIGTERM"); } catch { /* already exiting */ } + signaled = true; + } + if (!signalWorker) return { acquired: false, owner }; } - }, - clear() { - try { - unlinkSync(progressPath); - } catch { /* already gone */ } - }, - }; + if (Date.now() - started >= waitMs) return { acquired: false, owner }; + await sleep(40); + } + } } -async function snapshotWithStore(storePath, home, { rebuild = false } = {}) { - return withStoreLock(storePath, async () => { +async function removeStalePendingFile(storePath, store, now) { + const pending = store.scan?.pendingFile; + if (!pending?.path) return false; + let info = null; + try { + info = await stat(pending.path); + } catch (error) { + if (error?.code !== "ENOENT") return false; + } + if (info?.isFile()) return false; + store.scan.pendingFile = null; + store.scan.updatedAt = new Date(now()).toISOString(); + return true; +} + +function updateScanFromResult(store, result, { nowIso, previousStatus }) { + const scan = store.scan; + const cycleStart = ( + !scan.startedAt + || previousStatus === "complete" + || previousStatus === "failed" + || previousStatus === "idle" + ) ? nowIso : scan.startedAt; + scan.startedAt = cycleStart; + scan.updatedAt = nowIso; + scan.generation = store.generation; + scan.processed = result.processed; + scan.total = result.total; + scan.stats = result.stats; + scan.sourceStats = result.sourceStats; + scan.warnings = result.warnings; + scan.currentFile = null; + scan.workerPid = null; + scan.error = null; + if (result.abortedFile) { + scan.status = "partial"; + scan.pendingFile = { ...result.abortedFile, requestedAt: nowIso }; + } else { + scan.pendingFile = null; + if (result.complete) { + scan.status = "complete"; + scan.completedAt = nowIso; + } else { + scan.status = "partial"; + } + } + return scan; +} + +async function scanAndPersist( + storePath, + home, + store, + { + loaded, + dirty, + budgetMs, + now, + testStreamDelayMs, + mode = "collector", + guard = null, + }, +) { + const previousStatus = store.scan?.status ?? "idle"; + const previousPending = store.scan?.pendingFile ?? null; + const startedAt = now(); + const deadline = startedAt + Math.max(0, Number(budgetMs) || 0); + const progress = progressWriter(storePath); + let result; + try { + result = await scanSources(store, { + home, + deadline, + now, + mode, + streamDelayMs: testStreamDelayMs, + shouldStop: () => false, + onProgress: (state) => progress.update({ ...state, workerPid: null }), + onStreamProgress: (state) => progress.update({ + ...state, + processed: 0, + total: 1, + mode, + workerPid: null, + }), + }); + } finally { + progress.clear(); + } + + const finishedIso = new Date(now()).toISOString(); + updateScanFromResult(store, result, { nowIso: finishedIso, previousStatus }); + const pendingChanged = (store.scan.pendingFile?.path ?? null) !== (previousPending?.path ?? null); + const completed = Boolean(result.complete); + const statusChanged = store.scan.status !== previousStatus; + let mustSave = Boolean( + dirty + || result.dirty + || result.stopped + || statusChanged + || pendingChanged + || loaded?.dirty + ); + if (completed && !mustSave && store.lastCompletedScanAt === null) mustSave = true; + + let worker; + const guarded = guard ? await guard() : null; + if (guarded) return { fenced: true }; + if (result.abortedFile) { + // Persist the abandoned file marker before the worker starts so a restart + // promotes the same file instead of repeating a foreground prefix. + await saveStore(storePath, store, { status: "partial", now: finishedIso }); + worker = await ensureIndexer(storePath, home, { + generation: store.generation, + ackMs: 0, + testStreamDelayMs, + }); + if (worker.pid) { + store.scan.workerPid = worker.pid; + store.scan.status = "running"; + } + const snapshot = buildSnapshot(store, result, finishedIso, loaded?.warnings, loaded?.errors); + return { snapshot, worker }; + } + + if (mustSave) { + if (completed) store.lastCompletedScanAt = finishedIso; + await saveStore(storePath, store, { + completed, + status: completed ? "complete" : "partial", + now: finishedIso, + }); + } else if (completed) { + // No-op refresh: verification succeeded without changing committed data, + // so keep the committed timestamps and do not rewrite the store. + store.lastCompletedScanAt = loaded?.store?.lastCompletedScanAt ?? store.lastCompletedScanAt; + } + const snapshot = buildSnapshot(store, result, finishedIso, loaded?.warnings, loaded?.errors); + return { snapshot, worker: null }; +} + +async function finishWorkerAck(snapshot, worker, storePath, workerAckMs) { + if (!worker?.pid || !worker.spawned) return snapshot; + const ack = await waitForIndexerAck(storePath, worker.claim, { + timeoutMs: workerAckMs ?? DEFAULT_WORKER_ACK_MS, + }); + if (ack.ok) { + snapshot.scan = { ...snapshot.scan, status: "running", workerPid: worker.pid }; + return snapshot; + } + snapshot.warnings = [ + ...(snapshot.warnings ?? []), + "The background indexer did not acknowledge startup; the next refresh retries it.", + ]; + snapshot.scan = { ...snapshot.scan, status: "partial", workerPid: null }; + return snapshot; +} + +async function snapshotWithStore( + storePath, + home, + { + rebuild = false, + budgetMs = DEFAULT_SCAN_BUDGET_MS, + now = Date.now, + lockWaitMs, + workerAckMs = DEFAULT_WORKER_ACK_MS, + testStreamDelayMs = 0, + } = {}, +) { + let baseline = null; + let fence = null; + if (rebuild) { + baseline = await generationBaseline(storePath); + fence = await bumpFence(storePath, baseline); + } + const lock = await runWithStoreLock(storePath, async () => { const loaded = await loadStore(storePath); const store = loaded.store; let dirty = loaded.dirty; + const fenceNow = await readEffectiveFence(storePath, { now: now() }); if (rebuild) { + store.generation = Math.max(storeGeneration(store) + 1, fence?.generation ?? 0); clearCollectedData(store); + store.scan = emptyScanState(); + store.scan.generation = store.generation; + store.scan.startedAt = new Date(now()).toISOString(); + store.lastCompletedScanAt = null; dirty = true; + } else if (fenceNow && !fenceNow.stale && fenceNow.generation > store.generation) { + const nowIso = new Date(now()).toISOString(); + const snapshot = buildSnapshot( + store, + scanResultFromStore(store), + nowIso, + loaded.warnings, + loaded.errors, + ); + snapshot.scan = { ...snapshot.scan, status: "reconfiguring" }; + return { snapshot, worker: null, fenced: true }; } - const progress = progressWriter(storePath); - let scan; - try { - scan = await scanSources(store, { home, onProgress: (state) => progress.update(state) }); - } finally { - progress.clear(); + + if (await removeStalePendingFile(storePath, store, now)) dirty = true; + if (store.scan.pendingFile) { + const worker = await ensureIndexer(storePath, home, { + generation: store.generation, + ackMs: 0, + testStreamDelayMs, + }); + if (worker.pid) { + store.scan.workerPid = worker.pid; + store.scan.status = "running"; + } + const nowIso = new Date(now()).toISOString(); + if (dirty || worker.pid) { + await saveStore(storePath, store, { status: "running", now: nowIso }); + } + const snapshot = buildSnapshot( + store, + scanResultFromStore(store), + nowIso, + loaded.warnings, + loaded.errors, + ); + snapshot.scan = { ...snapshot.scan, status: worker.pid ? "running" : "partial" }; + return { snapshot, worker, fenced: false }; + } + + const scanned = await scanAndPersist(storePath, home, store, { + loaded, + dirty, + budgetMs, + now, + testStreamDelayMs, + mode: "foreground", + guard: async () => { + const fenceAfter = await readEffectiveFence(storePath, { now: now() }); + return Boolean(fenceAfter && !fenceAfter.stale && fenceAfter.generation > store.generation); + }, + }); + if (scanned.fenced) { + // A configure/rebuild is waiting. Discard this slice rather than leak an + // old-generation write; the committed store is still authoritative. + const committed = await loadStore(storePath, { readOnly: true }); + const nowIso = new Date(now()).toISOString(); + const snapshot = buildSnapshot( + committed.store, + scanResultFromStore(committed.store), + nowIso, + loaded.warnings, + loaded.errors, + ); + snapshot.scan = { ...snapshot.scan, status: "reconfiguring" }; + return { snapshot, worker: null, fenced: true }; } - if (dirty || scan.dirty) await saveStore(storePath, store); - const now = new Date().toISOString(); - return buildSnapshot(store, scan, now, loaded.warnings, loaded.errors); + return { ...scanned, fenced: false }; + }, { + waitMs: lockWaitMs ?? (rebuild ? CONFIGURE_LOCK_WAIT_MS : FOREGROUND_LOCK_WAIT_MS), + meta: { role: "collector", generation: fence?.generation ?? null }, + signalWorker: rebuild, + fenceGeneration: fence?.generation ?? null, }); + + if (!lock.acquired) { + if (rebuild && fence) await restoreFence(storePath, baseline); + return busySnapshot(storePath, lock.owner, now); + } + const { snapshot, worker } = lock.value; + return finishWorkerAck(snapshot, worker, storePath, workerAckMs); } -async function configureStore(storePath, home, rawConfig, { rebuild = false } = {}) { +async function configureStore( + storePath, + home, + rawConfig, + { + rebuild = false, + budgetMs = DEFAULT_SCAN_BUDGET_MS, + now = Date.now, + lockWaitMs = CONFIGURE_LOCK_WAIT_MS, + workerAckMs = DEFAULT_WORKER_ACK_MS, + testStreamDelayMs = 0, + } = {}, +) { let config; try { const parsed = typeof rawConfig === "string" ? JSON.parse(rawConfig) : rawConfig; @@ -130,40 +548,118 @@ async function configureStore(storePath, home, rawConfig, { rebuild = false } = throw invalid; } - return withStoreLock(storePath, async () => { - const loaded = await loadStore(storePath); - const store = loaded.store; - let dirty = loaded.dirty; - const configChanged = JSON.stringify(store.config) !== JSON.stringify(config); - const mappingsChanged = - sourcesSignature(store.config) !== sourcesSignature(config); - if (configChanged) { - store.config = config; - dirty = true; - } - if (mappingsChanged || rebuild) { - clearCollectedData(store); - dirty = true; - } + const baseline = await generationBaseline(storePath); + const fence = await bumpFence(storePath, baseline); + let applied = false; + try { + const lock = await runWithStoreLock(storePath, async () => { + const loaded = await loadStore(storePath); + const store = loaded.store; + let dirty = loaded.dirty; + store.generation = Math.max(storeGeneration(store) + 1, fence.generation); + const previousStatus = store.scan.status; + const configChanged = JSON.stringify(store.config) !== JSON.stringify(config); + const mappingsChanged = sourcesSignature(store.config) !== sourcesSignature(config); + const cleared = mappingsChanged || rebuild; + if (cleared) { + clearCollectedData(store); + store.scan = emptyScanState(); + store.scan.generation = store.generation; + store.scan.startedAt = new Date(now()).toISOString(); + store.lastCompletedScanAt = null; + dirty = true; + } + if (configChanged) { + store.config = config; + dirty = true; + } + applied = true; + // Persist accepted config first. Config acceptance is never reported as a + // completed scan; retained committed data keeps its completed timestamp. + const acceptedComplete = !cleared && previousStatus === "complete"; + const acceptedIso = new Date(now()).toISOString(); + if (dirty) { + store.scan.generation = store.generation; + store.scan.updatedAt = acceptedIso; + await saveStore(storePath, store, { + completed: false, + status: acceptedComplete ? "complete" : "partial", + now: acceptedIso, + }); + dirty = false; + } - // Persist the accepted config before scanning. If the scan fails, the UI - // still has the newly configured sources and can show the scan error. - if (dirty) { - await saveStore(storePath, store); - dirty = false; - } + if (await removeStalePendingFile(storePath, store, now)) dirty = true; + if (store.scan.pendingFile) { + const worker = await ensureIndexer(storePath, home, { + generation: store.generation, + ackMs: 0, + testStreamDelayMs, + }); + if (worker.pid) { + store.scan.workerPid = worker.pid; + store.scan.status = "running"; + } + const nowIso = new Date(now()).toISOString(); + if (dirty || worker.pid) { + await saveStore(storePath, store, { status: "running", now: nowIso }); + } + const snapshot = buildSnapshot( + store, + scanResultFromStore(store), + nowIso, + loaded.warnings, + loaded.errors, + ); + snapshot.scan = { ...snapshot.scan, status: worker.pid ? "running" : "partial" }; + return { snapshot, worker }; + } - const progress = progressWriter(storePath); - let scan; - try { - scan = await scanSources(store, { home, onProgress: (state) => progress.update(state) }); - } finally { - progress.clear(); + const scanned = await scanAndPersist(storePath, home, store, { + loaded, + dirty, + budgetMs, + now, + testStreamDelayMs, + mode: "configure", + guard: async () => { + const fenceAfter = await readEffectiveFence(storePath, { now: now() }); + return Boolean(fenceAfter && !fenceAfter.stale && fenceAfter.generation > store.generation); + }, + }); + if (scanned.fenced) { + // A newer configure is already queued; its generation fence owns the + // next commit. Return committed data without writing this generation. + const committed = await loadStore(storePath, { readOnly: true }); + const nowIso = new Date(now()).toISOString(); + const snapshot = buildSnapshot( + committed.store, + scanResultFromStore(committed.store), + nowIso, + loaded.warnings, + loaded.errors, + ); + snapshot.scan = { ...snapshot.scan, status: "reconfiguring" }; + return { snapshot, worker: null }; + } + return scanned; + }, { + waitMs: lockWaitMs, + meta: { role: "configure", generation: fence.generation }, + signalWorker: true, + fenceGeneration: fence.generation, + }); + + if (!lock.acquired) { + if (!applied) await restoreFence(storePath, baseline); + throw new Error("insights store is busy; configuration was not applied"); } - if (dirty || scan.dirty) await saveStore(storePath, store); - const now = new Date().toISOString(); - return buildSnapshot(store, scan, now, loaded.warnings, loaded.errors); - }); + const { snapshot, worker } = lock.value; + return finishWorkerAck(snapshot, worker, storePath, workerAckMs); + } catch (error) { + if (!applied) await restoreFence(storePath, baseline).catch(() => {}); + throw error; + } } export async function run(argv) { @@ -171,7 +667,7 @@ export async function run(argv) { if (options.help || !command) { return { exitCode: 0, stdout: "", stderr: `${usage()}\n` }; } - if (command !== "snapshot" && command !== "configure") { + if (!["snapshot", "configure", "status"].includes(command)) { return { exitCode: 2, stdout: "", stderr: `unknown command ${command}\n${usage()}\n` }; } if (!options.store) { @@ -190,14 +686,26 @@ export async function run(argv) { /** In-process API: the presenter never serializes and reparses raw rows. */ export async function collectSnapshot(command, options) { - if (!["snapshot", "configure"].includes(command)) throw new Error("unknown collector command"); + if (!["snapshot", "configure", "status"].includes(command)) throw new Error("unknown collector command"); if (!options?.store) throw new Error("--store is required"); if (command === "configure" && options.config === undefined) throw new Error("--config is required"); const storePath = path.resolve(options.store); const home = path.resolve(options.home ?? os.homedir()); + const now = typeof options.now === "function" ? options.now : Date.now; + const shared = { + budgetMs: options.budgetMs, + now, + lockWaitMs: options.lockWaitMs, + workerAckMs: options.workerAckMs, + testStreamDelayMs: options.testStreamDelayMs, + }; + if (command === "status") { + const owner = await readStoreLockInfo(storePath); + return busySnapshot(storePath, owner, now); + } return command === "snapshot" - ? snapshotWithStore(storePath, home, { rebuild: options.rebuild }) - : configureStore(storePath, home, options.config, { rebuild: options.rebuild }); + ? snapshotWithStore(storePath, home, { ...shared, rebuild: options.rebuild }) + : configureStore(storePath, home, options.config, { ...shared, rebuild: options.rebuild }); } async function main() { diff --git a/herdr-insights/mods/insights/core/cache.mjs b/herdr-insights/mods/insights/core/cache.mjs new file mode 100644 index 00000000..ba606a76 --- /dev/null +++ b/herdr-insights/mods/insights/core/cache.mjs @@ -0,0 +1,71 @@ +import { readFile, stat } from "node:fs/promises"; +import { atomicWriteJson, isPlainObject } from "./util.mjs"; + +// The dashboard and status paths must stay fast while a long indexer holds the +// writer lock. This sidecar is a compact, metadata-only copy of the committed +// aggregate rows plus the truthful scan state. It is always derived from the +// committed store and is never read as a source for mutation. +const CACHE_VERSION = 1; +const STALE_MTIME_MS = 2; + +export function presentationCachePath(storePath) { + return `${storePath}.cache.json`; +} + +export async function writePresentationCache(storePath, { savedAt, store, rows }) { + const scan = isPlainObject(store?.scan) ? store.scan : {}; + const value = { + version: CACHE_VERSION, + savedAt: savedAt ?? store?.lastScanAt ?? null, + lastScanAt: store?.lastScanAt ?? null, + lastCompletedScanAt: store?.lastCompletedScanAt ?? null, + generation: Number.isInteger(store?.generation) ? store.generation : 0, + config: store?.config ?? {}, + rows: Array.isArray(rows) ? rows : [], + scan, + }; + await atomicWriteJson(presentationCachePath(storePath), value); + return value; +} + +function validCache(value) { + return Boolean( + isPlainObject(value) + && value.version === CACHE_VERSION + && isPlainObject(value.config) + && Array.isArray(value.rows) + && isPlainObject(value.scan) + && (value.lastScanAt === null || typeof value.lastScanAt === "string") + && (value.lastCompletedScanAt === null || typeof value.lastCompletedScanAt === "string") + && Number.isInteger(value.generation) + && value.generation >= 0, + ); +} + +/** + * Returns the cached presentation data only when it is at least as new as the + * committed store file. A missing, corrupt, or older cache falls back to the + * full store so prepare never invents data. + */ +export async function readPresentationCache(storePath) { + try { + const [storeStat, cacheStat] = await Promise.all([ + stat(storePath), + stat(presentationCachePath(storePath)), + ]); + if (!storeStat.isFile() || !cacheStat.isFile()) return null; + if (cacheStat.mtimeMs + STALE_MTIME_MS < storeStat.mtimeMs) return null; + const value = JSON.parse(await readFile(presentationCachePath(storePath), "utf8")); + if (!validCache(value)) return null; + return value; + } catch { + return null; + } +} + +export async function removePresentationCache(storePath) { + const { unlink } = await import("node:fs/promises"); + await unlink(presentationCachePath(storePath)).catch((error) => { + if (error?.code !== "ENOENT") throw error; + }); +} diff --git a/herdr-insights/mods/insights/core/fence.mjs b/herdr-insights/mods/insights/core/fence.mjs new file mode 100644 index 00000000..39a4eaf7 --- /dev/null +++ b/herdr-insights/mods/insights/core/fence.mjs @@ -0,0 +1,139 @@ +import { unlink } from "node:fs/promises"; +import os from "node:os"; +import { + atomicWriteJson, + isPlainObject, + isProcessAlive, + readJsonFile, +} from "./util.mjs"; + +// A generation fence is written BEFORE a configure/rebuild waits for the store +// lock. Any worker spawned for an older generation must stop and must not +// commit. The fence is advisory metadata next to the store (no transcript text) +// and is always written atomically with 0600 permissions. +const FENCE_VERSION = 1; +// A configure/rebuild holds its in-process work for seconds, never minutes. +// Only a provably dead local writer (or a long-abandoned fence) is ignored. +const FENCE_ABANDONED_MS = 10 * 60 * 1000; +// A claim may exist for a few milliseconds before the spawned pid is recorded. +export const WORKER_CLAIM_STARTUP_MS = 5_000; + +export function fencePath(storePath) { + return `${storePath}.generation.json`; +} + +export function workerClaimPath(storePath) { + return `${storePath}.worker.json`; +} + +export function storeGeneration(store) { + const value = Number(store?.generation); + return Number.isInteger(value) && value >= 0 ? value : 0; +} + +export function nextGeneration(...values) { + let highest = 0; + for (const value of values) { + const number = Number(value); + if (Number.isInteger(number) && number > highest) highest = number; + } + return highest + 1; +} + +export async function readFence(storePath) { + try { + const value = await readJsonFile(fencePath(storePath)); + if (!isPlainObject(value)) return null; + if (value.version !== FENCE_VERSION) return null; + if (!Number.isInteger(value.generation) || value.generation < 1) return null; + return value; + } catch { + return null; + } +} + +/** + * A fence from a crashed configure must not block indexing forever. Ignore it + * only when its local owner is provably dead or it is far older than any real + * configure. Never kill the pid here; the lock remains the sole writer gate. + */ +export async function readEffectiveFence(storePath, { now = Date.now() } = {}) { + const fence = await readFence(storePath); + if (!fence) return null; + const age = now - Number(fence.at ?? 0); + const sameHost = fence.host === os.hostname(); + const dead = sameHost && Number.isInteger(fence.pid) && !isProcessAlive(fence.pid); + const abandoned = Number.isFinite(age) && age > FENCE_ABANDONED_MS; + return { ...fence, stale: dead || abandoned }; +} + +export async function writeFence(storePath, generation) { + const value = { + version: FENCE_VERSION, + generation, + pid: process.pid, + host: os.hostname(), + at: Date.now(), + }; + await atomicWriteJson(fencePath(storePath), value); + return value; +} + +/** Raise the fence above both the committed generation and any prior fence. */ +export async function bumpFence(storePath, committedGeneration = 0) { + const current = await readFence(storePath); + const generation = nextGeneration(committedGeneration, current?.generation ?? 0); + return writeFence(storePath, generation); +} + +/** Best-effort undo for a configure that failed before it committed. */ +export async function restoreFence(storePath, generation) { + const current = await readFence(storePath); + if (!current || current.pid !== process.pid) return false; + if (current.generation <= generation) return false; + await writeFence(storePath, generation); + return true; +} + +export async function readWorkerClaim(storePath) { + try { + const value = await readJsonFile(workerClaimPath(storePath)); + if (!isPlainObject(value)) return null; + if (typeof value.token !== "string" || value.token.length === 0) return null; + if (!Number.isInteger(value.generation) || value.generation < 0) return null; + return value; + } catch { + return null; + } +} + +export async function writeWorkerClaim(storePath, claim) { + await atomicWriteJson(workerClaimPath(storePath), claim); + return claim; +} + +export async function clearWorkerClaim(storePath, token) { + const current = await readWorkerClaim(storePath); + if (!current || current.token !== token) return false; + try { + await unlink(workerClaimPath(storePath)); + return true; + } catch (error) { + if (error?.code === "ENOENT") return false; + throw error; + } +} + +/** + * A claim is live while its recorded pid is alive. A claim without a pid is + * only a spawn in progress; it expires quickly so a crashed launcher cannot + * pin the store. A claim from another host fails closed (never spawn a second + * long-lived writer against a possibly shared directory). + */ +export function workerClaimActive(claim, { now = Date.now() } = {}) { + if (!claim) return false; + if (claim.host !== os.hostname()) return true; + if (Number.isInteger(claim.pid) && claim.pid > 0) return isProcessAlive(claim.pid); + const age = now - Number(claim.at ?? claim.startedAt ?? 0); + return Number.isFinite(age) && age < WORKER_CLAIM_STARTUP_MS; +} diff --git a/herdr-insights/mods/insights/core/indexer-control.mjs b/herdr-insights/mods/insights/core/indexer-control.mjs new file mode 100644 index 00000000..c57272d4 --- /dev/null +++ b/herdr-insights/mods/insights/core/indexer-control.mjs @@ -0,0 +1,139 @@ +import { spawn } from "node:child_process"; +import { randomBytes } from "node:crypto"; +import os from "node:os"; +import { fileURLToPath } from "node:url"; +import { + clearWorkerClaim, + readEffectiveFence, + readWorkerClaim, + storeGeneration, + workerClaimActive, + writeWorkerClaim, +} from "./fence.mjs"; +import { isProcessAlive, readStoreLockInfo, sleep } from "./util.mjs"; + +export const DEFAULT_WORKER_ACK_MS = 1_500; + +export function indexerPath() { + return fileURLToPath(new URL("../indexer.mjs", import.meta.url)); +} + +function claimToken() { + return `${process.pid}-${Date.now()}-${randomBytes(8).toString("hex")}`; +} + +/** + * Ensure one detached local indexer is responsible for the store. The store + * writer lock remains the authority; this claim only coalesces spawns so + * several panes do not create several long-lived workers. A live claim is + * never evicted; only a provably dead local pid (or an unfinished spawn) is. + */ +export async function ensureIndexer( + storePath, + home, + { + generation = null, + ackMs = 0, + testStreamDelayMs = 0, + testWorkerPath = null, + } = {}, +) { + const existing = await readWorkerClaim(storePath); + if (workerClaimActive(existing)) { + return { spawned: false, pid: Number.isInteger(existing.pid) ? existing.pid : null, claim: existing }; + } + if (existing) { + // A live claim (including a foreign host) is handled above. Only a + // provably dead local pid or an expired spawn placeholder reaches here. + await clearWorkerClaim(storePath, existing.token); + } + + const claim = { + version: 1, + token: claimToken(), + pid: null, + host: os.hostname(), + generation: Number.isInteger(generation) ? generation : 0, + startedAt: Date.now(), + at: Date.now(), + }; + // The worker waits briefly for its pid to be recorded, so writing the token + // before spawn makes concurrent launchers coalesce even during spawn. + await writeWorkerClaim(storePath, claim); + + const args = [ + testWorkerPath ?? indexerPath(), + "--store", storePath, + "--home", home, + "--generation", String(claim.generation), + "--claim-token", claim.token, + ]; + if (Number.isFinite(testStreamDelayMs) && testStreamDelayMs > 0) { + args.push("--test-stream-delay-ms", String(testStreamDelayMs)); + } + + let child; + try { + child = spawn(process.execPath, args, { + detached: true, + stdio: "ignore", + shell: false, + env: { ...process.env }, + }); + } catch (error) { + await clearWorkerClaim(storePath, claim.token); + return { spawned: false, pid: null, error }; + } + // A detached child's process handle must not keep this presenter alive. + child.unref(); + child.on("error", () => { /* startup failure is observed through the claim */ }); + + const recorded = { ...claim, pid: child.pid, at: Date.now() }; + try { + await writeWorkerClaim(storePath, recorded); + } catch (error) { + await clearWorkerClaim(storePath, claim.token); + return { spawned: false, pid: null, error }; + } + + if (ackMs > 0) { + const ack = await waitForIndexerAck(storePath, recorded, { timeoutMs: ackMs }); + return { ...ack, spawned: true, pid: child.pid, claim: recorded }; + } + return { spawned: true, pid: child.pid, claim: recorded, alive: isProcessAlive(child.pid) }; +} + +/** Bounded acknowledgement that the spawned worker became the lock owner. */ +export async function waitForIndexerAck(storePath, claim, { timeoutMs = DEFAULT_WORKER_ACK_MS } = {}) { + const deadline = Date.now() + Math.max(0, timeoutMs); + for (;;) { + const owner = await readStoreLockInfo(storePath); + if (owner && Number.isInteger(claim?.pid) && owner.pid === claim.pid) { + return { ok: true, pid: claim.pid, owner }; + } + if (Number.isInteger(claim?.pid) && claim.pid > 0 && !isProcessAlive(claim.pid)) { + const fence = await readEffectiveFence(storePath); + return { ok: false, pid: claim.pid, exited: true, fence }; + } + if (Date.now() >= deadline) { + return { ok: false, pid: claim?.pid ?? null, timedOut: true }; + } + await sleep(40); + } +} + +export async function indexerGeneration(storePath) { + const fence = await readEffectiveFence(storePath); + return { + fence, + generation: fence && !fence.stale ? fence.generation : null, + }; +} + +export function indexerScriptPath() { + return indexerPath(); +} + +export function storeGenerationOrZero(store) { + return storeGeneration(store); +} diff --git a/herdr-insights/mods/insights/core/progress.mjs b/herdr-insights/mods/insights/core/progress.mjs new file mode 100644 index 00000000..be37ce7a --- /dev/null +++ b/herdr-insights/mods/insights/core/progress.mjs @@ -0,0 +1,34 @@ +import { randomUUID } from "node:crypto"; +import { renameSync, unlinkSync, writeFileSync } from "node:fs"; + +/** + * Best-effort scan progress sidecar next to the store. The native view polls + * it while a refresh or background indexer is running; it is removed when the + * writer settles. It contains paths and counters only, never transcript text. + */ +export function progressWriter(storePath) { + const progressPath = `${storePath}.progress.json`; + const tempPath = `${progressPath}.${process.pid}-${randomUUID()}.tmp`; + let lastWrite = 0; + return { + update(state) { + const now = Date.now(); + const done = Number(state?.processed) >= Number(state?.total) && Number(state?.total) > 0; + if (now - lastWrite < 250 && !done) return; + lastWrite = now; + try { + // Atomic replacement also avoids following an existing sidecar link. + writeFileSync(tempPath, `${JSON.stringify(state)}\n`, { mode: 0o600, flag: "wx" }); + renameSync(tempPath, progressPath); + } catch { /* progress is best-effort */ } + finally { + try { unlinkSync(tempPath); } catch { /* renamed or not created */ } + } + }, + clear() { + try { + unlinkSync(progressPath); + } catch { /* already gone */ } + }, + }; +} diff --git a/herdr-insights/mods/insights/core/scan.mjs b/herdr-insights/mods/insights/core/scan.mjs index 86c29b59..85e6dee0 100644 --- a/herdr-insights/mods/insights/core/scan.mjs +++ b/herdr-insights/mods/insights/core/scan.mjs @@ -14,12 +14,14 @@ import { import { addTask, addUsage, + beginStoreTransaction, + commitStoreTransaction, dedupKeyFor, fileKey, newFileEntry, - prepareStore, removeLatestFallbackTaskForSession, resetFileEntry, + rollbackStoreTransaction, taskKeyFor, } from "./store.mjs"; @@ -27,6 +29,34 @@ const CHUNK_BYTES = 256 * 1024; const MAX_WARNINGS = 50; const MAX_TRAVERSAL_ENTRIES = 250_000; const PARSER_VERSIONS = { pi: 3, codex: 5, claude: 2 }; +const PROGRESS_CHUNK_INTERVAL = 64; + +export class ScanStopError extends Error { + constructor(reason) { + super(`scan stopped (${reason})`); + this.name = "ScanStopError"; + this.reason = reason; + } +} + +function nowFn(options) { + return typeof options?.now === "function" ? options.now : Date.now; +} + +function deadlineReached(options, now = nowFn(options)()) { + const deadline = options?.deadline; + return Number.isFinite(deadline) && now >= deadline; +} + +function stopRequested(options) { + if (typeof options?.shouldStop !== "function") return false; + return Boolean(options.shouldStop()); +} + +function pausableDelay(ms) { + if (!Number.isFinite(ms) || ms <= 0) return null; + return () => new Promise((resolve) => setTimeout(resolve, ms)); +} function warningSink(warnings, stats, sourceState) { let hidden = 0; @@ -286,12 +316,18 @@ function createLineConsumer(onLine) { }; } -async function streamRange(filePath, start, end, onChunk) { +async function streamRange(filePath, start, end, onChunk, { shouldStop } = {}) { const handle = await open(filePath, "r"); let position = start; let bytes = 0; + let stopped = false; + let first = true; try { while (position < end) { + if (!first && shouldStop?.()) { + stopped = true; + break; + } const toRead = Math.min(CHUNK_BYTES, end - position); const buffer = Buffer.allocUnsafe(toRead); const { bytesRead } = await handle.read(buffer, 0, toRead, position); @@ -299,11 +335,12 @@ async function streamRange(filePath, start, end, onChunk) { position += bytesRead; bytes += bytesRead; await onChunk(buffer.subarray(0, bytesRead), position - bytesRead); + first = false; } } finally { await handle.close(); } - return bytes; + return { bytes, stopped, position }; } function hasZstdSupport() { @@ -348,6 +385,7 @@ async function parseBuffers( startOffset, endOffset, sink, + options = {}, ) { const context = lineContext(store, entry, source, sink); const consumer = createLineConsumer((record, info) => { @@ -369,10 +407,28 @@ async function parseBuffers( }); consumer.offset = startOffset; consumer.lineNo = entry.lineNo ?? 0; + const now = nowFn(options); + const delay = pausableDelay(options.streamDelayMs); let chunkStart = startOffset; + let chunksSeen = 0; + let bytes = 0; for await (const chunk of chunks) { + if (chunksSeen > 0) { + if (stopRequested(options)) throw new ScanStopError("stop"); + if (deadlineReached(options, now())) throw new ScanStopError("budget"); + } + if (delay) await delay(); consumer.feedAt(chunk, chunkStart); chunkStart += chunk.length; + bytes += chunk.length; + chunksSeen += 1; + if (chunksSeen % PROGRESS_CHUNK_INTERVAL === 0) { + options.onStreamProgress?.({ + file: entry.path, + bytes, + phase: "decompress", + }); + } } consumer.finish(endOffset ?? chunkStart); if (consumer.skippedLargeLine) { @@ -380,7 +436,7 @@ async function parseBuffers( `${source.adapter} ${entry.path}: a line larger than ${MAX_LINE_BYTES} bytes was skipped.`, ); } - return { offset: consumer.offset, lineNo: consumer.lineNo }; + return { offset: consumer.offset, lineNo: consumer.lineNo, bytes }; } async function processStringChunks( @@ -391,8 +447,9 @@ async function processStringChunks( startOffset, endOffset, sink, + options = {}, ) { - return parseBuffers(store, entry, source, chunks, startOffset, endOffset, sink); + return parseBuffers(store, entry, source, chunks, startOffset, endOffset, sink, options); } async function scanPlainFile( @@ -403,63 +460,74 @@ async function scanPlainFile( fileStat, sink, stats, + options = {}, ) { const adapter = source.adapter; - let dirty = false; const start = entry.offset; - if (fileStat.size > start) { - const context = lineContext(store, entry, source, sink); - const consumer = createLineConsumer((record, info) => { - if (info.parseError) { - sink.warn( - `${adapter} ${entry.path}: line ${info.lineNo} is not valid JSON; skipped.`, - ); - return; - } - if (!isPlainObject(record)) return; - context.lineHash = info.hash; - try { - parseAdapterRecord(adapter, record, context); - } catch (error) { - sink.warn( - `${adapter} ${entry.path}: line ${info.lineNo} could not be parsed: ${error.message}`, - ); - } - }); - consumer.offset = start; - consumer.lineNo = entry.lineNo ?? 0; - consumer.discarding = entry.state.discardingLargeLine; - let bytes = 0; + if (fileStat.size <= start) { + if (entry.state.scanPending) { + delete entry.state.scanPending; + return { dirty: true, pending: false }; + } + return { dirty: false, pending: false }; + } + + const context = lineContext(store, entry, source, sink); + const consumer = createLineConsumer((record, info) => { + if (info.parseError) { + sink.warn( + `${adapter} ${entry.path}: line ${info.lineNo} is not valid JSON; skipped.`, + ); + return; + } + if (!isPlainObject(record)) return; + context.lineHash = info.hash; try { - await streamRange(filePath, start, fileStat.size, async (chunk, chunkStart) => { - bytes += chunk.length; - consumer.feedAt(chunk, chunkStart); - }); - consumer.finish(start + bytes); - delete entry.state.readFailed; + parseAdapterRecord(adapter, record, context); } catch (error) { - // Keep the accepted complete prefix and its parser state together. The - // unfinished suffix is retried even if the file metadata is unchanged. - entry.state.readFailed = true; - sink.warn(`${adapter} ${entry.path}: read failed; accepted prefix retained: ${error.message}`); - } - stats.bytesRead += bytes; - if (bytes > 0) stats.filesRead += 1; - if (consumer.skippedLargeLine) { sink.warn( - `${adapter} ${entry.path}: a line larger than ${MAX_LINE_BYTES} bytes was skipped.`, + `${adapter} ${entry.path}: line ${info.lineNo} could not be parsed: ${error.message}`, ); } - entry.offset = consumer.offset; - entry.lineNo = consumer.lineNo; - if (consumer.discarding) entry.state.discardingLargeLine = true; - else delete entry.state.discardingLargeLine; - dirty = true; - return dirty; + }); + consumer.offset = start; + consumer.lineNo = entry.lineNo ?? 0; + consumer.discarding = entry.state.discardingLargeLine; + const now = nowFn(options); + let bytes = 0; + let stopped = false; + try { + const result = await streamRange(filePath, start, fileStat.size, async (chunk, chunkStart) => { + bytes += chunk.length; + consumer.feedAt(chunk, chunkStart); + }, { + shouldStop: () => stopRequested(options) || deadlineReached(options, now()), + }); + bytes = result.bytes; + stopped = result.stopped; + consumer.finish(start + bytes); + if (!stopped) delete entry.state.readFailed; + } catch (error) { + // Keep the accepted complete prefix and its parser state together. The + // unfinished suffix is retried even if the file metadata is unchanged. + entry.state.readFailed = true; + sink.warn(`${adapter} ${entry.path}: read failed; accepted prefix retained: ${error.message}`); } - - return dirty; + stats.bytesRead += bytes; + if (bytes > 0) stats.filesRead += 1; + if (consumer.skippedLargeLine) { + sink.warn( + `${adapter} ${entry.path}: a line larger than ${MAX_LINE_BYTES} bytes was skipped.`, + ); + } + entry.offset = consumer.offset; + entry.lineNo = consumer.lineNo; + if (consumer.discarding) entry.state.discardingLargeLine = true; + else delete entry.state.discardingLargeLine; + if (stopped) entry.state.scanPending = true; + else delete entry.state.scanPending; + return { dirty: true, pending: stopped }; } async function scanCompressedFile( @@ -470,7 +538,7 @@ async function scanCompressedFile( fileStat, sink, stats, - checkpoint, + options = {}, ) { if (!hasZstdSupport()) { entry.size = fileStat.size; @@ -480,9 +548,10 @@ async function scanCompressedFile( sink.warn( `${source.adapter} ${entry.path}: .jsonl.zst needs Node's zstd API (Node 22.15+); skipped.`, ); - return { ok: true, dirty: true }; + return { status: "unsupported", dirty: true }; } - // Roll back metadata if a stream fails after producing valid records. Raw + // The caller owns a per-file transaction. Any decompression failure or + // deadline stop discards this open archive without exposing its rows. Raw // transcript bytes are never retained: the line consumer is bounded. let result; try { @@ -490,21 +559,23 @@ async function scanCompressedFile( result = await pipeline( createReadStream(filePath), zlib.createZstdDecompress(), - async (chunks) => processStringChunks(store, entry, source, chunks, 0, undefined, sink), + async (chunks) => processStringChunks( + store, entry, source, chunks, 0, undefined, sink, options, + ), ); } catch (error) { - Object.assign(store, checkpoint); - prepareStore(store); - sink.warn( - `${source.adapter} ${entry.path}: zstd file could not be decompressed: ${error.message}`, - ); - return { ok: false, dirty: false }; + if (error instanceof ScanStopError) { + return { status: "stopped", reason: error.reason, dirty: false }; + } + return { status: "corrupt", error, dirty: false }; } stats.bytesRead += fileStat.size; if (fileStat.size > 0) stats.filesRead += 1; + stats.decompressedBytes = (stats.decompressedBytes ?? 0) + (result.bytes ?? 0); entry.offset = fileStat.size; entry.lineNo = result.lineNo; - return { ok: true, dirty: true }; + delete entry.state.zstdError; + return { status: "ok", dirty: true }; } function openCodeString(value) { @@ -523,7 +594,7 @@ function openCodeInt(value) { * every write while the main database may not, so the checkpoint fingerprint * covers both files. */ -async function scanOpenCodeDb(store, source, filePath, sink, stats) { +async function scanOpenCodeDb(store, source, filePath, sink, stats, options = {}) { let dbStat; try { dbStat = await stat(filePath); @@ -531,9 +602,9 @@ async function scanOpenCodeDb(store, source, filePath, sink, stats) { if (error?.code !== "ENOENT") { sink.warn(`opencode ${filePath}: cannot stat file: ${error.message}`); } - return false; + return { dirty: false, aborted: false }; } - if (!dbStat.isFile()) return false; + if (!dbStat.isFile()) return { dirty: false, aborted: false }; let walStat = null; try { walStat = await stat(`${filePath}-wal`); @@ -542,50 +613,69 @@ async function scanOpenCodeDb(store, source, filePath, sink, stats) { const key = fileKey(source.id, filePath); let entry = store.files[key]; - let dirty = false; - if (!entry) { - entry = newFileEntry(store, { - sourceId: source.id, - adapter: source.adapter, - path: filePath, - dev: String(dbStat.dev), - ino: String(dbStat.ino), - size: 0, - mtimeMs: 0, - compressed: false, - }); - dirty = true; - } - if (entry.state.opencodeParserVersion === 2 && entry.state.opencodeFingerprint === fingerprint) { + if (entry?.state.opencodeParserVersion === 2 && entry.state.opencodeFingerprint === fingerprint) { if (entry.state.opencodeMalformedRows) sink.warn(`opencode ${filePath}: ${entry.state.opencodeMalformedRows} malformed message rows were skipped.`); stats.unchangedFiles += 1; - return dirty; + return { dirty: false, aborted: false }; } let sqlite; try { sqlite = await import("node:sqlite"); } catch { + const tx = beginStoreTransaction(store, { fileId: entry?.id }); + let dirty = false; + if (!entry) { + entry = newFileEntry(store, { + sourceId: source.id, + adapter: source.adapter, + path: filePath, + dev: String(dbStat.dev), + ino: String(dbStat.ino), + size: 0, + mtimeMs: 0, + compressed: false, + }); + dirty = true; + } if (entry.state.unsupported !== "sqlite") { sink.warn(`opencode ${filePath}: node:sqlite is unavailable in this Node runtime; skipped.`); entry.state.unsupported = "sqlite"; dirty = true; } + commitStoreTransaction(tx); stats.unchangedFiles += 1; - return dirty; + return { dirty, aborted: false }; } let db; - let checkpoint; let malformedRows = 0; + const tx = beginStoreTransaction(store, { fileId: entry?.id }); + if (!entry) { + entry = newFileEntry(store, { + sourceId: source.id, + adapter: source.adapter, + path: filePath, + dev: String(dbStat.dev), + ino: String(dbStat.ino), + size: 0, + mtimeMs: 0, + compressed: false, + }); + } + const now = nowFn(options); + let rowsSeen = 0; try { // Never open an external source writable, including on recovery errors. db = new sqlite.DatabaseSync(filePath, { readOnly: true }); const records = db.prepare("SELECT session_id, id, time_created, data FROM message").iterate(); - checkpoint = structuredClone(store); resetFileEntry(store, entry); delete entry.state.unsupported; for (const record of records) { + if (stopRequested(options) || (rowsSeen > 0 && deadlineReached(options, now()))) { + throw new ScanStopError("budget"); + } + rowsSeen += 1; let data; try { data = JSON.parse(record.data); @@ -633,13 +723,14 @@ async function scanOpenCodeDb(store, source, filePath, sink, stats) { ); } } + commitStoreTransaction(tx); } catch (error) { - if (checkpoint) { - Object.assign(store, checkpoint); - prepareStore(store); + rollbackStoreTransaction(tx); + if (error instanceof ScanStopError) { + return { dirty: false, aborted: true, reason: error.reason }; } sink.warn(`opencode ${filePath}: scan failed: ${error.message}`); - return dirty; + return { dirty: false, aborted: false }; } finally { try { db?.close(); } catch { /* close is best-effort */ } } @@ -654,10 +745,10 @@ async function scanOpenCodeDb(store, source, filePath, sink, stats) { if (malformedRows) sink.warn(`opencode ${filePath}: ${malformedRows} malformed message rows were skipped.`); stats.filesRead += 1; stats.bytesRead += dbStat.size + (walStat?.size ?? 0); - return true; + return { dirty: true, aborted: false }; } -async function scanOneFile(store, source, filePath, sink, stats) { +async function scanOneFile(store, source, filePath, sink, stats, options = {}) { const sourceSink = { warn(message) { sink.warn(`[${source.id}] ${message}`, source.id); @@ -667,7 +758,7 @@ async function scanOneFile(store, source, filePath, sink, stats) { }, }; if (source.adapter === "opencode") { - return scanOpenCodeDb(store, source, filePath, sourceSink, stats); + return scanOpenCodeDb(store, source, filePath, sourceSink, stats, options); } let fileStat; try { @@ -676,84 +767,149 @@ async function scanOneFile(store, source, filePath, sink, stats) { if (error?.code !== "ENOENT") { sourceSink.warn(`${source.adapter} ${filePath}: cannot stat file: ${error.message}`); } - return false; - } - if (!fileStat.isFile()) return false; - - const key = fileKey(source.id, filePath); - let entry = store.files[key]; - const isNew = !entry; - let dirty = false; - if (!entry) { - entry = newFileEntry(store, { - sourceId: source.id, - adapter: source.adapter, - path: filePath, - dev: String(fileStat.dev), - ino: String(fileStat.ino), - size: 0, - mtimeMs: 0, - compressed: filePath.endsWith(".jsonl.zst"), - }); - dirty = true; + return { dirty: false, aborted: false }; } + if (!fileStat.isFile()) return { dirty: false, aborted: false }; const dev = String(fileStat.dev); const ino = String(fileStat.ino); - const parserVersion = PARSER_VERSIONS[entry.adapter]; - const parserKey = `${entry.adapter}ParserVersion`; - const parserCurrent = (!parserVersion || entry.state[parserKey] === parserVersion) - && !(entry.state.unsupported === "zstd" && hasZstdSupport()); - if (parserCurrent && !entry.state.readFailed && entry.dev === dev && entry.ino === ino - && entry.size === fileStat.size && entry.mtimeMs === fileStat.mtimeMs) { - stats.unchangedFiles += 1; - return dirty; - } - // Capture before migrations/truncation reset anything, including ownership - // transferred to copied files. Unchanged archives never allocate a checkpoint. - const checkpoint = entry.compressed && hasZstdSupport() ? structuredClone(store) : null; - let forceRead = false; - if (!isNew && parserVersion && entry.state[parserKey] !== parserVersion) { - // Migrate existing history once for accounting fixes. New files own no - // metadata and must never trigger a global ownership cleanup scan. - resetFileEntry(store, entry); - forceRead = true; - dirty = true; - } - if ( - entry.state.unsupported === "zstd" && - hasZstdSupport() && - filePath.endsWith(".jsonl.zst") - ) { - resetFileEntry(store, entry); - forceRead = true; - dirty = true; - } - if ( - entry.dev !== dev || - entry.ino !== ino || - fileStat.size < entry.size || - fileStat.size < entry.offset - ) { - resetFileEntry(store, entry); - entry.dev = dev; - entry.ino = ino; - forceRead = true; - dirty = true; - } - + const key = fileKey(source.id, filePath); + const existing = store.files[key]; + const isNew = !existing; + const parserVersion = PARSER_VERSIONS[existing?.adapter ?? source.adapter]; + const parserKey = `${existing?.adapter ?? source.adapter}ParserVersion`; + const parserCurrent = (!parserVersion || existing?.state[parserKey] === parserVersion) + && !(existing?.state.unsupported === "zstd" && hasZstdSupport()); if ( - !forceRead && - !entry.state.readFailed && - entry.size === fileStat.size && - entry.mtimeMs === fileStat.mtimeMs + existing + && parserCurrent + && !existing.state.readFailed + && !existing.state.scanPending + && existing.dev === dev + && existing.ino === ino + && existing.size === fileStat.size + && existing.mtimeMs === fileStat.mtimeMs ) { stats.unchangedFiles += 1; - return dirty; + return { dirty: false, aborted: false }; } - if (filePath.endsWith(".jsonl.zst")) { - const compressed = await scanCompressedFile( + // Every changed file gets a precise transaction. The previous committed + // contributions stay visible until the whole file is accepted, and a failure + // restores rows, dedup/task ownership, parser state, and nextFileId. + const tx = beginStoreTransaction(store, { fileId: existing?.id }); + let entry = existing; + let dirty = false; + try { + if (!entry) { + entry = newFileEntry(store, { + sourceId: source.id, + adapter: source.adapter, + path: filePath, + dev, + ino, + size: 0, + mtimeMs: 0, + compressed: filePath.endsWith(".jsonl.zst"), + }); + dirty = true; + } + + let forceRead = false; + if (!isNew && parserVersion && entry.state[parserKey] !== parserVersion) { + // Migrate existing history once for accounting fixes. New files own no + // metadata and must never trigger a global ownership cleanup scan. + resetFileEntry(store, entry); + forceRead = true; + dirty = true; + } + if ( + entry.state.unsupported === "zstd" && + hasZstdSupport() && + filePath.endsWith(".jsonl.zst") + ) { + resetFileEntry(store, entry); + forceRead = true; + dirty = true; + } + if ( + entry.dev !== dev || + entry.ino !== ino || + fileStat.size < entry.size || + fileStat.size < entry.offset + ) { + resetFileEntry(store, entry); + entry.dev = dev; + entry.ino = ino; + forceRead = true; + dirty = true; + } + + if ( + !forceRead && + !entry.state.readFailed && + !entry.state.scanPending && + entry.size === fileStat.size && + entry.mtimeMs === fileStat.mtimeMs + ) { + stats.unchangedFiles += 1; + commitStoreTransaction(tx); + return { dirty, aborted: false }; + } + + if (filePath.endsWith(".jsonl.zst")) { + const compressed = await scanCompressedFile( + store, + entry, + source, + filePath, + fileStat, + sourceSink, + stats, + options, + ); + if (compressed.status === "stopped") { + rollbackStoreTransaction(tx); + return { + dirty: false, + aborted: true, + abortedFile: { + sourceId: source.id, + adapter: source.adapter, + path: filePath, + size: fileStat.size, + mtimeMs: fileStat.mtimeMs, + dev, + ino, + reason: compressed.reason ?? "budget", + }, + }; + } + if (compressed.status === "corrupt") { + rollbackStoreTransaction(tx); + // Preserve the previous good archive and remember this exact corrupt + // fingerprint so an unchanged broken file is not re-read every refresh. + entry.size = fileStat.size; + entry.mtimeMs = fileStat.mtimeMs; + entry.offset = fileStat.size; + entry.state.zstdError = `${fileStat.mtimeMs}:${fileStat.size}`; + sourceSink.warn( + `${source.adapter} ${entry.path}: zstd file could not be decompressed: ${compressed.error.message}`, + ); + return { dirty: true, aborted: false }; + } + dirty = compressed.dirty || dirty; + commitStoreTransaction(tx); + entry.size = fileStat.size; + entry.mtimeMs = fileStat.mtimeMs; + entry.path = filePath; + if (parserVersion) entry.state[parserKey] = parserVersion; + delete entry.state.zstdError; + delete entry.state.scanPending; + return { dirty, aborted: false }; + } + + const plain = await scanPlainFile( store, entry, source, @@ -761,31 +917,36 @@ async function scanOneFile(store, source, filePath, sink, stats) { fileStat, sourceSink, stats, - checkpoint, + options, ); - if (!compressed.ok) return dirty; - dirty = compressed.dirty || dirty; - } else { - dirty = - (await scanPlainFile( - store, - entry, - source, - filePath, - fileStat, - sourceSink, - stats, - )) || dirty; + dirty = plain.dirty || dirty; + commitStoreTransaction(tx); + entry.path = filePath; + if (parserVersion) entry.state[parserKey] = parserVersion; + if (!plain.pending) { + entry.size = fileStat.size; + entry.mtimeMs = fileStat.mtimeMs; + delete entry.state.scanPending; + } + return { dirty, aborted: false, pending: plain.pending }; + } catch (error) { + rollbackStoreTransaction(tx); + sourceSink.warn(`${source.adapter} ${filePath}: scan failed: ${error.message}`); + return { dirty: false, aborted: false }; } - - entry.size = fileStat.size; - entry.mtimeMs = fileStat.mtimeMs; - entry.path = filePath; - if (parserVersion) entry.state[parserKey] = parserVersion; - return dirty; } -export async function scanSources(store, { home, onProgress } = {}) { +export async function scanSources(store, { + home, + onProgress, + onCheckpoint, + onStreamProgress, + deadline, + shouldStop, + now, + streamDelayMs, + mode = "foreground", +} = {}) { const warnings = []; const stats = { files: 0, @@ -793,6 +954,7 @@ export async function scanSources(store, { home, onProgress } = {}) { bytesRead: 0, unchangedFiles: 0, errors: 0, + decompressedBytes: 0, }; const sourceState = {}; for (const source of store.config.sources) { @@ -810,11 +972,42 @@ export async function scanSources(store, { home, onProgress } = {}) { const discovered = await discoverFiles(store.config, home, sink, sourceState); const seenInodes = new Set(); const seenPaths = new Set(); + const options = { + deadline, + shouldStop, + now: now ?? Date.now, + streamDelayMs, + onStreamProgress: (progress) => { + onStreamProgress?.(progress); + }, + }; let dirty = false; let processed = 0; + let stopReason = null; + let abortedFile = null; for (const found of discovered) { - onProgress?.({ processed, total: discovered.length, file: found.file }); + if (stopRequested(options)) { + stopReason = "stop"; + break; + } + if (processed > 0 && deadlineReached(options, options.now()) + && (stats.filesRead > 0 || dirty)) { + // A tiny budget still guarantees one changed file per slice. Otherwise a + // slice could repeatedly stop on the same unchanged prefix and never + // converge; unchanged files remain cheap discovery/stat work. + stopReason = "budget"; + break; + } + onProgress?.({ + processed, + total: discovered.length, + file: found.file, + mode, + filesRead: stats.filesRead, + bytesRead: stats.bytesRead, + phase: "scan", + }); processed += 1; if (seenPaths.has(found.file)) continue; let fileStat; @@ -838,24 +1031,52 @@ export async function scanSources(store, { home, onProgress } = {}) { sourceState[found.source.id].files += 1; const beforeRead = stats.filesRead; const beforeBytes = stats.bytesRead; - const changed = await scanOneFile( + const result = await scanOneFile( store, found.source, found.file, sink, stats, + options, ).catch((error) => { sink.warn( `[${found.source.id}] ${found.source.adapter} ${found.file}: scan failed: ${error.message}`, found.source.id, ); - return false; + return { dirty: false, aborted: false }; }); sourceState[found.source.id].filesRead += stats.filesRead - beforeRead; sourceState[found.source.id].bytesRead += stats.bytesRead - beforeBytes; - dirty = changed || dirty; + dirty = Boolean(result?.dirty) || dirty; + if (result?.aborted) { + stopReason = result.reason ?? "budget"; + abortedFile = result.abortedFile ?? { + sourceId: found.source.id, + adapter: found.source.adapter, + path: found.file, + size: fileStat.size, + mtimeMs: fileStat.mtimeMs, + dev: String(fileStat.dev), + ino: String(fileStat.ino), + reason: result.reason ?? "budget", + }; + break; + } + if (result?.pending) { + // Plain JSONL reached a slice boundary inside a file. The accepted + // complete prefix is committed and its offset resumes next time. + stopReason = "budget"; + break; + } + await onCheckpoint?.({ + processed, + total: discovered.length, + file: found.file, + stats, + dirty, + }); } const sourceStats = {}; @@ -877,11 +1098,22 @@ export async function scanSources(store, { home, onProgress } = {}) { } sink.finish(); - return { stats, warnings, dirty, sourceStats }; + return { + stats, + warnings, + dirty, + sourceStats, + complete: stopReason === null, + stopped: stopReason !== null, + stopReason, + abortedFile, + processed, + total: discovered.length, + }; } -export async function rebuildScan(store, { home }) { - return scanSources(store, { home }); +export async function rebuildScan(store, options = {}) { + return scanSources(store, options); } export { createLineConsumer, processStringChunks, scanOneFile, walkRoot }; diff --git a/herdr-insights/mods/insights/core/store.mjs b/herdr-insights/mods/insights/core/store.mjs index 5202acd8..9f9f585c 100644 --- a/herdr-insights/mods/insights/core/store.mjs +++ b/herdr-insights/mods/insights/core/store.mjs @@ -1,4 +1,5 @@ import { readFile, stat, unlink } from "node:fs/promises"; +import { writePresentationCache } from "./cache.mjs"; import { defaultConfig, migrateLoadedConfig, normalizeConfig } from "./config.mjs"; import { STORE_VERSION, @@ -16,15 +17,76 @@ export function fileKey(sourceId, filePath) { return `${sourceId}\u0000${filePath}`; } +const SCAN_STATUSES = new Set([ + "idle", + "running", + "partial", + "complete", + "failed", + "reconfiguring", +]); + +function zeroScanStats() { + return { files: 0, filesRead: 0, bytesRead: 0, unchangedFiles: 0, errors: 0 }; +} + +export function emptyScanState() { + return { + status: "idle", + generation: 0, + startedAt: null, + updatedAt: null, + completedAt: null, + processed: 0, + total: 0, + stats: zeroScanStats(), + sourceStats: {}, + warnings: [], + currentFile: null, + pendingFile: null, + workerPid: null, + error: null, + }; +} + +/** Additive, backward-compatible store metadata for truthful scan state. */ +export function ensureStoreMeta(store) { + if (!Number.isInteger(store.generation) || store.generation < 0) store.generation = 0; + if (store.lastCompletedScanAt === undefined) store.lastCompletedScanAt = null; + if (!isPlainObject(store.scan)) store.scan = emptyScanState(); + const scan = store.scan; + if (!SCAN_STATUSES.has(scan.status)) scan.status = "idle"; + if (!Number.isInteger(scan.generation) || scan.generation < 0) scan.generation = 0; + for (const field of ["startedAt", "updatedAt", "completedAt", "currentFile", "error"]) { + if (scan[field] === undefined) scan[field] = null; + } + if (!isPlainObject(scan.stats)) scan.stats = zeroScanStats(); + if (!isPlainObject(scan.sourceStats)) scan.sourceStats = {}; + if (!Array.isArray(scan.warnings)) scan.warnings = []; + if (!Number.isInteger(scan.processed) || scan.processed < 0) scan.processed = 0; + if (!Number.isInteger(scan.total) || scan.total < 0) scan.total = 0; + if (scan.pendingFile !== null && !isPlainObject(scan.pendingFile)) scan.pendingFile = null; + if (!Number.isInteger(scan.workerPid) || scan.workerPid <= 0) scan.workerPid = null; + return store; +} + +export function storeGeneration(store) { + const value = Number(store?.generation); + return Number.isInteger(value) && value >= 0 ? value : 0; +} + export function emptyStore(config = defaultConfig()) { const store = { version: STORE_VERSION, config, + generation: 0, nextFileId: 1, files: {}, dedup: {}, tasks: {}, lastScanAt: null, + lastCompletedScanAt: null, + scan: emptyScanState(), }; prepareStore(store); return store; @@ -33,7 +95,10 @@ export function emptyStore(config = defaultConfig()) { export function prepareStore(store) { const byId = new Map(); for (const entry of Object.values(store.files ?? {})) { - if (entry && Number.isInteger(entry.id)) byId.set(entry.id, entry); + if (entry && Number.isInteger(entry.id)) { + byId.set(entry.id, entry); + attachStore(entry, store); + } } Object.defineProperty(store, "_byId", { value: byId, @@ -44,6 +109,13 @@ export function prepareStore(store) { return store; } +function attachStore(entry, store) { + if (entry._store === store) return; + Object.defineProperty(entry, "_store", { + value: store, enumerable: false, writable: true, configurable: true, + }); +} + export function findFileById(store, id) { if (!store._byId) prepareStore(store); return store._byId.get(id) ?? null; @@ -51,11 +123,185 @@ export function findFileById(store, id) { export function addFileEntry(store, entry) { if (!store._byId) prepareStore(store); - store.files[fileKey(entry.sourceId, entry.path)] = entry; + const key = fileKey(entry.sourceId, entry.path); + const tx = store._tx; + if (tx) { + const existing = store.files[key]; + if (existing && existing !== entry) tx.trackFile(existing); + if (!existing) tx.createdFileKeys.add(key); + if (!existing || existing.id !== entry.id) tx.createdFileIds.add(entry.id); + } + store.files[key] = entry; store._byId.set(entry.id, entry); + attachStore(entry, store); return entry; } +const TX_MISSING = Symbol("insights-tx-missing"); + +function cloneAux(value, index) { + const aux = value[index]; + if (!isPlainObject(aux)) return value; + const copy = { ...aux }; + if (Array.isArray(aux.copies)) copy.copies = [...aux.copies]; + if (isPlainObject(aux.meta)) copy.meta = { ...aux.meta }; + value[index] = copy; + return value; +} + +function cloneRecord(value, auxIndex) { + if (!Array.isArray(value)) return value; + return cloneAux([...value], auxIndex); +} + +function snapshotFileEntry(entry) { + const fields = {}; + for (const [key, value] of Object.entries(entry)) { + if (key === "rows" || key === "tasks" || key === "state" || key === "_rowIndex") continue; + fields[key] = value; + } + return { + entry, + fields, + rowsRef: Array.isArray(entry.rows) ? entry.rows : null, + rowsLength: Array.isArray(entry.rows) ? entry.rows.length : 0, + rows: new Map(), + tasksRef: Array.isArray(entry.tasks) ? entry.tasks : null, + tasks: Array.isArray(entry.tasks) ? [...entry.tasks] : [], + state: structuredClone(entry.state ?? {}), + }; +} + +/** + * Precise per-file transaction. Existing mutation helpers keep their single + * accounting implementation; they only announce which dedup/task/row records + * they are about to touch. Unchanged history is never cloned, so cost follows + * the records a file actually changes rather than the whole accumulated store. + */ +export function beginStoreTransaction(store, { fileId } = {}) { + if (store._tx) throw new Error("a store transaction is already active"); + const tx = { + store, + savedFiles: new Map(), + savedDedup: new Map(), + savedTasks: new Map(), + createdFileKeys: new Set(), + createdFileIds: new Set(), + deletedFiles: new Map(), + nextFileId: store.nextFileId, + active: true, + trackFile(entry) { + if (!entry || !Number.isInteger(entry.id)) return null; + if (tx.createdFileIds.has(entry.id)) return null; + if (!tx.savedFiles.has(entry.id)) tx.savedFiles.set(entry.id, snapshotFileEntry(entry)); + return tx.savedFiles.get(entry.id); + }, + trackFileById(id) { + const entry = findFileById(store, id); + return tx.trackFile(entry); + }, + trackRow(entry, index) { + if (!entry || !Number.isInteger(index) || index < 0) return; + const record = tx.trackFile(entry); + if (!record) return; + // When a reset replaced the rows array, rollback restores the original + // array wholesale; per-index snapshots of the new array would overwrite it. + if (record.rowsRef && entry.rows !== record.rowsRef) return; + // Rows created during the transaction are removed by truncating to the + // original length; do not re-add them from a later snapshot. + if (index >= record.rowsLength) return; + if (!record.rows.has(index)) { + const row = entry.rows?.[index]; + record.rows.set(index, isPlainObject(row) ? { ...row } : row); + } + }, + touchDedup(key) { + if (tx.savedDedup.has(key)) return; + const value = Object.hasOwn(store.dedup, key) ? store.dedup[key] : TX_MISSING; + tx.savedDedup.set(key, value === TX_MISSING ? TX_MISSING : cloneRecord(value, DEDUP_AUX_INDEX)); + }, + touchTask(key) { + if (tx.savedTasks.has(key)) return; + const value = Object.hasOwn(store.tasks, key) ? store.tasks[key] : TX_MISSING; + tx.savedTasks.set(key, value === TX_MISSING ? TX_MISSING : cloneRecord(value, TASK_AUX_INDEX)); + }, + deleteFileEntry(key, entry) { + tx.trackFile(entry); + tx.deletedFiles.set(key, entry); + tx.createdFileKeys.delete(key); + }, + }; + Object.defineProperty(store, "_tx", { + value: tx, enumerable: false, writable: true, configurable: true, + }); + if (Number.isInteger(fileId)) tx.trackFileById(fileId); + return tx; +} + +export function commitStoreTransaction(tx) { + if (!tx?.active) return false; + tx.active = false; + if (tx.store._tx === tx) delete tx.store._tx; + return true; +} + +export function rollbackStoreTransaction(tx) { + if (!tx?.active) return false; + tx.active = false; + const store = tx.store; + for (const saved of tx.savedFiles.values()) { + const entry = saved.entry; + for (const key of Object.keys(entry)) { + if (key === "rows" || key === "tasks" || key === "state" || key === "_rowIndex") continue; + if (!Object.hasOwn(saved.fields, key)) delete entry[key]; + } + Object.assign(entry, saved.fields); + if (saved.rowsRef) { + saved.rowsRef.length = saved.rowsLength; + for (const [index, row] of saved.rows) saved.rowsRef[index] = row; + entry.rows = saved.rowsRef; + } else { + entry.rows = []; + } + if (saved.tasksRef) { + saved.tasksRef.length = 0; + saved.tasksRef.push(...saved.tasks); + entry.tasks = saved.tasksRef; + } else { + entry.tasks = []; + } + entry.state = saved.state; + // A row mutation can invalidate the transient row index. Rebuilding it + // lazily after rollback is cheap relative to a rare corrupt archive. + delete entry._rowIndex; + } + for (const [key, value] of tx.savedDedup) { + if (value === TX_MISSING) delete store.dedup[key]; + else store.dedup[key] = value; + } + for (const [key, value] of tx.savedTasks) { + if (value === TX_MISSING) delete store.tasks[key]; + else store.tasks[key] = value; + } + for (const key of tx.createdFileKeys) { + const entry = store.files[key]; + if (entry && tx.createdFileIds.has(entry.id)) { + delete store.files[key]; + store._byId?.delete(entry.id); + } + } + for (const [key, entry] of tx.deletedFiles) { + store.files[key] = entry; + store._byId?.set(entry.id, entry); + } + store.nextFileId = tx.nextFileId; + if (store._tx === tx) delete store._tx; + // Rebuild the transient id index from the restored file map. Rollback is a + // rare recovery path, so an O(files) rebuild is preferable to a stale index. + prepareStore(store); + return true; +} + export function newFileEntry(store, values) { const entry = { id: store.nextFileId++, @@ -73,6 +319,7 @@ export function newFileEntry(store, values) { export function deleteFileEntry(store, key) { const entry = store.files[key]; if (!entry) return null; + store._tx?.deleteFileEntry(key, entry); removeOwnedMetadata(store, entry.id); delete store.files[key]; if (store._byId) store._byId.delete(entry.id); @@ -80,23 +327,33 @@ export function deleteFileEntry(store, key) { } export function removeOwnedMetadata(store, fileId) { + const tx = store._tx; for (const [key, value] of Object.entries(store.dedup)) { if (!Array.isArray(value)) continue; - dropCopyCandidate(value, DEDUP_AUX_INDEX, fileId); + const aux = value[DEDUP_AUX_INDEX]; + const hasCopy = isPlainObject(aux) && Array.isArray(aux.copies) && aux.copies.includes(fileId); + if (hasCopy) tx?.touchDedup(key); + if (hasCopy) dropCopyCandidate(value, DEDUP_AUX_INDEX, fileId); if (value[0] !== fileId) continue; + tx?.touchDedup(key); if (transferDedupOwnership(store, value)) continue; delete store.dedup[key]; } for (const [key, value] of Object.entries(store.tasks)) { if (!Array.isArray(value)) continue; - dropCopyCandidate(value, TASK_AUX_INDEX, fileId); + const aux = value[TASK_AUX_INDEX]; + const hasCopy = isPlainObject(aux) && Array.isArray(aux.copies) && aux.copies.includes(fileId); + if (hasCopy) tx?.touchTask(key); + if (hasCopy) dropCopyCandidate(value, TASK_AUX_INDEX, fileId); if (value[0] !== fileId) continue; + tx?.touchTask(key); if (transferTaskOwnership(store, value)) continue; delete store.tasks[key]; } } export function resetFileEntry(store, entry) { + store._tx?.trackFile(entry); removeOwnedMetadata(store, entry.id); entry.offset = 0; entry.lineNo = 0; @@ -108,6 +365,7 @@ export function resetFileEntry(store, entry) { } export function clearCollectedData(store) { + if (store._tx) throw new Error("cannot clear collected data inside a transaction"); store.files = {}; store.dedup = {}; store.tasks = {}; @@ -116,6 +374,7 @@ export function clearCollectedData(store) { } export function getOrCreateRow(file, meta) { + storeTrackFileForRow(file); // OpenCode puts the entire history in one file. A linear search here makes // a cold scan quadratic in the number of session/model buckets. if (!file._rowIndex || file._rowIndex.length !== file.rows.length) { @@ -151,6 +410,12 @@ export function getOrCreateRow(file, meta) { return index; } +function storeTrackFileForRow(file) { + const store = file?._store; + if (!store?._tx) return; + store._tx.trackFile(file); +} + function addRowTokens(row, tokens) { row.input = toNonNegInt(row.input + tokens.input); row.output = toNonNegInt(row.output + tokens.output); @@ -237,7 +502,9 @@ function transferDedupOwnership(store, entry) { } const candidate = takeCandidate(store, aux); if (!candidate) return false; + store._tx?.trackFile(candidate); const rowIdx = getOrCreateRow(candidate, aux.meta); + store._tx?.trackRow(candidate, rowIdx); const row = candidate.rows[rowIdx]; addRowTokens(row, { input: entry[2], @@ -259,6 +526,7 @@ function transferTaskOwnership(store, entry) { } const candidate = takeCandidate(store, aux); if (!candidate) return false; + store._tx?.trackFile(candidate); const rowIdx = getOrCreateRow(candidate, aux.meta); entry[0] = candidate.id; entry[1] = rowIdx; @@ -268,11 +536,15 @@ function transferTaskOwnership(store, entry) { // Usage dedup is a max-upsert: Claude streams the same message.id repeatedly // and revises its numbers upward. One message, maximum numeric value per field. export function addUsage(store, file, meta, tokens, dedupKey) { + const tx = store._tx; const entry = store.dedup[dedupKey]; if (Array.isArray(entry)) { const owner = findFileById(store, entry[0]); const row = owner?.rows?.[entry[1]]; if (owner && row) { + tx?.trackFile(owner); + tx?.trackRow(owner, entry[1]); + tx?.touchDedup(dedupKey); let changed = refreshConcreteProvider(owner, row, entry, meta); const added = { input: 0, @@ -303,10 +575,16 @@ export function addUsage(store, file, meta, tokens, dedupKey) { added, }; } + // Save the stale entry before removing it so a rollback restores the + // exact committed dedup map, including an owner that no longer exists. + tx?.touchDedup(dedupKey); delete store.dedup[dedupKey]; } + tx?.trackFile(file); + tx?.touchDedup(dedupKey); const rowIdx = getOrCreateRow(file, meta); + tx?.trackRow(file, rowIdx); const row = file.rows[rowIdx]; addRowTokens(row, tokens); row.messages = toNonNegInt(row.messages + 1); @@ -346,11 +624,14 @@ export function addTask( candidateFileId = fileId, ) { const ms = toNonNegInt(durationMs); + const tx = store._tx; const owner = findFileById(store, fileId); const row = owner?.rows?.[rowIdx]; if (!owner || !row) return false; + tx?.trackFile(owner); const existing = store.tasks[taskKey]; if (Array.isArray(existing)) { + tx?.touchTask(taskKey); if (existing[0] !== candidateFileId) { const existingOwner = findFileById(store, existing[0]); const existingRow = existingOwner?.rows?.[existing[1]]; @@ -370,6 +651,7 @@ export function addTask( return false; } const sHash = sessionHash(session); + tx?.touchTask(taskKey); store.tasks[taskKey] = [fileId, rowIdx, ms, exact ? 1 : 0, sHash]; if (!Array.isArray(owner.tasks)) owner.tasks = []; owner.tasks.push(taskKey); @@ -383,8 +665,10 @@ export function removeLatestFallbackTaskForSession(store, session) { const key = keys[index]; const entry = store.tasks[key]; if (Array.isArray(entry) && entry[3] === 0 && entry[4] === sHash) { + store._tx?.touchTask(key); delete store.tasks[key]; const owner = findFileById(store, entry[0]); + store._tx?.trackFile(owner); if (Array.isArray(owner?.tasks)) { owner.tasks = owner.tasks.filter((taskKey) => taskKey !== key); } @@ -401,6 +685,7 @@ export function removeLatestFallbackTaskForSession(store, session) { export function removeFallbackTasksForSession(store, file, session) { if (!Array.isArray(file.tasks) || file.tasks.length === 0) return; + store._tx?.trackFile(file); const sHash = sessionHash(session); const keep = []; for (const key of file.tasks) { @@ -411,6 +696,7 @@ export function removeFallbackTasksForSession(store, file, session) { entry[3] === 0 && entry[4] === sHash ) { + store._tx?.touchTask(key); delete store.tasks[key]; } else { keep.push(key); @@ -635,6 +921,7 @@ export async function loadStore(storePath, { readOnly = false } = {}) { try { parsed = JSON.parse(text); const migrated = validateLoadedStore(parsed); + ensureStoreMeta(parsed); prepareStore(parsed); return { store: parsed, warnings, errors, exists: true, dirty: migrated }; } catch (error) { @@ -667,9 +954,34 @@ export async function loadStore(storePath, { readOnly = false } = {}) { } } -export async function saveStore(storePath, store) { - store.lastScanAt = new Date().toISOString(); +export async function saveStore( + storePath, + store, + { completed = false, status, now = new Date() } = {}, +) { + const savedAt = now instanceof Date ? now.toISOString() : new Date(now).toISOString(); + ensureStoreMeta(store); + store.lastScanAt = savedAt; + const scan = store.scan; + scan.updatedAt = savedAt; + scan.generation = storeGeneration(store); + if (status) scan.status = status; + if (completed) { + scan.status = "complete"; + scan.completedAt = savedAt; + scan.currentFile = null; + scan.pendingFile = null; + scan.workerPid = null; + scan.error = null; + store.lastCompletedScanAt = savedAt; + } await atomicWriteJson(storePath, store); + // The presentation cache is derived from this exact committed store. A + // failure here must not fail the commit; prepare falls back to the store. + try { + await writePresentationCache(storePath, { savedAt, store, rows: snapshotRows(store) }); + } catch { /* the cache is an optional read accelerator */ } + return savedAt; } export async function fileStatOrNull(file) { diff --git a/herdr-insights/mods/insights/core/util.mjs b/herdr-insights/mods/insights/core/util.mjs index 128969b1..f3f1b61c 100644 --- a/herdr-insights/mods/insights/core/util.mjs +++ b/herdr-insights/mods/insights/core/util.mjs @@ -237,7 +237,7 @@ async function tryUnlinkStaleLock(lockPath, observed) { export async function withStoreLock( storePath, callback, - { timeoutMs = 20_000, staleMs = STORE_LOCK_STALE_MS } = {}, + { timeoutMs = 20_000, staleMs = STORE_LOCK_STALE_MS, meta = {} } = {}, ) { const lockPath = `${storePath}.lock`; await ensureParent(lockPath); @@ -255,7 +255,7 @@ export async function withStoreLock( created = true; handle = await open(ownerPath, "wx", 0o600); await handle.writeFile( - `${JSON.stringify({ pid: process.pid, host, startedAt, token })}\n`, + `${JSON.stringify({ ...meta, pid: process.pid, host, startedAt, token })}\n`, "utf8", ); await handle.sync(); @@ -302,3 +302,12 @@ export async function withStoreLock( } } } + +/** The committed lock owner, if any. Used for fast status responses. */ +export async function readStoreLockInfo(storePath) { + return readLockInfo(`${storePath}.lock`); +} + +export function isLockBusyError(error) { + return /insights store is busy/.test(String(error?.message ?? "")); +} diff --git a/herdr-insights/mods/insights/indexer.mjs b/herdr-insights/mods/insights/indexer.mjs new file mode 100644 index 00000000..0d99aab3 --- /dev/null +++ b/herdr-insights/mods/insights/indexer.mjs @@ -0,0 +1,278 @@ +#!/usr/bin/env node +// Managed detached indexer for legitimate giant, non-seekable compressed +// archives that cannot finish inside a foreground exec slice. It keeps one +// streaming decoder alive to completion, owns the store writer lock, and +// commits completed files incrementally. No transcript text is stored. +// +// A decoder restart (after this process dies) re-reads only the open archive; +// Node exposes no zstd decoder-state snapshot, so continuing mid-frame is not +// promised. Polls never spawn or restart the decoder. +import { realpathSync } from "node:fs"; +import { stat } from "node:fs/promises"; +import path from "node:path"; +import { fileURLToPath } from "node:url"; +import { + clearWorkerClaim, + readEffectiveFence, + readWorkerClaim, + storeGeneration, +} from "./core/fence.mjs"; +import { progressWriter } from "./core/progress.mjs"; +import { scanSources } from "./core/scan.mjs"; +import { + emptyScanState, + ensureStoreMeta, + loadStore, + saveStore, +} from "./core/store.mjs"; +import { isLockBusyError, withStoreLock } from "./core/util.mjs"; + +const SAVE_INTERVAL_MS = 1_500; +const FENCE_POLL_MS = 250; +const CLAIM_WAIT_MS = 5_000; + +function parseArgs(argv) { + const options = { testStreamDelayMs: 0 }; + for (let index = 0; index < argv.length; index += 1) { + const arg = argv[index]; + const match = /^--(store|home|generation|claim-token|test-stream-delay-ms)(?:=(.*))?$/.exec(arg); + if (!match) throw new Error(`unknown argument ${arg}`); + let value = match[2]; + if (value === undefined) { + index += 1; + value = argv[index]; + if (value === undefined) throw new Error(`--${match[1]} needs a value`); + } + options[match[1]] = value; + } + for (const required of ["store", "home", "generation", "claim-token"]) { + if (options[required] === undefined) throw new Error(`--${required} is required`); + } + const generation = Number(options.generation); + if (!Number.isInteger(generation) || generation < 0) { + throw new Error("--generation must be a non-negative integer"); + } + const delay = Number(options["test-stream-delay-ms"]); + return { + storePath: path.resolve(options.store), + home: path.resolve(options.home), + generation, + claimToken: options["claim-token"], + testStreamDelayMs: Number.isFinite(delay) && delay > 0 ? delay : 0, + }; +} + +async function waitForClaim(storePath, token) { + const deadline = Date.now() + CLAIM_WAIT_MS; + for (;;) { + const claim = await readWorkerClaim(storePath); + if (!claim || claim.token !== token) return { ok: false, reason: "claim-replaced" }; + if (claim.pid === process.pid) return { ok: true, claim }; + if (Number.isInteger(claim.pid) && claim.pid > 0) return { ok: false, reason: "claim-other-pid" }; + if (Date.now() >= deadline) return { ok: false, reason: "claim-timeout" }; + await new Promise((resolve) => setTimeout(resolve, 40)); + } +} + +function scanViewUpdate(store, state, { status = "running" } = {}) { + const scan = store.scan; + scan.status = status; + scan.generation = storeGeneration(store); + scan.updatedAt = new Date().toISOString(); + scan.processed = Number.isInteger(state?.processed) ? state.processed : scan.processed; + scan.total = Number.isInteger(state?.total) ? state.total : scan.total; + if (state?.stats) scan.stats = state.stats; + if (state?.sourceStats) scan.sourceStats = state.sourceStats; + if (Array.isArray(state?.warnings)) scan.warnings = state.warnings; + scan.currentFile = state?.file ?? null; + return scan; +} + +export async function runIndexer(argv) { + const options = parseArgs(argv); + const { storePath, home, generation, claimToken, testStreamDelayMs } = options; + const claim = await waitForClaim(storePath, claimToken); + if (!claim.ok) return { exitCode: 3, reason: claim.reason }; + + let stopping = false; + let fenceAborted = false; + let fenceCheck = null; + const onSignal = () => { stopping = true; }; + process.on("SIGTERM", onSignal); + process.on("SIGINT", onSignal); + + const progress = progressWriter(storePath); + let lastSave = Date.now(); + try { + const result = await withStoreLock(storePath, async () => { + const loaded = await loadStore(storePath); + const store = loaded.store; + ensureStoreMeta(store); + if (storeGeneration(store) !== generation) { + return { exitCode: 0, reason: "stale-generation" }; + } + const startFence = await readEffectiveFence(storePath); + if (startFence && !startFence.stale && startFence.generation > generation) { + return { exitCode: 0, reason: "fenced" }; + } + + store.scan.status = "running"; + store.scan.workerPid = process.pid; + store.scan.generation = generation; + store.scan.startedAt = store.scan.startedAt ?? new Date().toISOString(); + store.scan.currentFile = null; + store.scan.error = null; + if (store.scan.pendingFile?.path) { + const pendingStat = await stat(store.scan.pendingFile.path).catch(() => null); + if (!pendingStat?.isFile()) store.scan.pendingFile = null; + } + const startedIso = new Date().toISOString(); + await saveStore(storePath, store, { status: "running", now: startedIso }); + + fenceCheck = async () => { + if (fenceAborted) return; + const fence = await readEffectiveFence(storePath); + if (fence && !fence.stale && fence.generation > generation) fenceAborted = true; + }; + const fenceTimer = setInterval(() => { + if (fenceAborted || stopping) return; + fenceCheck().catch(() => { /* best-effort fence poll */ }); + }, FENCE_POLL_MS); + fenceTimer.unref(); + + let scanned; + try { + scanned = await scanSources(store, { + home, + deadline: null, + mode: "worker", + streamDelayMs: testStreamDelayMs, + shouldStop: () => stopping || fenceAborted, + onProgress: (state) => { + progress.update({ + ...state, + mode: "worker", + workerPid: process.pid, + generation, + }); + }, + onStreamProgress: (state) => { + progress.update({ + ...state, + mode: "worker", + workerPid: process.pid, + generation, + }); + }, + onCheckpoint: async (state) => { + scanViewUpdate(store, state); + store.scan.workerPid = process.pid; + const finished = new Date().toISOString(); + progress.update({ + processed: state.processed, + total: state.total, + file: state.file, + filesRead: state.stats?.filesRead ?? 0, + bytesRead: state.stats?.bytesRead ?? 0, + mode: "worker", + workerPid: process.pid, + generation, + phase: "scan", + }); + if (Date.now() - lastSave >= SAVE_INTERVAL_MS) { + // Never persist a checkpoint after this generation has been + // fenced by configure/rebuild; fail closed if the fence cannot + // be read so an old generation cannot leak a late commit. + try { + await fenceCheck(); + } catch { + return; + } + if (fenceAborted) return; + await saveStore(storePath, store, { status: "running", now: finished }); + lastSave = Date.now(); + } + }, + }); + } finally { + clearInterval(fenceTimer); + } + + await fenceCheck(); + const finishedIso = new Date().toISOString(); + scanViewUpdate(store, scanned, { + status: scanned.complete ? "complete" : "partial", + }); + store.scan.currentFile = null; + store.scan.workerPid = null; + if (scanned.abortedFile) { + store.scan.pendingFile = { ...scanned.abortedFile, requestedAt: finishedIso }; + store.scan.status = "partial"; + } else if (scanned.complete) { + store.scan.pendingFile = null; + store.scan.completedAt = finishedIso; + } + if (fenceAborted) { + // A configure/rebuild owns the next generation. Leave the last + // committed files intact and write nothing from this open generation. + return { exitCode: 0, reason: "fenced" }; + } + if (scanned.complete) { + await saveStore(storePath, store, { completed: true, now: finishedIso }); + } else { + await saveStore(storePath, store, { status: "partial", now: finishedIso }); + } + return { + exitCode: 0, + reason: scanned.complete ? "complete" : (stopping ? "stopped" : "partial"), + }; + }, { + timeoutMs: 0, + meta: { role: "worker", generation }, + }); + return result ?? { exitCode: 0 }; + } catch (error) { + if (isLockBusyError(error)) return { exitCode: 0, reason: "lock-busy" }; + try { + const loaded = await loadStore(storePath); + const store = loaded.store; + ensureStoreMeta(store); + if (storeGeneration(store) === generation) { + if (!store.scan) store.scan = emptyScanState(); + store.scan.status = "failed"; + store.scan.workerPid = null; + store.scan.error = error.message; + store.scan.updatedAt = new Date().toISOString(); + await saveStore(storePath, store, { status: "failed" }); + } + } catch { /* failure reporting is best-effort */ } + return { exitCode: 1, reason: "error", error }; + } finally { + progress.clear(); + process.off("SIGTERM", onSignal); + process.off("SIGINT", onSignal); + await clearWorkerClaim(storePath, claimToken).catch(() => {}); + } +} + +async function main() { + try { + const result = await runIndexer(process.argv.slice(2)); + process.exitCode = result.exitCode ?? 0; + } catch (error) { + process.stderr.write(`insights indexer: ${error.message}\n`); + process.exitCode = 1; + } +} + +function isInvokedDirectly() { + const entry = process.argv[1]; + if (!entry) return false; + try { + return realpathSync(entry) === realpathSync(fileURLToPath(import.meta.url)); + } catch { + return false; + } +} + +if (isInvokedDirectly()) await main(); diff --git a/herdr-insights/mods/insights/main.js b/herdr-insights/mods/insights/main.js index c48ce35f..f13aeca9 100644 --- a/herdr-insights/mods/insights/main.js +++ b/herdr-insights/mods/insights/main.js @@ -485,6 +485,7 @@ export default class Insights extends View { this.dashboardTimer = null; this.dashboardTask = null; this.progressTimer = null; + this.scanContinueTimer = null; this.scanProgress = null; this.progressTask = null; this.progressGeneration = 0; @@ -548,6 +549,7 @@ export default class Insights extends View { try { this.dashboardWatchdog?.cancel(); } catch { /* already gone */ } try { this.refreshWatchdog?.cancel(); } catch { /* already gone */ } try { this.progressTimer?.cancel(); } catch { /* already gone */ } + this.cancelScanContinue(); try { this.saveWatchdog?.cancel(); } catch { /* already gone */ } try { this.settingsTask?.cancel(); } catch { /* already gone */ } try { this.commandTask?.cancel(); } catch { /* already gone */ } @@ -773,6 +775,51 @@ export default class Insights extends View { this.scanProgress = null; } + // A bounded collector slice returns committed data with an explicit scan + // state. Continue automatically while work remains; the next request + // coalesces with any in-flight pane through the store lock. + cancelScanContinue() { + try { this.scanContinueTimer?.cancel(); } catch { /* already gone */ } + this.scanContinueTimer = null; + } + + scheduleScanContinue(cx) { + const scan = this.snapshot?.scan; + if (!scan || this.closed) { + this.cancelScanContinue(); + return; + } + const delay = scan.status === "partial" ? 250 + : scan.status === "running" || scan.status === "starting" ? 900 + : scan.status === "reconfiguring" ? 500 + : null; + if (delay === null) { + this.cancelScanContinue(); + return; + } + if (this.scanContinueTimer) return; + this.scanContinueTimer = cx.timer.after(delay, (timerCx) => { + this.scanContinueTimer = null; + if (!this.closed) this.refresh(timerCx); + }); + } + + scanStatusLabel(scan) { + if (!scan || typeof scan !== "object") return ""; + if (scan.status === "reconfiguring") return "Applying configuration…"; + if (scan.status === "running" || scan.status === "starting") { + return Number(scan.total) > 0 + ? `Indexing ${formatInt(scan.processed)} of ${formatInt(scan.total)} files…` + : "Indexing in the background…"; + } + if (scan.status === "partial") { + return Number(scan.total) > 0 + ? `Scanning ${formatInt(scan.processed)} of ${formatInt(scan.total)} files…` + : "Scanning…"; + } + return ""; + } + armWatchdog(kind, cx) { const field = kind === "refresh" ? "refreshWatchdog" : "saveWatchdog"; try { this[field]?.cancel(); } catch { /* already gone */ } @@ -908,6 +955,7 @@ export default class Insights extends View { return; } if (this.busy()) return; + this.cancelScanContinue(); const generation = (this.activityGeneration ?? 0) + 1; this.activityGeneration = generation; this.loading = true; @@ -997,6 +1045,7 @@ export default class Insights extends View { // Only compact metadata and the prepared dashboard are applied here. The // native settings controls are created or synced by a separate Task after // this notification, so they can never interrupt the first dashboard frame. + this.scheduleScanContinue(cx); cx.notify(); if (this.showSettings) this.scheduleSettingsSync(cx); } @@ -1342,14 +1391,18 @@ export default class Insights extends View { } header(c) { - const updated = this.snapshot ? formatTimestamp(this.snapshot.lastScanAt || this.snapshot.generatedAt) : ""; + const scan = this.snapshot?.scan; + const updated = this.snapshot + ? formatTimestamp(this.snapshot.lastCompletedScanAt || this.snapshot.lastScanAt || this.snapshot.generatedAt) + : ""; const preparing = !this.dashboard && (this.dashboardPending || this.dashboardQueued); + const scanning = this.scanStatusLabel(scan); const status = this.loading ? "Refreshing…" : this.saving ? "Saving…" : preparing ? "Preparing…" : this.error ? (this.snapshot ? "Showing last snapshot" : "Unavailable") : this.dashboardError ? "Could not prepare dashboard" - : updated ? `Updated ${updated}` : ""; + : scanning || (updated ? `Updated ${updated}` : ""); const statusError = Boolean(this.error || this.dashboardError); return h_flex().w_full().flex_wrap().gap(G.md).items_center().justify_between() .child(pageTitle(this.showSettings ? "Settings" : "Insights", c)) diff --git a/herdr-insights/mods/insights/presenter.mjs b/herdr-insights/mods/insights/presenter.mjs index 6195f6fc..f45639da 100644 --- a/herdr-insights/mods/insights/presenter.mjs +++ b/herdr-insights/mods/insights/presenter.mjs @@ -13,8 +13,7 @@ import { compactDashboard, deriveDashboard, } from "./analytics.js"; -import { collectSnapshot } from "./collector.mjs"; -import { loadStore, snapshotRows } from "./core/store.mjs"; +import { collectSnapshot, readCommittedSnapshot } from "./collector.mjs"; function usage() { return [ @@ -22,6 +21,7 @@ function usage() { " presenter.mjs snapshot --store --home [--options ]", " presenter.mjs configure --store --home --config [--options ]", " presenter.mjs prepare --store [--options ]", + " presenter.mjs status --store [--options ]", ].join("\n"); } @@ -71,15 +71,20 @@ function compactSnapshot(snapshot, rowCount = 0) { version = 1, generatedAt = null, lastScanAt = null, + lastCompletedScanAt = null, + savedAt = null, config = {}, stats, sourceStats, + scan, warnings = [], } = snapshot ?? {}; return { version, generatedAt, lastScanAt, + lastCompletedScanAt, + savedAt, config, rowCount: Array.isArray(snapshot?.rows) ? snapshot.rows.length @@ -88,6 +93,7 @@ function compactSnapshot(snapshot, rowCount = 0) { : 0, stats, sourceStats, + scan: scan && typeof scan === "object" ? scan : null, warnings: Array.isArray(warnings) ? warnings : [], }; } @@ -104,19 +110,6 @@ function presentation(command, snapshot, options, dashboard, error) { }; } -function storeSnapshot(store, loaded) { - const rows = snapshotRows(store); - return { - version: 1, - generatedAt: store.lastScanAt, - lastScanAt: store.lastScanAt, - config: store.config, - rows, - stats: loaded?.errors ? { errors: loaded.errors } : undefined, - warnings: loaded?.warnings ?? [], - }; -} - function derive(snapshot, options) { try { return { dashboard: deriveDashboard(snapshot, options), error: null }; @@ -130,7 +123,7 @@ export async function run(argv, environment = {}) { if (options.help || !command) { return { exitCode: 0, stdout: "", stderr: `${usage()}\n` }; } - if (!["snapshot", "configure", "prepare"].includes(command)) { + if (!["snapshot", "configure", "prepare", "status"].includes(command)) { return { exitCode: 2, stdout: "", stderr: `unknown command ${command}\n${usage()}\n` }; } if (!options.store) { @@ -144,13 +137,13 @@ export async function run(argv, environment = {}) { const home = path.resolve(options.home ?? environment.home ?? os.homedir()); const dashboardOptions = parseOptions(options.options); - if (command === "prepare") { - const loaded = await loadStore(storePath, { readOnly: true }); - const snapshot = storeSnapshot(loaded.store, loaded); + if (command === "prepare" || command === "status") { + // Never takes the writer lock: a long indexer must not block the cached UI. + const snapshot = await readCommittedSnapshot(storePath); const derived = derive(snapshot, dashboardOptions); return { exitCode: 0, - stdout: `${JSON.stringify(presentation("prepare", snapshot, dashboardOptions, derived.dashboard, derived.error))}\n`, + stdout: `${JSON.stringify(presentation(command, snapshot, dashboardOptions, derived.dashboard, derived.error))}\n`, stderr: "", }; } diff --git a/tests/herdr-insights/collector-bounded.test.mjs b/tests/herdr-insights/collector-bounded.test.mjs new file mode 100644 index 00000000..2f61d2a3 --- /dev/null +++ b/tests/herdr-insights/collector-bounded.test.mjs @@ -0,0 +1,253 @@ +import assert from "node:assert/strict"; +import { mkdir, mkdtemp, readFile, stat, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import path from "node:path"; +import test from "node:test"; + +import { collectSnapshot } from "../../herdr-insights/mods/insights/collector.mjs"; +import { normalizeConfig } from "../../herdr-insights/mods/insights/core/config.mjs"; +import { scanSources } from "../../herdr-insights/mods/insights/core/scan.mjs"; +import { emptyStore, snapshotRows } from "../../herdr-insights/mods/insights/core/store.mjs"; + +const piSource = [{ + id: "pi", + label: "Pi", + adapter: "pi", + enabled: true, + roots: ["~/.pi/agent/sessions"], +}]; + +function piSession(sessionId, input, timestamp = "2026-04-01T10:00:00Z") { + return [ + JSON.stringify({ type: "session", id: sessionId, timestamp, cwd: "/repo/bounded" }), + JSON.stringify({ + type: "message", + id: `user-${sessionId}`, + timestamp, + message: { role: "user", content: "prompt" }, + }), + JSON.stringify({ + type: "message", + id: `assistant-${sessionId}`, + timestamp, + message: { + role: "assistant", + model: "bounded-model", + provider: "bounded-provider", + usage: { input, output: 1, cacheRead: 0, cacheWrite: 0, reasoning: 0 }, + stopReason: "stop", + content: [], + }, + }), + ].map((line) => `${line}\n`).join(""); +} + +async function context(prefix) { + const root = await mkdtemp(path.join(tmpdir(), prefix)); + const home = path.join(root, "home"); + const store = path.join(root, "data", "insights.json"); + await mkdir(path.join(home, ".pi", "agent", "sessions"), { recursive: true }); + return { root, home, store }; +} + +function sourceConfig() { + return { sources: piSource }; +} + +test("tiny deterministic budgets converge to the one-shot oracle without retries", async () => { + const bounded = await context("insights-bounded-"); + for (let index = 0; index < 7; index += 1) { + await writeFile( + path.join(bounded.home, ".pi", "agent", "sessions", `s${index}.jsonl`), + piSession(`s${index}`, index + 1), + ); + } + const oracleStore = path.join(bounded.root, "oracle", "insights.json"); + const oracle = await collectSnapshot("configure", { + store: oracleStore, + home: bounded.home, + config: sourceConfig(), + }); + assert.equal(oracle.scan.status, "complete"); + + let tick = Date.now(); + const fastClock = () => { + tick += 1_000; + return tick; + }; + const first = await collectSnapshot("configure", { + store: bounded.store, + home: bounded.home, + config: sourceConfig(), + budgetMs: 1, + now: fastClock, + }); + assert.equal(first.scan.status, "partial", "the first slice is bounded"); + + const statuses = [first.scan.status]; + let snapshot = first; + for (let attempt = 0; attempt < 30 && snapshot.scan.status !== "complete"; attempt += 1) { + snapshot = await collectSnapshot("snapshot", { + store: bounded.store, + home: bounded.home, + budgetMs: 1, + now: fastClock, + }); + statuses.push(snapshot.scan.status); + } + assert.equal(snapshot.scan.status, "complete"); + assert.ok(statuses.length > 2, `expected several bounded slices, got ${statuses.join(",")}`); + assert.deepEqual(snapshot.rows, oracle.rows); + assert.equal(snapshot.scan.pendingFile, null); + assert.equal(snapshot.stats.errors, 0); + + const saved = JSON.parse(await readFile(bounded.store, "utf8")); + assert.equal(saved.scan.status, "complete"); + assert.equal(typeof saved.lastCompletedScanAt, "string"); + assert.equal(saved.scan.pendingFile, null); +}); + +test("plain JSONL continuation reads only the unfinished tail and is not fingerprinted as complete", async () => { + const contextInfo = await context("insights-plain-tail-"); + const file = path.join(contextInfo.home, ".pi", "agent", "sessions", "big.jsonl"); + const filler = JSON.stringify({ type: "event", payload: "x".repeat(1024) }); + const lines = [piSession("plain-tail", 5)]; + for (let index = 0; index < 2_400; index += 1) lines.push(`${filler}\n`); + await writeFile(file, lines.join("")); + const totalSize = (await stat(file)).size; + assert.ok(totalSize > 2_000_000, "fixture must exceed one read chunk"); + + const store = emptyStore(normalizeConfig(sourceConfig())); + const first = await scanSources(store, { + home: contextInfo.home, + deadline: Date.now(), + now: Date.now, + }); + assert.equal(first.complete, false); + assert.equal(first.stopped, true); + const entry = Object.values(store.files)[0]; + assert.ok(entry.offset > 0 && entry.offset < totalSize, "offset advanced inside the file"); + assert.equal(entry.state.scanPending, true, "unfinished tail is not marked complete"); + assert.notEqual(entry.size, totalSize, "an unfinished tail must not match the file fingerprint"); + assert.equal(snapshotRows(store)[0].messages, 1, "accepted prefix contributes exactly once"); + const accepted = entry.offset; + + const second = await scanSources(store, { home: contextInfo.home }); + assert.equal(second.complete, true); + assert.equal(second.stats.bytesRead, totalSize - accepted, "only the unfinished tail is read"); + assert.equal(second.stats.filesRead, 1); + assert.equal(entry.offset, totalSize); + assert.equal(entry.size, totalSize); + assert.equal(entry.state.scanPending, undefined); + assert.equal(snapshotRows(store)[0].messages, 1, "continuation does not replay the prefix"); + + const third = await scanSources(store, { home: contextInfo.home }); + assert.equal(third.stats.bytesRead, 0); + assert.equal(third.stats.filesRead, 0); +}); + +test("partial scans expose savedAt separately from the last completed scan", async () => { + const contextInfo = await context("insights-timestamps-"); + for (let index = 0; index < 3; index += 1) { + await writeFile( + path.join(contextInfo.home, ".pi", "agent", "sessions", `s${index}.jsonl`), + piSession(`t${index}`, index + 1), + ); + } + let tick = Date.parse("2026-04-02T00:00:00Z"); + const fastClock = () => { + tick += 5_000; + return tick; + }; + const partial = await collectSnapshot("configure", { + store: contextInfo.store, + home: contextInfo.home, + config: sourceConfig(), + budgetMs: 1, + now: fastClock, + }); + assert.equal(partial.scan.status, "partial"); + assert.equal(partial.lastCompletedScanAt, null); + const saved = JSON.parse(await readFile(contextInfo.store, "utf8")); + assert.equal(saved.scan.status, "partial"); + assert.equal(saved.lastCompletedScanAt, null); + assert.equal(typeof saved.lastScanAt, "string"); + assert.equal(partial.lastScanAt, saved.lastScanAt); + assert.equal(partial.savedAt, saved.lastScanAt); + assert.equal(typeof partial.generatedAt, "string"); + + let complete = await collectSnapshot("snapshot", { + store: contextInfo.store, + home: contextInfo.home, + budgetMs: 60_000, + now: fastClock, + }); + for (let attempt = 0; attempt < 10 && complete.scan.status !== "complete"; attempt += 1) { + complete = await collectSnapshot("snapshot", { + store: contextInfo.store, + home: contextInfo.home, + budgetMs: 60_000, + now: fastClock, + }); + } + assert.equal(complete.scan.status, "complete"); + assert.equal(typeof complete.lastCompletedScanAt, "string"); + assert.ok(Date.parse(complete.lastCompletedScanAt) >= Date.parse(saved.lastScanAt)); + + const beforeWarm = await stat(contextInfo.store); + const warm = await collectSnapshot("snapshot", { + store: contextInfo.store, + home: contextInfo.home, + budgetMs: 60_000, + now: fastClock, + }); + const afterWarm = await stat(contextInfo.store); + assert.equal(warm.scan.status, "complete"); + assert.equal(warm.stats.filesRead, 0); + assert.equal(afterWarm.mtimeMs, beforeWarm.mtimeMs, "no-op refresh must not rewrite the store"); + assert.equal(afterWarm.size, beforeWarm.size); + assert.equal(warm.lastCompletedScanAt, complete.lastCompletedScanAt); +}); + +test("a bounded scan never exposes rows from an unfinished archive", async () => { + const { zstdCompressSync } = await import("node:zlib"); + const contextInfo = await mkdtemp(path.join(tmpdir(), "insights-archive-visibility-")); + const home = path.join(contextInfo, "home"); + const sessions = path.join(home, ".codex", "sessions"); + await mkdir(sessions, { recursive: true }); + const lines = [ + JSON.stringify({ type: "session_meta", timestamp: "2026-04-03T00:00:00Z", payload: { id: "archive", cwd: "/repo", model_provider: "openai" } }), + JSON.stringify({ type: "turn_context", timestamp: "2026-04-03T00:00:01Z", payload: { turn_id: "t", model: "archive-model", cwd: "/repo" } }), + ]; + for (let index = 0; index < 20_000; index += 1) { + lines.push(JSON.stringify({ + type: "event_msg", + timestamp: "2026-04-03T00:00:02Z", + payload: { + type: "token_count", + info: { + last_token_usage: { input_tokens: 1, cached_input_tokens: 0, cache_write_input_tokens: 0, output_tokens: 0, reasoning_output_tokens: 0 }, + total_token_usage: { input_tokens: index + 1, cached_input_tokens: 0, cache_write_input_tokens: 0, output_tokens: 0, reasoning_output_tokens: 0 }, + }, + }, + })); + } + await writeFile(path.join(sessions, "big.jsonl.zst"), zstdCompressSync(lines.join("\n"))); + const store = emptyStore(normalizeConfig({ + sources: [{ + id: "codex", + label: "Codex", + adapter: "codex", + enabled: true, + roots: ["~/.codex/sessions"], + }], + })); + const result = await scanSources(store, { + home, + deadline: Date.now(), + now: Date.now, + }); + assert.equal(result.abortedFile?.path?.endsWith("big.jsonl.zst"), true); + assert.equal(Object.keys(store.files).length, 0, "an aborted archive contributes no file entry"); + assert.deepEqual(snapshotRows(store), []); +}); diff --git a/tests/herdr-insights/collector-transactions.test.mjs b/tests/herdr-insights/collector-transactions.test.mjs new file mode 100644 index 00000000..74a3786e --- /dev/null +++ b/tests/herdr-insights/collector-transactions.test.mjs @@ -0,0 +1,191 @@ +import assert from "node:assert/strict"; +import { readFile, mkdir, mkdtemp, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import path from "node:path"; +import { fileURLToPath } from "node:url"; +import { zstdCompressSync } from "node:zlib"; +import test from "node:test"; + +import { + addTask, + addUsage, + beginStoreTransaction, + commitStoreTransaction, + dedupKeyFor, + emptyStore, + findFileById, + newFileEntry, + removeLatestFallbackTaskForSession, + resetFileEntry, + rollbackStoreTransaction, + snapshotRows, + taskKeyFor, +} from "../../herdr-insights/mods/insights/core/store.mjs"; +import { scanSources } from "../../herdr-insights/mods/insights/core/scan.mjs"; + +const here = path.dirname(fileURLToPath(import.meta.url)); + +function meta(overrides = {}) { + return { + day: "2026-04-01", + session: "session-a", + harness: "codex", + provider: "openai", + model: "model-a", + project: "/repo/a", + ...overrides, + }; +} + +function seedStore() { + const store = emptyStore(); + const first = newFileEntry(store, { + sourceId: "codex", adapter: "codex", path: "/a.jsonl", size: 10, mtimeMs: 1, + }); + const second = newFileEntry(store, { + sourceId: "codex", adapter: "codex", path: "/b.jsonl", size: 10, mtimeMs: 1, + }); + first.state.sessions = { "codex:a": { turnStart: 1, turnFirst: 1, lastRow: null } }; + const usageKey = dedupKeyFor("codex", "response-a"); + addUsage(store, first, meta(), { input: 10, output: 2, cacheRead: 0, cacheWrite: 0, reasoning: 0 }, usageKey); + addUsage(store, second, meta(), { input: 10, output: 2, cacheRead: 0, cacheWrite: 0, reasoning: 0 }, usageKey); + addTask(store, first.id, 0, taskKeyFor("codex", "turn-1"), 1000, "codex:a", false, first.id); + addTask(store, first.id, 0, taskKeyFor("codex", "turn-2"), 500, "codex:a", false, first.id); + return { store, first, second, usageKey }; +} + +test("per-file transaction rollback restores rows, dedup, tasks, ownership and nextFileId", () => { + const { store, first, second } = seedStore(); + const before = JSON.stringify(store); + const beforeRows = JSON.stringify(snapshotRows(store)); + const beforeDedup = JSON.stringify(store.dedup); + const beforeTasks = JSON.stringify(store.tasks); + const beforeNext = store.nextFileId; + + const tx = beginStoreTransaction(store, { fileId: first.id }); + // Mutate the open file, a shared dedup owner, a sibling copy, and task state. + resetFileEntry(store, first); + const bumped = dedupKeyFor("codex", "response-a"); + addUsage(store, first, meta({ provider: "concrete" }), { + input: 999, output: 99, cacheRead: 9, cacheWrite: 9, reasoning: 9, + }, bumped); + addUsage(store, second, meta(), { + input: 11, output: 3, cacheRead: 0, cacheWrite: 0, reasoning: 0, + }, bumped); + addTask(store, second.id, 0, taskKeyFor("codex", "turn-1"), 5000, "codex:a", true, second.id); + removeLatestFallbackTaskForSession(store, "codex:a"); + const created = newFileEntry(store, { + sourceId: "codex", adapter: "codex", path: "/c.jsonl", size: 0, mtimeMs: 0, + }); + created.state.scratch = true; + assert.notEqual(store.nextFileId, beforeNext); + assert.ok(findFileById(store, created.id)); + + assert.equal(rollbackStoreTransaction(tx), true); + assert.equal(JSON.stringify(store), before, "every touched store record is restored"); + assert.equal(JSON.stringify(snapshotRows(store)), beforeRows); + assert.equal(JSON.stringify(store.dedup), beforeDedup); + assert.equal(JSON.stringify(store.tasks), beforeTasks); + assert.equal(store.nextFileId, beforeNext); + assert.equal(findFileById(store, created.id), null); + assert.ok(findFileById(store, first.id)); + assert.ok(findFileById(store, second.id)); + assert.equal(first.rows[0].input, 10); + assert.equal(first.state.sessions["codex:a"].turnStart, 1); +}); + +test("rollback restores a stale dedup entry that a new owner replaced", () => { + const { store, first } = seedStore(); + const staleKey = dedupKeyFor("codex", "stale-response"); + // A committed entry whose owner no longer exists is discarded on reuse. + store.dedup[staleKey] = [9999, 0, 5, 1, 0, 0, 0]; + const before = JSON.stringify(store.dedup); + + const tx = beginStoreTransaction(store, { fileId: first.id }); + addUsage(store, first, meta(), { + input: 7, output: 1, cacheRead: 0, cacheWrite: 0, reasoning: 0, + }, staleKey); + assert.equal(store.dedup[staleKey][0], first.id, "the stale owner was replaced"); + + assert.equal(rollbackStoreTransaction(tx), true); + assert.equal(JSON.stringify(store.dedup), before, "the stale entry comes back unchanged"); +}); + +test("per-file transaction commit keeps mutations and releases the transaction", () => { + const { store, first } = seedStore(); + const tx = beginStoreTransaction(store, { fileId: first.id }); + addUsage(store, first, meta(), { + input: 50, output: 5, cacheRead: 0, cacheWrite: 0, reasoning: 0, + }, dedupKeyFor("codex", "response-new")); + assert.equal(commitStoreTransaction(tx), true); + assert.equal(store._tx, undefined); + assert.equal(snapshotRows(store)[0].tokens.input, 60); + assert.equal(rollbackStoreTransaction(tx), false, "a committed transaction cannot roll back"); +}); + +test("a corrupt archive after shared revisions preserves sibling rows, dedup and tasks", async () => { + const root = await mkdtemp(path.join(tmpdir(), "insights-transaction-")); + const home = path.join(root, "home"); + const sessions = path.join(home, ".codex", "sessions"); + await mkdir(sessions, { recursive: true }); + const plain = await readFile(path.join(here, "fixtures", "codex-modern.jsonl"), "utf8"); + const good = zstdCompressSync(plain); + const first = path.join(sessions, "a.jsonl.zst"); + const second = path.join(sessions, "b.jsonl.zst"); + await writeFile(first, good); + await writeFile(second, good); + + const store = emptyStore(); + store.config = { + ...store.config, + sources: [{ id: "codex", label: "Codex", adapter: "codex", enabled: true, roots: ["~/.codex/sessions"] }], + }; + const initial = await scanSources(store, { home }); + assert.equal(initial.complete, true); + assert.equal(initial.stats.errors, 0); + const beforeRows = JSON.stringify(snapshotRows(store)); + const beforeDedup = JSON.stringify(store.dedup); + const beforeTasks = JSON.stringify(store.tasks); + const entryFor = (file) => Object.values(store.files).find( + (item) => item.path === file || item.path.endsWith(path.basename(file)), + ); + const beforeSecond = JSON.stringify(entryFor(second)); + + // Compress a revision that would raise the shared usage before failing. + const revised = plain.replaceAll('"input_tokens":100', '"input_tokens":999'); + assert.notEqual(revised, plain); + await writeFile(second, Buffer.concat([ + zstdCompressSync(revised), + Buffer.from([0x28, 0xb5, 0x2f, 0xfd, 0]), + ])); + const corrupted = await scanSources(store, { home }); + assert.equal(corrupted.stats.errors, 1); + assert.equal(JSON.stringify(snapshotRows(store)), beforeRows); + assert.equal(JSON.stringify(store.dedup), beforeDedup); + assert.equal(JSON.stringify(store.tasks), beforeTasks); + const entry = entryFor(second); + assert.ok(entry); + assert.deepEqual(entry.rows, JSON.parse(beforeSecond).rows); + assert.deepEqual(entry.tasks, JSON.parse(beforeSecond).tasks); + assert.equal(entry.state.zstdError !== undefined, true, "corrupt fingerprint is recorded"); + + // The same broken fingerprint is not repeatedly re-read or re-warned. + const repeated = await scanSources(store, { home }); + assert.equal(repeated.stats.errors, 0); + assert.equal(repeated.stats.filesRead, 0); + assert.equal(JSON.stringify(snapshotRows(store)), beforeRows); +}); + +test("production scanning does not clone the accumulated store", async () => { + const scanSource = await readFile( + path.join(here, "../../herdr-insights/mods/insights/core/scan.mjs"), + "utf8", + ); + const collectorSource = await readFile( + path.join(here, "../../herdr-insights/mods/insights/collector.mjs"), + "utf8", + ); + assert.doesNotMatch(scanSource, /structuredClone\(\s*store\s*\)/); + assert.doesNotMatch(collectorSource, /structuredClone\(\s*store\s*\)/); + assert.doesNotMatch(scanSource, /Object\.assign\(store,\s*checkpoint\)/); +}); From b3f9023bc7e776af38bb1b278c14407edf171739 Mon Sep 17 00:00:00 2001 From: GaganSD Date: Mon, 2 Nov 2026 19:03:52 +0530 Subject: [PATCH 02/15] fix(insights): writer-authoritative worker failure and post-unlock promotion - Record indexer failure status inside the store lock, re-reading the committed generation; never write when the lock was never held. - Spawn the promoted indexer only after the launcher releases the lock; the worker waits a bounded handoff instead of exiting on EEXIST. - Make worker claims exclusive-create and token-verified on release instead of last-write-wins/check-then-unlink. - Fail closed on an unreadable generation fence during worker commits. - Regression tests: queued generation survives a failing worker; one worker survives promotion with a held lock, no respawn, non-blocking status. --- herdr-insights/mods/insights/collector.mjs | 75 +++--- herdr-insights/mods/insights/core/fence.mjs | 103 +++++++- .../mods/insights/core/indexer-control.mjs | 28 +- herdr-insights/mods/insights/indexer.mjs | 76 ++++-- .../collector-worker-lifecycle.test.mjs | 241 ++++++++++++++++++ 5 files changed, 452 insertions(+), 71 deletions(-) create mode 100644 tests/herdr-insights/collector-worker-lifecycle.test.mjs diff --git a/herdr-insights/mods/insights/collector.mjs b/herdr-insights/mods/insights/collector.mjs index e29f1882..f47b3537 100644 --- a/herdr-insights/mods/insights/collector.mjs +++ b/herdr-insights/mods/insights/collector.mjs @@ -359,24 +359,15 @@ async function scanAndPersist( ); if (completed && !mustSave && store.lastCompletedScanAt === null) mustSave = true; - let worker; const guarded = guard ? await guard() : null; if (guarded) return { fenced: true }; if (result.abortedFile) { - // Persist the abandoned file marker before the worker starts so a restart - // promotes the same file instead of repeating a foreground prefix. + // Persist the abandoned file marker under the writer lock; the worker is + // spawned by the caller only AFTER this lock is released so it can take + // ownership immediately instead of exiting on a busy lock. await saveStore(storePath, store, { status: "partial", now: finishedIso }); - worker = await ensureIndexer(storePath, home, { - generation: store.generation, - ackMs: 0, - testStreamDelayMs, - }); - if (worker.pid) { - store.scan.workerPid = worker.pid; - store.scan.status = "running"; - } const snapshot = buildSnapshot(store, result, finishedIso, loaded?.warnings, loaded?.errors); - return { snapshot, worker }; + return { snapshot, worker: null, promote: true }; } if (mustSave) { @@ -395,6 +386,20 @@ async function scanAndPersist( return { snapshot, worker: null }; } +async function promoteIndexer(storePath, home, snapshot, testStreamDelayMs = 0) { + // Called only after the caller released the store lock: the child is then + // the next writer and waits out any bounded handoff instead of livelocking. + const worker = await ensureIndexer(storePath, home, { + generation: snapshot.scan?.generation ?? null, + ackMs: 0, + testStreamDelayMs, + }); + if (worker.pid) { + snapshot.scan = { ...snapshot.scan, status: "running", workerPid: worker.pid }; + } + return worker; +} + async function finishWorkerAck(snapshot, worker, storePath, workerAckMs) { if (!worker?.pid || !worker.spawned) return snapshot; const ack = await waitForIndexerAck(storePath, worker.claim, { @@ -458,19 +463,8 @@ async function snapshotWithStore( if (await removeStalePendingFile(storePath, store, now)) dirty = true; if (store.scan.pendingFile) { - const worker = await ensureIndexer(storePath, home, { - generation: store.generation, - ackMs: 0, - testStreamDelayMs, - }); - if (worker.pid) { - store.scan.workerPid = worker.pid; - store.scan.status = "running"; - } const nowIso = new Date(now()).toISOString(); - if (dirty || worker.pid) { - await saveStore(storePath, store, { status: "running", now: nowIso }); - } + if (dirty) await saveStore(storePath, store, { status: "partial", now: nowIso }); const snapshot = buildSnapshot( store, scanResultFromStore(store), @@ -478,8 +472,8 @@ async function snapshotWithStore( loaded.warnings, loaded.errors, ); - snapshot.scan = { ...snapshot.scan, status: worker.pid ? "running" : "partial" }; - return { snapshot, worker, fenced: false }; + snapshot.scan = { ...snapshot.scan, status: "partial" }; + return { snapshot, worker: null, promote: true, fenced: false }; } const scanned = await scanAndPersist(storePath, home, store, { @@ -521,7 +515,10 @@ async function snapshotWithStore( if (rebuild && fence) await restoreFence(storePath, baseline); return busySnapshot(storePath, lock.owner, now); } - const { snapshot, worker } = lock.value; + const { snapshot, worker: lockedWorker, promote } = lock.value; + const worker = promote + ? await promoteIndexer(storePath, home, snapshot, testStreamDelayMs) + : (lockedWorker ?? null); return finishWorkerAck(snapshot, worker, storePath, workerAckMs); } @@ -591,19 +588,8 @@ async function configureStore( if (await removeStalePendingFile(storePath, store, now)) dirty = true; if (store.scan.pendingFile) { - const worker = await ensureIndexer(storePath, home, { - generation: store.generation, - ackMs: 0, - testStreamDelayMs, - }); - if (worker.pid) { - store.scan.workerPid = worker.pid; - store.scan.status = "running"; - } const nowIso = new Date(now()).toISOString(); - if (dirty || worker.pid) { - await saveStore(storePath, store, { status: "running", now: nowIso }); - } + if (dirty) await saveStore(storePath, store, { status: "partial", now: nowIso }); const snapshot = buildSnapshot( store, scanResultFromStore(store), @@ -611,8 +597,8 @@ async function configureStore( loaded.warnings, loaded.errors, ); - snapshot.scan = { ...snapshot.scan, status: worker.pid ? "running" : "partial" }; - return { snapshot, worker }; + snapshot.scan = { ...snapshot.scan, status: "partial" }; + return { snapshot, worker: null, promote: true }; } const scanned = await scanAndPersist(storePath, home, store, { @@ -654,7 +640,10 @@ async function configureStore( if (!applied) await restoreFence(storePath, baseline); throw new Error("insights store is busy; configuration was not applied"); } - const { snapshot, worker } = lock.value; + const { snapshot, worker: lockedWorker, promote } = lock.value; + const worker = promote + ? await promoteIndexer(storePath, home, snapshot, testStreamDelayMs) + : (lockedWorker ?? null); return finishWorkerAck(snapshot, worker, storePath, workerAckMs); } catch (error) { if (!applied) await restoreFence(storePath, baseline).catch(() => {}); diff --git a/herdr-insights/mods/insights/core/fence.mjs b/herdr-insights/mods/insights/core/fence.mjs index 39a4eaf7..ff054b7a 100644 --- a/herdr-insights/mods/insights/core/fence.mjs +++ b/herdr-insights/mods/insights/core/fence.mjs @@ -1,5 +1,6 @@ -import { unlink } from "node:fs/promises"; +import { link, open, readFile, rename, unlink } from "node:fs/promises"; import os from "node:os"; +import { randomBytes } from "node:crypto"; import { atomicWriteJson, isPlainObject, @@ -40,6 +41,32 @@ export function nextGeneration(...values) { return highest + 1; } +/** + * Strict fence read for writer authority. A missing fence is a legitimate + * "no configure queued" answer, but a corrupt/unreadable fence must NOT be + * treated as absent: callers that gate commits fail closed on the thrown error. + */ +export async function readFenceStrict(storePath) { + let text; + try { + text = await readFile(fencePath(storePath), "utf8"); + } catch (error) { + if (error?.code === "ENOENT") return null; + throw error; + } + let value; + try { + value = JSON.parse(text); + } catch (error) { + throw new Error(`insights generation fence is unreadable: ${error.message}`, { cause: error }); + } + if (!isPlainObject(value) || value.version !== FENCE_VERSION + || !Number.isInteger(value.generation) || value.generation < 1) { + throw new Error("insights generation fence is malformed"); + } + return value; +} + export async function readFence(storePath) { try { const value = await readJsonFile(fencePath(storePath)); @@ -112,18 +139,84 @@ export async function writeWorkerClaim(storePath, claim) { return claim; } -export async function clearWorkerClaim(storePath, token) { - const current = await readWorkerClaim(storePath); - if (!current || current.token !== token) return false; +/** + * Exclusive claim creation (O_CREAT|O_EXCL); never last-write-wins. Concurrent + * launchers either create the claim or observe EEXIST and coalesce. + */ +export async function tryCreateWorkerClaim(storePath, claim) { + const file = workerClaimPath(storePath); + let handle; try { - await unlink(workerClaimPath(storePath)); + handle = await open(file, "wx", 0o600); + await handle.writeFile(`${JSON.stringify(claim)}\n`, "utf8"); + await handle.sync(); + await handle.close(); + return true; + } catch (error) { + if (handle) await handle.close().catch(() => {}); + if (error?.code === "EEXIST") return false; + throw error; + } +} + +/** + * Record the spawned pid without clobbering a claim that was replaced + * concurrently: the open handle pins the inode, so a rename by a reaper makes + * this write land in the quarantined file instead of a newer claim. + */ +export async function updateWorkerClaimPid(storePath, token, pid) { + let handle; + try { + handle = await open(workerClaimPath(storePath), "r+"); + const text = await handle.readFile("utf8"); + let current; + try { current = JSON.parse(text); } catch { current = null; } + if (!isPlainObject(current) || current.token !== token) return false; + const next = { ...current, pid, at: Date.now() }; + await handle.truncate(0); + await handle.write(`${JSON.stringify(next)}\n`, 0, "utf8"); + await handle.sync(); return true; } catch (error) { if (error?.code === "ENOENT") return false; throw error; + } finally { + if (handle) await handle.close().catch(() => {}); } } +/** + * Token-safe release/reclaim. The claim file is renamed to a private + * quarantine name first (atomic, single winner), then verified by token. A + * claim that was replaced between observation and rename is restored with an + * exclusive link so it is never destroyed by a stale reaper. + */ +async function reapWorkerClaim(storePath, token) { + const file = workerClaimPath(storePath); + const quarantine = `${file}.${process.pid}-${randomBytes(6).toString("hex")}.reap`; + try { + await rename(file, quarantine); + } catch (error) { + if (error?.code === "ENOENT") return false; + throw error; + } + let current; + try { current = JSON.parse(await readFile(quarantine, "utf8")); } catch { current = null; } + if (current?.token !== token) { + // We moved a claim that had already been replaced. Put it back only if the + // slot is still empty; otherwise a newer claim exists and wins. + try { await link(quarantine, file); } catch { /* newer claim present */ } + await unlink(quarantine).catch(() => {}); + return false; + } + await unlink(quarantine).catch(() => {}); + return true; +} + +export async function clearWorkerClaim(storePath, token) { + return reapWorkerClaim(storePath, token); +} + /** * A claim is live while its recorded pid is alive. A claim without a pid is * only a spawn in progress; it expires quickly so a crashed launcher cannot diff --git a/herdr-insights/mods/insights/core/indexer-control.mjs b/herdr-insights/mods/insights/core/indexer-control.mjs index c57272d4..10c91189 100644 --- a/herdr-insights/mods/insights/core/indexer-control.mjs +++ b/herdr-insights/mods/insights/core/indexer-control.mjs @@ -7,8 +7,9 @@ import { readEffectiveFence, readWorkerClaim, storeGeneration, + tryCreateWorkerClaim, + updateWorkerClaimPid, workerClaimActive, - writeWorkerClaim, } from "./fence.mjs"; import { isProcessAlive, readStoreLockInfo, sleep } from "./util.mjs"; @@ -44,8 +45,13 @@ export async function ensureIndexer( } if (existing) { // A live claim (including a foreign host) is handled above. Only a - // provably dead local pid or an expired spawn placeholder reaches here. + // provably dead local pid or an expired spawn placeholder is reaped, and + // only under the exact token we observed. await clearWorkerClaim(storePath, existing.token); + const reread = await readWorkerClaim(storePath); + if (workerClaimActive(reread)) { + return { spawned: false, pid: Number.isInteger(reread.pid) ? reread.pid : null, claim: reread }; + } } const claim = { @@ -57,9 +63,14 @@ export async function ensureIndexer( startedAt: Date.now(), at: Date.now(), }; - // The worker waits briefly for its pid to be recorded, so writing the token - // before spawn makes concurrent launchers coalesce even during spawn. - await writeWorkerClaim(storePath, claim); + // The worker waits briefly for its pid to be recorded, so publishing the + // token before spawn makes concurrent launchers coalesce even during spawn. + // Exclusive create, not last-write-wins: a launcher that loses EEXIST never + // overwrites the winner's claim. + if (!(await tryCreateWorkerClaim(storePath, claim))) { + const other = await readWorkerClaim(storePath); + return { spawned: false, pid: Number.isInteger(other?.pid) ? other.pid : null, claim: other }; + } const args = [ testWorkerPath ?? indexerPath(), @@ -90,7 +101,12 @@ export async function ensureIndexer( const recorded = { ...claim, pid: child.pid, at: Date.now() }; try { - await writeWorkerClaim(storePath, recorded); + const published = await updateWorkerClaimPid(storePath, claim.token, child.pid); + if (!published) { + // The claim was reaped/replaced while spawning; do not resurrect it. The + // child will observe the missing/replaced token and exit without work. + return { spawned: false, pid: null, claim: recorded, lostClaim: true }; + } } catch (error) { await clearWorkerClaim(storePath, claim.token); return { spawned: false, pid: null, error }; diff --git a/herdr-insights/mods/insights/indexer.mjs b/herdr-insights/mods/insights/indexer.mjs index 0d99aab3..32bee306 100644 --- a/herdr-insights/mods/insights/indexer.mjs +++ b/herdr-insights/mods/insights/indexer.mjs @@ -14,6 +14,7 @@ import { fileURLToPath } from "node:url"; import { clearWorkerClaim, readEffectiveFence, + readFenceStrict, readWorkerClaim, storeGeneration, } from "./core/fence.mjs"; @@ -30,12 +31,17 @@ import { isLockBusyError, withStoreLock } from "./core/util.mjs"; const SAVE_INTERVAL_MS = 1_500; const FENCE_POLL_MS = 250; const CLAIM_WAIT_MS = 5_000; +// The launcher may still hold the store lock for a bounded moment after it +// published the pending file and spawned this process. Wait for that handoff +// instead of exiting on the first busy lock (which cleared the claim and +// livelocked promotion). +const HANDOFF_WAIT_MS = 5_000; function parseArgs(argv) { - const options = { testStreamDelayMs: 0 }; + const options = { testStreamDelayMs: 0, testFailAfterCheckpointMs: 0 }; for (let index = 0; index < argv.length; index += 1) { const arg = argv[index]; - const match = /^--(store|home|generation|claim-token|test-stream-delay-ms)(?:=(.*))?$/.exec(arg); + const match = /^--(store|home|generation|claim-token|test-stream-delay-ms|test-fail-after-checkpoint)(?:=(.*))?$/.exec(arg); if (!match) throw new Error(`unknown argument ${arg}`); let value = match[2]; if (value === undefined) { @@ -59,9 +65,33 @@ function parseArgs(argv) { generation, claimToken: options["claim-token"], testStreamDelayMs: Number.isFinite(delay) && delay > 0 ? delay : 0, + testFailAfterCheckpointMs: Number(options["test-fail-after-checkpoint"] ?? options.testFailAfterCheckpointMs ?? 0) || 0, }; } +/** + * Record a worker failure only under writer authority (inside the store lock), + * immediately re-reading the committed generation so a newer configure or + * rebuild can never be overwritten back to "failed". + */ +async function recordWorkerFailure(storePath, generation, error) { + try { + const loaded = await loadStore(storePath); + ensureStoreMeta(loaded.store); + if (storeGeneration(loaded.store) !== generation) return false; + const scan = loaded.store.scan ?? emptyScanState(); + loaded.store.scan = scan; + scan.status = "failed"; + scan.workerPid = null; + scan.error = error.message; + scan.updatedAt = new Date().toISOString(); + await saveStore(storePath, loaded.store, { status: "failed" }); + return true; + } catch { + return false; + } +} + async function waitForClaim(storePath, token) { const deadline = Date.now() + CLAIM_WAIT_MS; for (;;) { @@ -90,7 +120,7 @@ function scanViewUpdate(store, state, { status = "running" } = {}) { export async function runIndexer(argv) { const options = parseArgs(argv); - const { storePath, home, generation, claimToken, testStreamDelayMs } = options; + const { storePath, home, generation, claimToken, testStreamDelayMs, testFailAfterCheckpointMs } = options; const claim = await waitForClaim(storePath, claimToken); if (!claim.ok) return { exitCode: 3, reason: claim.reason }; @@ -105,12 +135,21 @@ export async function runIndexer(argv) { let lastSave = Date.now(); try { const result = await withStoreLock(storePath, async () => { + try { const loaded = await loadStore(storePath); const store = loaded.store; ensureStoreMeta(store); if (storeGeneration(store) !== generation) { return { exitCode: 0, reason: "stale-generation" }; } + // A corrupt or unreadable fence is not "no fence": fail closed before + // this generation writes anything, rather than trusting a helper that + // maps every read error to null. + try { + await readFenceStrict(storePath); + } catch (error) { + return { exitCode: 1, reason: "fence-unreadable", error }; + } const startFence = await readEffectiveFence(storePath); if (startFence && !startFence.stale && startFence.generation > generation) { return { exitCode: 0, reason: "fenced" }; @@ -131,6 +170,9 @@ export async function runIndexer(argv) { fenceCheck = async () => { if (fenceAborted) return; + // Throws if the fence exists but cannot be trusted; callers treat that + // as "do not commit" (fail closed). Missing fence stays legitimate. + await readFenceStrict(storePath); const fence = await readEffectiveFence(storePath); if (fence && !fence.stale && fence.generation > generation) fenceAborted = true; }; @@ -165,6 +207,12 @@ export async function runIndexer(argv) { }); }, onCheckpoint: async (state) => { + if (testFailAfterCheckpointMs > 0) { + // Test hook: keep writer authority for a bounded window so a + // concurrent client can queue on the lock, then fail. + await new Promise((resolve) => setTimeout(resolve, testFailAfterCheckpointMs)); + throw new Error("injected indexer failure after checkpoint"); + } scanViewUpdate(store, state); store.scan.workerPid = process.pid; const finished = new Date().toISOString(); @@ -226,26 +274,20 @@ export async function runIndexer(argv) { exitCode: 0, reason: scanned.complete ? "complete" : (stopping ? "stopped" : "partial"), }; + } catch (error) { + // Still inside the lock: the failure status is written under the same + // writer authority as every other commit, then the lock is released. + await recordWorkerFailure(storePath, generation, error); + return { exitCode: 1, reason: "error", error }; + } }, { - timeoutMs: 0, + timeoutMs: HANDOFF_WAIT_MS, meta: { role: "worker", generation }, }); return result ?? { exitCode: 0 }; } catch (error) { + // Lock was never held: report without writing any store state. if (isLockBusyError(error)) return { exitCode: 0, reason: "lock-busy" }; - try { - const loaded = await loadStore(storePath); - const store = loaded.store; - ensureStoreMeta(store); - if (storeGeneration(store) === generation) { - if (!store.scan) store.scan = emptyScanState(); - store.scan.status = "failed"; - store.scan.workerPid = null; - store.scan.error = error.message; - store.scan.updatedAt = new Date().toISOString(); - await saveStore(storePath, store, { status: "failed" }); - } - } catch { /* failure reporting is best-effort */ } return { exitCode: 1, reason: "error", error }; } finally { progress.clear(); diff --git a/tests/herdr-insights/collector-worker-lifecycle.test.mjs b/tests/herdr-insights/collector-worker-lifecycle.test.mjs new file mode 100644 index 00000000..0876f8bc --- /dev/null +++ b/tests/herdr-insights/collector-worker-lifecycle.test.mjs @@ -0,0 +1,241 @@ +import assert from "node:assert/strict"; +import { spawn } from "node:child_process"; +import { mkdir, mkdtemp, readFile, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import path from "node:path"; +import test from "node:test"; + +import { collectSnapshot } from "../../herdr-insights/mods/insights/collector.mjs"; +import { normalizeConfig } from "../../herdr-insights/mods/insights/core/config.mjs"; +import { readWorkerClaim, workerClaimPath } from "../../herdr-insights/mods/insights/core/fence.mjs"; +import { + ensureIndexer, + indexerPath, + waitForIndexerAck, +} from "../../herdr-insights/mods/insights/core/indexer-control.mjs"; +import { + emptyStore, + loadStore, + newFileEntry, + saveStore, +} from "../../herdr-insights/mods/insights/core/store.mjs"; +import { + atomicWriteJson, + isLockBusyError, + isProcessAlive, + readStoreLockInfo, + sleep, + withStoreLock, +} from "../../herdr-insights/mods/insights/core/util.mjs"; + +const piSource = [{ + id: "pi", + label: "Pi", + adapter: "pi", + enabled: true, + roots: ["~/.pi/agent/sessions"], +}]; + +function piSession(sessionId, input, timestamp = "2026-04-01T10:00:00Z") { + return [ + JSON.stringify({ type: "session", id: sessionId, timestamp, cwd: "/repo/lifecycle" }), + JSON.stringify({ + type: "message", + id: `user-${sessionId}`, + timestamp, + message: { role: "user", content: "prompt" }, + }), + JSON.stringify({ + type: "message", + id: `assistant-${sessionId}`, + timestamp, + message: { + role: "assistant", + model: "lifecycle-model", + provider: "lifecycle-provider", + usage: { input, output: 1, cacheRead: 0, cacheWrite: 0, reasoning: 0 }, + stopReason: "stop", + content: [], + }, + }), + ].map((line) => `${line}\n`).join(""); +} + +async function context(prefix) { + const root = await mkdtemp(path.join(tmpdir(), prefix)); + const home = path.join(root, "home"); + const store = path.join(root, "data", "insights.json"); + await mkdir(path.join(home, ".pi", "agent", "sessions"), { recursive: true }); + await mkdir(path.dirname(store), { recursive: true }); + return { root, home, store }; +} + +async function seedStore(storePath, mutate) { + const store = emptyStore(normalizeConfig({ sources: piSource })); + store.generation = 3; + store.scan.generation = 3; + store.scan.status = "complete"; + store.lastCompletedScanAt = "2026-04-01T11:00:00.000Z"; + mutate?.(store); + await saveStore(storePath, store, { completed: true, now: "2026-04-01T11:00:00.000Z" }); + return store; +} + +async function waitFor(predicate, { timeoutMs = 5_000, intervalMs = 20 } = {}) { + const deadline = Date.now() + timeoutMs; + for (;;) { + const value = await predicate(); + if (value) return value; + if (Date.now() >= deadline) return null; + await sleep(intervalMs); + } +} + +async function commitQueuedGeneration(storePath, marker) { + const deadline = Date.now() + 8_000; + for (;;) { + try { + return await withStoreLock(storePath, async () => { + const loaded = await loadStore(storePath); + const store = loaded.store; + store.generation = Number(store.generation ?? 0) + 1; + newFileEntry(store, { + sourceId: "pi", + path: `/tmp/${marker}.jsonl`, + adapter: "pi", + size: 10, + mtimeMs: 1, + tokens: { input: 1, output: 1, cacheRead: 0, cacheWrite: 0, reasoning: 0 }, + messages: 1, + }); + await saveStore(storePath, store, { status: store.scan?.status ?? "partial" }); + return store; + }, { timeoutMs: 0, meta: { role: "test-client" } }); + } catch (error) { + if (!isLockBusyError(error) || Date.now() >= deadline) throw error; + await sleep(25); + } + } +} + +test("worker failure is recorded under the lock and never clobbers a queued generation", async () => { + const ctx = await context("insights-worker-failure-"); + const slow = [ + piSession("worker-failure", 9), + ...Array.from({ length: 2_600 }, () => JSON.stringify({ type: "event", payload: "x".repeat(1024) })).map((line) => `${line}\n`), + ].join(""); + await writeFile(path.join(ctx.home, ".pi", "agent", "sessions", "one.jsonl"), slow); + await seedStore(ctx.store, (store) => { + newFileEntry(store, { + sourceId: "pi", + path: "/tmp/kept.jsonl", + adapter: "pi", + size: 10, + mtimeMs: 1, + tokens: { input: 7, output: 3, cacheRead: 0, cacheWrite: 0, reasoning: 0 }, + messages: 2, + }); + }); + + const token = `test-${process.pid}-${Date.now()}`; + const { hostname } = await import("node:os"); + const claimSeed = { + version: 1, + token, + pid: null, + host: hostname(), + generation: 3, + startedAt: Date.now(), + at: Date.now(), + }; + await atomicWriteJson(workerClaimPath(ctx.store), claimSeed); + + const child = spawn(process.execPath, [ + indexerPath(), + "--store", ctx.store, + "--home", ctx.home, + "--generation", "3", + "--claim-token", token, + "--test-stream-delay-ms", "120", + "--test-fail-after-checkpoint", "1200", + ], { stdio: "ignore", detached: false }); + + try { + // Publish the spawned pid exactly like the launcher does. + await atomicWriteJson(workerClaimPath(ctx.store), { ...claimSeed, pid: child.pid, at: Date.now() }); + const owner = await waitFor(async () => { + const info = await readStoreLockInfo(ctx.store); + return info?.pid === child.pid ? info : null; + }); + assert.ok(owner, "worker became the store lock owner"); + + const queued = commitQueuedGeneration(ctx.store, "queued.jsonl"); + const exitCode = await new Promise((resolve) => child.once("exit", resolve)); + assert.equal(exitCode, 1, "injected worker failure exits non-zero"); + await queued; + + const final = JSON.parse(await readFile(ctx.store, "utf8")); + assert.equal(final.generation, 4, "the queued client's newer generation survives"); + const byPath = (suffix) => Object.values(final.files).find((entry) => entry.path.includes(suffix)); + assert.ok(byPath("queued.jsonl"), "queued client marker survives"); + assert.ok(byPath("kept.jsonl"), "previous committed rows survive"); + assert.equal(byPath("kept.jsonl").tokens.input, 7); + assert.equal(final.scan.status, "failed", "failure was recorded under writer authority"); + assert.equal(final.scan.workerPid, null); + assert.match(final.scan.error, /injected indexer failure/); + } finally { + if (isProcessAlive(child.pid)) { + try { process.kill(child.pid, "SIGTERM"); } catch { /* already gone */ } + } + } +}); + +test("promotion spawns one worker that waits out the launcher lock handoff", async () => { + const ctx = await context("insights-worker-promotion-"); + const big = path.join(ctx.home, ".pi", "agent", "sessions", "big.jsonl"); + const filler = JSON.stringify({ type: "event", payload: "x".repeat(1024) }); + const lines = [piSession("promoted", 11)]; + for (let index = 0; index < 8_200; index += 1) lines.push(`${filler}\n`); + await writeFile(big, lines.join("")); + await seedStore(ctx.store, (store) => { + store.scan.status = "partial"; + store.scan.pendingFile = { path: big, requestedAt: "2026-04-01T11:00:00.000Z" }; + }); + + let worker = null; + // The launcher holds the lock across spawn and a deliberately slow save. + const launch = await withStoreLock(ctx.store, async () => { + const promoted = await ensureIndexer(ctx.store, ctx.home, { + generation: 3, + testStreamDelayMs: 120, + }); + await sleep(300); + return promoted; + }, { timeoutMs: 5_000, meta: { role: "test-launcher" } }); + worker = launch; + assert.ok(worker.spawned && worker.pid > 0, "one detached worker was spawned"); + + try { + const ack = await waitForIndexerAck(ctx.store, worker.claim, { timeoutMs: 8_000 }); + assert.equal(ack.ok, true, "worker acquired the lock after the bounded handoff"); + const owner = await readStoreLockInfo(ctx.store); + assert.equal(owner.pid, worker.pid); + assert.equal(isProcessAlive(worker.pid), true, "worker survives promotion"); + + const again = await ensureIndexer(ctx.store, ctx.home, { generation: 3 }); + assert.equal(again.spawned, false, "no respawn while the worker is live"); + assert.equal(again.pid, worker.pid); + assert.equal((await readWorkerClaim(ctx.store))?.token, worker.claim.token); + + const started = Date.now(); + const status = await collectSnapshot("status", { store: ctx.store }); + assert.ok(Date.now() - started < 1_000, "status stays non-blocking during the worker lock"); + assert.equal(status.scan.workerPid, worker.pid); + } finally { + if (worker?.pid && isProcessAlive(worker.pid)) { + try { process.kill(worker.pid, "SIGTERM"); } catch { /* already gone */ } + } + await waitFor(async () => !(worker?.pid && isProcessAlive(worker.pid)), { timeoutMs: 5_000 }); + await waitFor(async () => (await readWorkerClaim(ctx.store)) === null, { timeoutMs: 5_000 }); + } +}); From d912fa405f1ac3afffcc539e8e758a9bb40c70e5 Mon Sep 17 00:00:00 2001 From: GaganSD Date: Mon, 2 Nov 2026 19:20:29 +0530 Subject: [PATCH 03/15] fix(insights): persist first-seen corrupt fingerprints and finish plain files only at EOF --- herdr-insights/mods/insights/core/scan.mjs | 27 ++++++++- .../herdr-insights/collector-bounded.test.mjs | 56 +++++++++++++++++++ .../collector-transactions.test.mjs | 44 ++++++++++++++- 3 files changed, 123 insertions(+), 4 deletions(-) diff --git a/herdr-insights/mods/insights/core/scan.mjs b/herdr-insights/mods/insights/core/scan.mjs index 85e6dee0..fe0d8bd7 100644 --- a/herdr-insights/mods/insights/core/scan.mjs +++ b/herdr-insights/mods/insights/core/scan.mjs @@ -506,7 +506,10 @@ async function scanPlainFile( }); bytes = result.bytes; stopped = result.stopped; - consumer.finish(start + bytes); + // Finish a partial tail only at true EOF. A budget stop can land after a + // prefix that happens to parse as JSON but is still part of a longer line; + // committing it would invent rows and advance the offset into the line. + if (!stopped) consumer.finish(start + bytes); if (!stopped) delete entry.state.readFailed; } catch (error) { // Keep the accepted complete prefix and its parser state together. The @@ -887,12 +890,30 @@ async function scanOneFile(store, source, filePath, sink, stats, options = {}) { } if (compressed.status === "corrupt") { rollbackStoreTransaction(tx); - // Preserve the previous good archive and remember this exact corrupt - // fingerprint so an unchanged broken file is not re-read every refresh. + // A first-seen archive was detached by rollback. Persist a rowless + // fingerprint stub so an unchanged corrupt file is not re-read and + // re-warned on every refresh. A previously indexed archive keeps its + // last good rows and metadata. + const key = fileKey(source.id, filePath); + if (!store.files[key]) { + entry = newFileEntry(store, { + sourceId: source.id, + adapter: source.adapter, + path: filePath, + dev, + ino, + size: 0, + mtimeMs: 0, + compressed: true, + }); + } entry.size = fileStat.size; entry.mtimeMs = fileStat.mtimeMs; entry.offset = fileStat.size; entry.state.zstdError = `${fileStat.mtimeMs}:${fileStat.size}`; + // Mark the fingerprint as current so a first-seen stub is not treated + // as a parser migration and re-read on the next refresh. + if (parserVersion) entry.state[parserKey] = parserVersion; sourceSink.warn( `${source.adapter} ${entry.path}: zstd file could not be decompressed: ${compressed.error.message}`, ); diff --git a/tests/herdr-insights/collector-bounded.test.mjs b/tests/herdr-insights/collector-bounded.test.mjs index 2f61d2a3..cd5259a4 100644 --- a/tests/herdr-insights/collector-bounded.test.mjs +++ b/tests/herdr-insights/collector-bounded.test.mjs @@ -146,6 +146,62 @@ test("plain JSONL continuation reads only the unfinished tail and is not fingerp assert.equal(third.stats.filesRead, 0); }); +test("a budget stop before a newline never commits a parseable mid-line prefix", async () => { + const contextInfo = await context("insights-plain-boundary-"); + const file = path.join(contextInfo.home, ".pi", "agent", "sessions", "boundary.jsonl"); + const record = { + type: "message", + id: "assistant-boundary", + timestamp: "2026-04-04T00:00:00Z", + message: { + role: "assistant", + model: "boundary-model", + provider: "boundary-provider", + usage: { input: 1_234, output: 5, cacheRead: 0, cacheWrite: 0, reasoning: 0 }, + stopReason: "stop", + content: [], + }, + }; + const chunkBytes = 256 * 1024; + const head = Buffer.from(JSON.stringify(record)); + assert.ok(head.length < chunkBytes, "the valid prefix must fit in one chunk"); + // The first chunk is complete JSON followed by whitespace, so a premature + // finish() would accept it. The non-whitespace suffix makes the whole line + // invalid, and a budget stop must leave it uncommitted. + const prefix = Buffer.concat([head, Buffer.alloc(chunkBytes - head.length, 0x20)]); + const suffix = Buffer.from("trailing-junk\n"); + await writeFile(file, Buffer.concat([prefix, suffix])); + const totalSize = (await stat(file)).size; + + const oracleStore = emptyStore(normalizeConfig(sourceConfig())); + await scanSources(oracleStore, { home: contextInfo.home }); + assert.deepEqual(snapshotRows(oracleStore), [], "one-shot rejects the whole line"); + + const store = emptyStore(normalizeConfig(sourceConfig())); + const first = await scanSources(store, { + home: contextInfo.home, + deadline: Date.now(), + now: Date.now, + }); + assert.equal(first.complete, false); + assert.equal(first.stopped, true); + assert.deepEqual(snapshotRows(store), [], "no phantom usage from the parseable prefix"); + const entry = Object.values(store.files)[0]; + assert.ok(entry); + assert.equal(entry.offset, 0, "the safe offset stays before the unfinished line"); + assert.equal(entry.state.scanPending, true); + assert.notEqual(entry.size, totalSize, "the unfinished tail is not fingerprinted as complete"); + + const second = await scanSources(store, { home: contextInfo.home }); + assert.equal(second.complete, true); + assert.equal(second.stats.bytesRead, totalSize, "continuation rereads the unfinished line"); + assert.equal(second.stats.filesRead, 1); + assert.deepEqual(snapshotRows(store), snapshotRows(oracleStore), "continuation matches one-shot"); + assert.equal(entry.offset, totalSize); + assert.equal(entry.size, totalSize); + assert.equal(entry.state.scanPending, undefined); +}); + test("partial scans expose savedAt separately from the last completed scan", async () => { const contextInfo = await context("insights-timestamps-"); for (let index = 0; index < 3; index += 1) { diff --git a/tests/herdr-insights/collector-transactions.test.mjs b/tests/herdr-insights/collector-transactions.test.mjs index 74a3786e..badd3331 100644 --- a/tests/herdr-insights/collector-transactions.test.mjs +++ b/tests/herdr-insights/collector-transactions.test.mjs @@ -1,5 +1,5 @@ import assert from "node:assert/strict"; -import { readFile, mkdir, mkdtemp, writeFile } from "node:fs/promises"; +import { readFile, mkdir, mkdtemp, stat, writeFile } from "node:fs/promises"; import { tmpdir } from "node:os"; import path from "node:path"; import { fileURLToPath } from "node:url"; @@ -176,6 +176,48 @@ test("a corrupt archive after shared revisions preserves sibling rows, dedup and assert.equal(JSON.stringify(snapshotRows(store)), beforeRows); }); +test("a first-seen corrupt archive persists a rowless fingerprint and recovers", async () => { + const root = await mkdtemp(path.join(tmpdir(), "insights-first-corrupt-")); + const home = path.join(root, "home"); + const sessions = path.join(home, ".codex", "sessions"); + await mkdir(sessions, { recursive: true }); + const file = path.join(sessions, "bad.jsonl.zst"); + await writeFile(file, Buffer.from("not a zstd frame")); + const store = emptyStore(); + store.config = { + ...store.config, + sources: [{ id: "codex", label: "Codex", adapter: "codex", enabled: true, roots: ["~/.codex/sessions"] }], + }; + + const first = await scanSources(store, { home }); + assert.equal(first.stats.errors, 1); + const keys = Object.keys(store.files); + assert.equal(keys.length, 1, "the corrupt file itself is fingerprinted"); + const entry = store.files[keys[0]]; + assert.equal(entry.path.endsWith(path.basename(file)), true); + assert.deepEqual(entry.rows, []); + assert.deepEqual(entry.tasks, []); + const badStat = await stat(file); + assert.equal(entry.size, badStat.size); + assert.equal(entry.offset, badStat.size); + assert.equal(entry.state.zstdError, `${badStat.mtimeMs}:${badStat.size}`); + + const repeated = await scanSources(store, { home }); + assert.equal(repeated.stats.errors, 0, "the stored fingerprint suppresses re-warning"); + assert.equal(repeated.stats.filesRead, 0); + assert.equal(repeated.stats.bytesRead, 0); + assert.equal(store.files[keys[0]], entry, "the same rowless stub stays in the store"); + assert.deepEqual(entry.rows, []); + + const plain = await readFile(path.join(here, "fixtures", "codex-modern.jsonl"), "utf8"); + await writeFile(file, zstdCompressSync(plain)); + const recovered = await scanSources(store, { home }); + assert.equal(recovered.stats.errors, 0); + assert.equal(recovered.stats.filesRead, 1); + assert.equal(entry.state.zstdError, undefined); + assert.ok(entry.rows.length > 0, "a valid replacement is indexed"); +}); + test("production scanning does not clone the accumulated store", async () => { const scanSource = await readFile( path.join(here, "../../herdr-insights/mods/insights/core/scan.mjs"), From 23cd07ae3aa2c754737c29884710d983f1be448f Mon Sep 17 00:00:00 2001 From: GaganSD Date: Mon, 2 Nov 2026 19:20:34 +0530 Subject: [PATCH 04/15] fix(insights): reschedule scan continuation when the view is busy --- herdr-insights/mods/insights/main.js | 9 +++++- tests/herdr-insights/analytics-ui.test.mjs | 32 ++++++++++++++++++++++ 2 files changed, 40 insertions(+), 1 deletion(-) diff --git a/herdr-insights/mods/insights/main.js b/herdr-insights/mods/insights/main.js index f13aeca9..ba44dd55 100644 --- a/herdr-insights/mods/insights/main.js +++ b/herdr-insights/mods/insights/main.js @@ -800,7 +800,14 @@ export default class Insights extends View { if (this.scanContinueTimer) return; this.scanContinueTimer = cx.timer.after(delay, (timerCx) => { this.scanContinueTimer = null; - if (!this.closed) this.refresh(timerCx); + if (this.closed) return; + // A tick during an in-flight load/save must not be lost: refresh is a + // no-op while busy, so keep the continuation armed until it can run. + if (this.busy()) { + this.scheduleScanContinue(timerCx); + return; + } + this.refresh(timerCx); }); } diff --git a/tests/herdr-insights/analytics-ui.test.mjs b/tests/herdr-insights/analytics-ui.test.mjs index e3135ae2..1684e72d 100644 --- a/tests/herdr-insights/analytics-ui.test.mjs +++ b/tests/herdr-insights/analytics-ui.test.mjs @@ -1140,6 +1140,38 @@ test("a failed prepare leaves the old dashboard and a retryable error", async () instance.close(); }); +test("partial scan continuation reschedules while busy and stops on close", async () => { + const { cx, calls } = liveCx(); + const instance = blankInstance(); + instance.sourceInputs = new Map(); + instance.refreshInput = null; + instance.configEditor = null; + const refreshes = []; + instance.refresh = (timerCx) => refreshes.push(timerCx); + instance.snapshot = { ...savedSnapshot(), scan: { status: "partial", processed: 1, total: 3 } }; + + instance.saving = true; + instance.scheduleScanContinue(cx); + assert.equal(calls.after.length, 1); + await calls.runAfter(); + assert.equal(refreshes.length, 0, "a busy tick must not drop the continuation"); + assert.equal(calls.after.filter((timer) => !timer.cancelled).length, 1, "the timer reschedules"); + + instance.saving = false; + instance.loading = false; + await calls.runAfter(); + assert.equal(refreshes.length, 1, "the continuation runs once the save finishes"); + + instance.snapshot.scan = { status: "partial", processed: 2, total: 3 }; + instance.scheduleScanContinue(cx); + const pending = calls.after.filter((timer) => !timer.cancelled); + assert.equal(pending.length, 1); + instance.close(); + assert.equal(pending[0].cancelled, true, "close cancels the pending continuation"); + await calls.runAfter(); + assert.equal(refreshes.length, 1, "a closed view never refreshes again"); +}); + test("a hung prepare times out into a retryable dashboard error", async () => { const { todayKey } = analytics; const snapshot = savedSnapshot(); From bf7943920602d46e35f3f454201b39a230bee537 Mon Sep 17 00:00:00 2001 From: GaganSD Date: Mon, 2 Nov 2026 19:20:34 +0530 Subject: [PATCH 05/15] fix(insights): atomically publish and quarantine the store lock directory --- herdr-insights/mods/insights/core/util.mjs | 91 ++++++++++++++------ tests/herdr-insights/ship-readiness.test.mjs | 34 ++++++++ 2 files changed, 98 insertions(+), 27 deletions(-) diff --git a/herdr-insights/mods/insights/core/util.mjs b/herdr-insights/mods/insights/core/util.mjs index f3f1b61c..a6e3d651 100644 --- a/herdr-insights/mods/insights/core/util.mjs +++ b/herdr-insights/mods/insights/core/util.mjs @@ -8,6 +8,7 @@ import { readFile, readdir, rename, + rm, rmdir, stat, unlink, @@ -234,6 +235,19 @@ async function tryUnlinkStaleLock(lockPath, observed) { return true; } +async function quarantineLockDirectory(lockPath, token) { + const quarantine = `${lockPath}.release-${token}`; + try { + // A rename is atomic: no observer can see the lock directory emptied and + // later removed in separate steps. A crash leaves only the quarantine dir. + await rename(lockPath, quarantine); + } catch (error) { + if (error?.code === "ENOENT") return; + throw error; + } + await rm(quarantine, { recursive: true, force: true }).catch(() => {}); +} + export async function withStoreLock( storePath, callback, @@ -245,51 +259,68 @@ export async function withStoreLock( const host = os.hostname(); const startedAt = Date.now(); const deadline = startedAt + timeoutMs; - const ownerPath = path.join(lockPath, `owner-${token}.json`); - - for (;;) { - let handle; - let created = false; + const ownerName = `owner-${token}.json`; + const ownerPath = path.join(lockPath, ownerName); + // Publish a fully populated unique directory with one atomic rename. Crash + // windows before this point leave only a private staging directory that no + // writer mistakes for the lock. + const stagingPath = path.join( + path.dirname(lockPath), + `.${path.basename(lockPath)}.acquire-${token}`, + ); + try { + await mkdir(stagingPath, { mode: 0o700 }); + const handle = await open(path.join(stagingPath, ownerName), "wx", 0o600); try { - await mkdir(lockPath, { mode: 0o700 }); - created = true; - handle = await open(ownerPath, "wx", 0o600); await handle.writeFile( `${JSON.stringify({ ...meta, pid: process.pid, host, startedAt, token })}\n`, "utf8", ); await handle.sync(); - await handle.close(); - handle = null; - break; - } catch (error) { - if (handle) await handle.close().catch(() => {}); - if (created) { - await unlink(ownerPath).catch(() => {}); - await rmdir(lockPath).catch(() => {}); - } - if (error?.code !== "EEXIST") throw error; + } finally { + await handle.close().catch(() => {}); + } + } catch (error) { + await rm(stagingPath, { recursive: true, force: true }).catch(() => {}); + throw error; + } + try { + for (;;) { const observed = await readLockInfo(lockPath); + if (!observed) { + try { + await rename(stagingPath, lockPath); + break; + } catch (error) { + if (!["EEXIST", "ENOTEMPTY", "ENOTDIR", "EISDIR"].includes(error?.code)) { + throw error; + } + // A writer won the race between the existence check and the rename. + } + } + const current = observed ?? await readLockInfo(lockPath); const age = await lockAgeMs(lockPath); - const sameHost = observed?.host === host; - const pid = Number(observed?.pid); + const sameHost = current?.host === host; + const pid = Number(current?.pid); const dead = sameHost && Number.isInteger(pid) && !isProcessAlive(pid); // Only provably dead local owners can be evicted. Foreign/empty locks // fail closed rather than allowing two writers based on elapsed time. - const malformedOld = !observed?.directory && observed?.invalid && age > Math.max(staleMs, 60_000); + const malformedOld = !current?.directory && current?.invalid + && age > Math.max(staleMs, 60_000); if (dead || malformedOld) { - const removed = await tryUnlinkStaleLock(lockPath, observed); + const removed = await tryUnlinkStaleLock(lockPath, current); if (removed) continue; } if (Date.now() >= deadline) { - throw new Error(`insights store is busy (lock ${lockPath})`, { - cause: error, - }); + throw new Error(`insights store is busy (lock ${lockPath})`); } await sleep(60 + Math.floor(Math.random() * 40)); } + } catch (error) { + await rm(stagingPath, { recursive: true, force: true }).catch(() => {}); + throw error; } try { @@ -297,8 +328,14 @@ export async function withStoreLock( } finally { const current = await readLockInfo(lockPath); if (current?.token === token) { - await unlink(ownerPath).catch(() => {}); - await rmdir(lockPath).catch(() => {}); + try { + await quarantineLockDirectory(lockPath, token); + } catch { + // Conservative fallback for platforms/filesystems that cannot rename + // the directory. It is only taken after the owner token matched. + await unlink(ownerPath).catch(() => {}); + await rmdir(lockPath).catch(() => {}); + } } } } diff --git a/tests/herdr-insights/ship-readiness.test.mjs b/tests/herdr-insights/ship-readiness.test.mjs index 78041d71..183b27a4 100644 --- a/tests/herdr-insights/ship-readiness.test.mjs +++ b/tests/herdr-insights/ship-readiness.test.mjs @@ -280,6 +280,40 @@ test("a live same-host lock cannot be stolen just because it is old", async (t) await assert.rejects(stat(`${store}.lock`), { code: "ENOENT" }); }); +test("lock acquisition publishes one populated directory and release leaves no artifacts", async (t) => { + const root = await fixture(t); + const store = path.join(root, "store.json"); + const lock = `${store}.lock`; + await withStoreLock(store, async () => { + assert.deepEqual(await readdir(root), [path.basename(lock)], "no staging directory survives"); + const owners = await readdir(lock); + assert.equal(owners.length, 1, "the published lock directory is never empty"); + const owner = JSON.parse(await readFile(path.join(lock, owners[0]), "utf8")); + assert.equal(owner.pid, process.pid); + assert.equal(typeof owner.token, "string"); + }); + assert.deepEqual(await readdir(root), [], "release quarantines the whole directory"); + await assert.rejects(stat(lock), { code: "ENOENT" }); +}); + +test("concurrent writers serialize through the atomically published lock", async (t) => { + const root = await fixture(t); + const store = path.join(root, "store.json"); + let active = 0; + let peak = 0; + let completed = 0; + await Promise.all(Array.from({ length: 4 }, () => withStoreLock(store, async () => { + active += 1; + peak = Math.max(peak, active); + await new Promise((resolve) => setTimeout(resolve, 5)); + active -= 1; + completed += 1; + }))); + assert.equal(peak, 1, "exactly one writer holds the lock"); + assert.equal(completed, 4); + assert.deepEqual(await readdir(root), [], "no lock, staging or quarantine remains"); +}); + test("concurrent dead-owner reclaimers cannot remove a newly acquired lock", async (t) => { const root = await fixture(t); const store = path.join(root, "store.json"); From fe53ef80dc33e44ebe3033164813229af011ff60 Mon Sep 17 00:00:00 2001 From: GaganSD Date: Mon, 2 Nov 2026 19:35:50 +0530 Subject: [PATCH 06/15] test(insights): verify worker lifecycle gates and long-run completion Add deterministic process integration coverage for four-launcher coalescing, SIGKILL checkpoint/restart exactness, generation fencing, and presenter responsiveness under the writer lock. Add a separate opt-in long-worker run that proves >30s synthetic throttled completion with one decoder pass. Clean up lifecycle temp roots and child processes. --- package.json | 3 +- .../collector-lifecycle-gates.test.mjs | 422 ++++++++++++++++++ .../collector-worker-lifecycle.test.mjs | 4 +- tests/herdr-insights/lifecycle-launcher.mjs | 35 ++ .../worker-long.integration.mjs | 182 ++++++++ 5 files changed, 644 insertions(+), 2 deletions(-) create mode 100644 tests/herdr-insights/collector-lifecycle-gates.test.mjs create mode 100644 tests/herdr-insights/lifecycle-launcher.mjs create mode 100644 tests/herdr-insights/worker-long.integration.mjs diff --git a/package.json b/package.json index 0a3a6053..624c0d4a 100644 --- a/package.json +++ b/package.json @@ -20,7 +20,8 @@ "release:prepare": "node scripts/release.mjs prepare", "release:apply": "node scripts/release.mjs apply", "release:check": "node scripts/release.mjs check", - "release:publish": "node scripts/release.mjs publish" + "release:publish": "node scripts/release.mjs publish", + "test:insights:long-worker": "node tests/herdr-insights/worker-long.integration.mjs" }, "pi": { "extensions": [ diff --git a/tests/herdr-insights/collector-lifecycle-gates.test.mjs b/tests/herdr-insights/collector-lifecycle-gates.test.mjs new file mode 100644 index 00000000..059b37f4 --- /dev/null +++ b/tests/herdr-insights/collector-lifecycle-gates.test.mjs @@ -0,0 +1,422 @@ +import assert from "node:assert/strict"; +import { spawn } from "node:child_process"; +import { mkdir, mkdtemp, readFile, rm, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import path from "node:path"; +import { fileURLToPath } from "node:url"; +import * as zlib from "node:zlib"; +import test from "node:test"; + +import { collectSnapshot } from "../../herdr-insights/mods/insights/collector.mjs"; +import { normalizeConfig } from "../../herdr-insights/mods/insights/core/config.mjs"; +import { readWorkerClaim } from "../../herdr-insights/mods/insights/core/fence.mjs"; +import { ensureIndexer } from "../../herdr-insights/mods/insights/core/indexer-control.mjs"; +import { emptyStore, loadStore, snapshotRows } from "../../herdr-insights/mods/insights/core/store.mjs"; +import { + isProcessAlive, + readStoreLockInfo, + sleep, +} from "../../herdr-insights/mods/insights/core/util.mjs"; + +const here = path.dirname(fileURLToPath(import.meta.url)); +const launcher = path.join(here, "lifecycle-launcher.mjs"); +const presenter = path.join(here, "../../herdr-insights/mods/insights/presenter.mjs"); + +const codexSource = { + id: "codex", + label: "Codex", + adapter: "codex", + enabled: true, + roots: ["~/.codex/sessions"], +}; + +function sourceConfig(roots = codexSource.roots) { + return { sources: [{ ...codexSource, roots }] }; +} + +function sharedSession(extra = []) { + return [ + { type: "session_meta", timestamp: "2026-04-02T10:00:00Z", payload: { id: "shared", cwd: "/repo/gate", model_provider: "openai" } }, + { type: "turn_context", timestamp: "2026-04-02T10:00:01Z", payload: { turn_id: "turn1", model: "gate-model", cwd: "/repo/gate" } }, + { + type: "token_usage_record", + timestamp: "2026-04-02T10:00:02Z", + payload: { + response_id: "resp1", + turn_id: "turn1", + usage: { input_tokens: 10, cached_input_tokens: 0, cache_write_input_tokens: 0, output_tokens: 2, reasoning_output_tokens: 0, total_tokens: 12 }, + }, + }, + { type: "event_msg", timestamp: "2026-04-02T10:00:03Z", payload: { type: "task_complete", turn_id: "turn1", duration_ms: 2000 } }, + ...extra, + ]; +} + +function noiseLines(bytes) { + const line = `${JSON.stringify({ type: "synthetic_noise", payload: "x".repeat(1024) })}\n`; + return line.repeat(Math.ceil(bytes / line.length)); +} + +function codexContent(bytes, extra = []) { + return `${sharedSession(extra).map((record) => JSON.stringify(record)).join("\n")}\n${noiseLines(bytes)}`; +} + +async function zstdFile(directory, name, content) { + const file = path.join(directory, name); + await writeFile(file, zlib.zstdCompressSync(Buffer.from(content))); + return file; +} + +async function context(prefix) { + const root = await mkdtemp(path.join(tmpdir(), prefix)); + const home = path.join(root, "home"); + const store = path.join(root, "data", "insights.json"); + await mkdir(path.join(home, ".codex", "sessions"), { recursive: true }); + await mkdir(path.dirname(store), { recursive: true }); + return { root, home, store }; +} + +async function waitFor(predicate, { timeoutMs = 8_000, intervalMs = 25 } = {}) { + const deadline = Date.now() + timeoutMs; + for (;;) { + const value = await predicate(); + if (value) return value; + if (Date.now() >= deadline) return null; + await sleep(intervalMs); + } +} + +async function stopWorker(pid) { + if (!Number.isInteger(pid) || pid <= 0 || !isProcessAlive(pid)) return; + try { process.kill(pid, "SIGTERM"); } catch { /* already gone */ } + const gone = await waitFor(() => !isProcessAlive(pid), { timeoutMs: 8_000 }); + if (!gone && isProcessAlive(pid)) { + try { process.kill(pid, "SIGKILL"); } catch { /* already gone */ } + await waitFor(() => !isProcessAlive(pid), { timeoutMs: 2_000 }); + } +} + +async function cleanup(ctx, pids = []) { + for (const pid of pids) await stopWorker(pid); + await rm(ctx.root, { recursive: true, force: true }); +} + +async function readStore(storePath) { + try { + return JSON.parse(await readFile(storePath, "utf8")); + } catch (error) { + if (error?.code === "ENOENT") return null; + throw error; + } +} + +test("four separate simultaneous launchers coalesce into one worker and one writer", async (t) => { + const ctx = await context("insights-lifecycle-coalesce-"); + const workers = new Set(); + t.after(() => cleanup(ctx, [...workers])); + try { + const dir = path.join(ctx.home, ".codex", "sessions", "race"); + await mkdir(dir, { recursive: true }); + const seeded = await collectSnapshot("configure", { + store: ctx.store, + home: ctx.home, + config: sourceConfig(), + }); + assert.equal(seeded.scan.status, "complete"); + await zstdFile(dir, "big.jsonl.zst", codexContent(1_400_000)); + + const gate = path.join(ctx.root, "start.gate"); + const children = Array.from({ length: 4 }, () => spawn(process.execPath, [ + launcher, ctx.store, ctx.home, gate, "45", + ], { stdio: ["ignore", "pipe", "pipe"] })); + const results = children.map((child) => new Promise((resolve, reject) => { + let stdout = ""; + let stderr = ""; + child.stdout.setEncoding("utf8"); + child.stderr.setEncoding("utf8"); + child.stdout.on("data", (chunk) => (stdout += chunk)); + child.stderr.on("data", (chunk) => (stderr += chunk)); + child.on("error", reject); + child.on("close", (code) => resolve({ code, stdout, stderr })); + })); + await writeFile(gate, "go\n"); + const finished = await Promise.all(results); + for (const [index, result] of finished.entries()) { + assert.equal(result.code, 0, `launcher ${index} failed: ${result.stderr}`); + const payload = JSON.parse(result.stdout.trim()); + assert.equal(payload.warnings.length, 0, `launcher ${index} warned`); + if (payload.workerPid) workers.add(payload.workerPid); + } + assert.equal(workers.size, 1, "all four launchers observed the same worker pid"); + + const claim = await readWorkerClaim(ctx.store); + assert.ok(claim?.token, "exactly one worker claim exists"); + assert.equal(Number.isInteger(claim.pid), true); + workers.add(claim.pid); + assert.equal(isProcessAlive(claim.pid), true, "the claimed worker stays alive"); + const owner = await waitFor(async () => { + const info = await readStoreLockInfo(ctx.store); + return info?.pid === claim.pid ? info : null; + }); + assert.ok(owner, "the one claimed worker is the store writer"); + assert.equal(owner.role, "worker"); + + const again = await ensureIndexer(ctx.store, ctx.home, { generation: (await readStore(ctx.store)).generation }); + assert.equal(again.spawned, false, "a live claim is never respawned"); + assert.equal(again.pid, claim.pid); + assert.equal((await readWorkerClaim(ctx.store))?.token, claim.token); + } finally { + for (const pid of [...workers]) await stopWorker(pid); + } +}); + +test("SIGKILL mid-archive preserves durable file checkpoints and restart completes exactly once", async (t) => { + const ctx = await context("insights-lifecycle-kill-"); + const oracle = await context("insights-lifecycle-kill-oracle-"); + let workerPid = null; + t.after(async () => { + if (workerPid) await stopWorker(workerPid); + await cleanup(ctx); + await cleanup(oracle); + }); + try { + const dir = path.join(ctx.home, ".codex", "sessions", "kill"); + await mkdir(dir, { recursive: true }); + const unique = [ + { type: "session_meta", timestamp: "2026-04-02T10:00:04Z", payload: { id: "open-unique", cwd: "/repo/gate", model_provider: "openai" } }, + { type: "turn_context", timestamp: "2026-04-02T10:00:05Z", payload: { turn_id: "turn-open", model: "open-model", cwd: "/repo/gate" } }, + { + type: "token_usage_record", + timestamp: "2026-04-02T10:00:06Z", + payload: { + response_id: "resp-open", + turn_id: "turn-open", + usage: { input_tokens: 7, cached_input_tokens: 0, cache_write_input_tokens: 0, output_tokens: 1, reasoning_output_tokens: 0, total_tokens: 8 }, + }, + }, + ]; + await zstdFile(dir, "01-shared.jsonl.zst", codexContent(2_400_000)); + await zstdFile(dir, "02-open.jsonl.zst", codexContent(3_200_000, unique)); + for (const name of ["01-shared.jsonl.zst", "02-open.jsonl.zst"]) { + const target = path.join(oracle.home, ".codex", "sessions", "kill"); + await mkdir(target, { recursive: true }); + await writeFile(path.join(target, name), await readFile(path.join(dir, name))); + } + const oracleSnapshot = await collectSnapshot("configure", { + store: oracle.store, + home: oracle.home, + config: sourceConfig(), + }); + assert.equal(oracleSnapshot.scan.status, "complete"); + const expected = snapshotRows((await loadStore(oracle.store)).store); + assert.equal(expected.length, 2, "shared copy deduplicates and the unique session appears once"); + const expectedShared = expected.find((row) => row.model === "gate-model"); + assert.equal(expectedShared.messages, 1); + assert.equal(expectedShared.tokens.input, 10); + assert.equal(expectedShared.activeMs, 2000); + assert.ok(expected.find((row) => row.model === "open-model")); + + const seedStore = emptyStore(normalizeConfig(sourceConfig())); + seedStore.generation = 1; + seedStore.scan.generation = 1; + seedStore.scan.status = "complete"; + await mkdir(path.dirname(ctx.store), { recursive: true }); + await writeFile(ctx.store, `${JSON.stringify(seedStore)}\n`); + + const launched = await collectSnapshot("snapshot", { + store: ctx.store, + home: ctx.home, + budgetMs: 0, + testStreamDelayMs: 50, + workerAckMs: 5_000, + }); + workerPid = launched.scan.workerPid; + assert.ok(workerPid, "foreground aborted and promoted one worker"); + const checkpointed = await waitFor(async () => { + const store = await readStore(ctx.store); + const entries = Object.values(store?.files ?? {}); + return entries.some((entry) => entry.path.endsWith("01-shared.jsonl.zst")) + && !entries.some((entry) => entry.path.endsWith("02-open.jsonl.zst")) + ? store + : null; + }, { timeoutMs: 10_000 }); + assert.ok(checkpointed, "first completed archive was durably checkpointed"); + assert.equal(isProcessAlive(workerPid), true, "worker is still mid open archive"); + await sleep(250); + process.kill(workerPid, "SIGKILL"); + await waitFor(() => !isProcessAlive(workerPid), { timeoutMs: 3_000 }); + + const killed = await readStore(ctx.store); + const killedRows = snapshotRows(killed); + const killedPaths = Object.values(killed.files).map((entry) => entry.path); + assert.ok(killedPaths.some((value) => value.endsWith("01-shared.jsonl.zst"))); + assert.equal(killedPaths.some((value) => value.endsWith("02-open.jsonl.zst")), false, + "the open archive contributed no store entry"); + assert.equal(killedRows.some((row) => row.model === "open-model"), false, + "the open archive contributed zero rows"); + assert.equal(killedRows.find((row) => row.model === "gate-model").messages, 1); + + const restarted = await ensureIndexer(ctx.store, ctx.home, { + generation: (await readStore(ctx.store)).generation, + }); + assert.equal(restarted.spawned, true, "the dead worker claim was reclaimed and restarted"); + workerPid = restarted.pid; + const completed = await waitFor(async () => { + const store = await readStore(ctx.store); + return store?.scan?.status === "complete" ? store : null; + }, { timeoutMs: 15_000 }); + assert.ok(completed, "restart completed the remaining archive"); + await waitFor(async () => (await readWorkerClaim(ctx.store)) === null, { timeoutMs: 5_000 }); + await waitFor(() => !isProcessAlive(restarted.pid), { timeoutMs: 5_000 }); + assert.equal(isProcessAlive(restarted.pid), false, "worker exited after completion"); + + const finalRows = snapshotRows(await readStore(ctx.store)); + assert.deepEqual(finalRows, expected, "exact tokens, messages, tasks and copied-session dedup"); + assert.equal(finalRows.length, expected.length, "no duplicate rows after restart"); + } finally { + if (workerPid) await stopWorker(workerPid); + } +}); + +test("configure during a slow worker fences the old generation and commits only new config history", async (t) => { + const ctx = await context("insights-lifecycle-fence-"); + let workerPid = null; + t.after(async () => { + if (workerPid) await stopWorker(workerPid); + await cleanup(ctx); + }); + try { + const oldRoot = path.join(ctx.home, ".codex", "sessions", "old"); + const newRoot = path.join(ctx.home, ".codex", "sessions", "new"); + await mkdir(oldRoot, { recursive: true }); + await mkdir(newRoot, { recursive: true }); + await writeFile(path.join(newRoot, "fresh.jsonl"), [ + { type: "session_meta", timestamp: "2026-04-02T11:00:00Z", payload: { id: "new-session", cwd: "/repo/new", model_provider: "openai" } }, + { type: "turn_context", timestamp: "2026-04-02T11:00:01Z", payload: { turn_id: "turn-new", model: "new-model", cwd: "/repo/new" } }, + { + type: "token_usage_record", + timestamp: "2026-04-02T11:00:02Z", + payload: { + response_id: "resp-new", + turn_id: "turn-new", + usage: { input_tokens: 4, cached_input_tokens: 0, cache_write_input_tokens: 0, output_tokens: 1, reasoning_output_tokens: 0, total_tokens: 5 }, + }, + }, + ].map((record) => `${JSON.stringify(record)}\n`).join("")); + const seededConfig = sourceConfig(["~/.codex/sessions/old"]); + await collectSnapshot("configure", { store: ctx.store, home: ctx.home, config: seededConfig }); + await zstdFile(oldRoot, "old-big.jsonl.zst", codexContent(1_500_000)); + + const launched = await collectSnapshot("snapshot", { + store: ctx.store, + home: ctx.home, + budgetMs: 0, + testStreamDelayMs: 40, + workerAckMs: 5_000, + }); + workerPid = launched.scan.workerPid; + assert.ok(workerPid); + const active = await waitFor(async () => { + const info = await readStoreLockInfo(ctx.store); + return info?.pid === workerPid ? info : null; + }); + assert.ok(active, "old worker owns the store lock"); + + const configured = await collectSnapshot("configure", { + store: ctx.store, + home: ctx.home, + config: sourceConfig(["~/.codex/sessions/new"]), + lockWaitMs: 25_000, + }); + assert.ok(["complete", "partial"].includes(configured.scan.status)); + assert.deepEqual(configured.config.sources[0].roots, ["~/.codex/sessions/new"]); + assert.equal(configured.rows.some((row) => row.model === "gate-model"), false, + "old history was rebuilt away with the source change"); + assert.equal(configured.rows.some((row) => row.model === "new-model"), true); + const configuredGeneration = configured.scan.generation; + assert.ok(configuredGeneration >= 2, "configure advanced the generation"); + + await waitFor(() => !isProcessAlive(workerPid), { timeoutMs: 8_000 }); + assert.equal(isProcessAlive(workerPid), false, "the fenced worker exited"); + await waitFor(async () => (await readWorkerClaim(ctx.store)) === null, { timeoutMs: 5_000 }); + assert.equal(await readWorkerClaim(ctx.store), null, "the worker claim was cleared"); + + await sleep(700); + const settled = await loadStore(ctx.store); + assert.equal(settled.store.generation, (await readStore(ctx.store)).generation); + assert.deepEqual(snapshotRows(settled.store), snapshotRows((await loadStore(ctx.store)).store)); + assert.deepEqual(snapshotRows(settled.store).map((row) => row.model), ["new-model"], + "no late old-generation checkpoint or final save leaked"); + assert.equal(await readStoreLockInfo(ctx.store), null, "single writer released the lock"); + } finally { + if (workerPid) await stopWorker(workerPid); + } +}); + +test("presenter prepare/status stay fast, running and truthful while a worker holds the lock", async (t) => { + const ctx = await context("insights-lifecycle-presenter-"); + let workerPid = null; + t.after(async () => { + if (workerPid) await stopWorker(workerPid); + await cleanup(ctx); + }); + try { + const dir = path.join(ctx.home, ".codex", "sessions", "ui"); + await mkdir(dir, { recursive: true }); + await collectSnapshot("configure", { store: ctx.store, home: ctx.home, config: sourceConfig() }); + await zstdFile(dir, "01-slow.jsonl.zst", codexContent(700_000)); + await zstdFile(dir, "02-big.jsonl.zst", codexContent(1_600_000)); + + const launched = await collectSnapshot("snapshot", { + store: ctx.store, + home: ctx.home, + budgetMs: 0, + testStreamDelayMs: 35, + workerAckMs: 5_000, + }); + workerPid = launched.scan.workerPid; + assert.ok(workerPid); + const owner = await waitFor(async () => { + const info = await readStoreLockInfo(ctx.store); + return info?.pid === workerPid ? info : null; + }); + assert.ok(owner, "worker owns the writer lock"); + await waitFor(async () => { + const store = await readStore(ctx.store); + return store?.scan?.status === "running" ? store : null; + }, { timeoutMs: 5_000 }); + + const samples = []; + for (const command of ["prepare", "status", "prepare", "status", "prepare"]) { + const started = Date.now(); + const result = await new Promise((resolve, reject) => { + const child = spawn(process.execPath, [presenter, command, "--store", ctx.store], { + stdio: ["ignore", "pipe", "pipe"], + }); + let stdout = ""; + let stderr = ""; + child.stdout.setEncoding("utf8"); + child.stderr.setEncoding("utf8"); + child.stdout.on("data", (chunk) => (stdout += chunk)); + child.stderr.on("data", (chunk) => (stderr += chunk)); + child.on("error", reject); + child.on("close", (code) => resolve({ code, stdout, stderr })); + }); + const elapsed = Date.now() - started; + samples.push(elapsed); + assert.equal(result.code, 0, result.stderr); + const payload = JSON.parse(result.stdout.trim()); + assert.equal(payload.command, command); + assert.ok(payload.snapshot, "presenter returned the committed snapshot"); + assert.equal(payload.snapshot.scan?.status, "running", "scan state is truthful while indexing"); + assert.equal(payload.snapshot.scan?.workerPid, workerPid, "worker identity is exposed"); + assert.ok(payload.snapshot.scan?.generation >= 1); + assert.ok(payload.dashboard, "dashboard derives from committed cache without waiting"); + assert.ok(elapsed < 1_000, `${command} returned in ${elapsed}ms, under the 1s local limit`); + } + const median = [...samples].sort((a, b) => a - b)[Math.floor(samples.length / 2)]; + assert.ok(median < 1_000, `median ${median}ms stays under the hard local limit`); + } finally { + if (workerPid) await stopWorker(workerPid); + } +}); diff --git a/tests/herdr-insights/collector-worker-lifecycle.test.mjs b/tests/herdr-insights/collector-worker-lifecycle.test.mjs index 0876f8bc..5f7349d9 100644 --- a/tests/herdr-insights/collector-worker-lifecycle.test.mjs +++ b/tests/herdr-insights/collector-worker-lifecycle.test.mjs @@ -1,6 +1,6 @@ import assert from "node:assert/strict"; import { spawn } from "node:child_process"; -import { mkdir, mkdtemp, readFile, writeFile } from "node:fs/promises"; +import { mkdir, mkdtemp, readFile, rm, writeFile } from "node:fs/promises"; import { tmpdir } from "node:os"; import path from "node:path"; import test from "node:test"; @@ -187,6 +187,7 @@ test("worker failure is recorded under the lock and never clobbers a queued gene if (isProcessAlive(child.pid)) { try { process.kill(child.pid, "SIGTERM"); } catch { /* already gone */ } } + await rm(ctx.root, { recursive: true, force: true }); } }); @@ -237,5 +238,6 @@ test("promotion spawns one worker that waits out the launcher lock handoff", asy } await waitFor(async () => !(worker?.pid && isProcessAlive(worker.pid)), { timeoutMs: 5_000 }); await waitFor(async () => (await readWorkerClaim(ctx.store)) === null, { timeoutMs: 5_000 }); + await rm(ctx.root, { recursive: true, force: true }); } }); diff --git a/tests/herdr-insights/lifecycle-launcher.mjs b/tests/herdr-insights/lifecycle-launcher.mjs new file mode 100644 index 00000000..73291f7b --- /dev/null +++ b/tests/herdr-insights/lifecycle-launcher.mjs @@ -0,0 +1,35 @@ +#!/usr/bin/env node +// Test-only launcher used by collector-lifecycle-gates.test.mjs. Four separate +// processes wait on the same gate file and then run one foreground slice with a +// deterministic zero budget, so every process promotes instead of finishing the +// archive itself. The delay is a synthetic test hook, never production behavior. +import { existsSync } from "node:fs"; +import { collectSnapshot } from "../../herdr-insights/mods/insights/collector.mjs"; + +const [store, home, gate, delayMs = "40"] = process.argv.slice(2); +if (!store || !home || !gate) { + process.stderr.write("usage: lifecycle-launcher.mjs [delayMs]\n"); + process.exit(2); +} +while (!existsSync(gate)) { + await new Promise((resolve) => setTimeout(resolve, 5)); +} +try { + const snapshot = await collectSnapshot("snapshot", { + store, + home, + budgetMs: 0, + testStreamDelayMs: Number(delayMs), + workerAckMs: 5_000, + }); + process.stdout.write(`${JSON.stringify({ + launcherPid: process.pid, + status: snapshot.scan?.status ?? null, + workerPid: snapshot.scan?.workerPid ?? null, + pending: Boolean(snapshot.scan?.pendingFile), + warnings: snapshot.warnings ?? [], + })}\n`); +} catch (error) { + process.stderr.write(`${error.stack ?? error}\n`); + process.exitCode = 1; +} diff --git a/tests/herdr-insights/worker-long.integration.mjs b/tests/herdr-insights/worker-long.integration.mjs new file mode 100644 index 00000000..81cfd7ca --- /dev/null +++ b/tests/herdr-insights/worker-long.integration.mjs @@ -0,0 +1,182 @@ +#!/usr/bin/env node +// Opt-in integration proof that a real background worker can exceed 30s of +// wall-clock lifetime while finishing one giant compressed archive, with no +// repeated prefix decompression. NOT part of the routine suite. +// +// The stream is synthetic and the delay is an ARTIFICIAL test hook +// (testStreamDelayMs per decompressed chunk). This is not a real-user speed +// claim and not native Herdr proof. +import { mkdir, mkdtemp, readFile, rm, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import path from "node:path"; +import * as zlib from "node:zlib"; + +import { collectSnapshot } from "../../herdr-insights/mods/insights/collector.mjs"; +import { readWorkerClaim } from "../../herdr-insights/mods/insights/core/fence.mjs"; +import { snapshotRows } from "../../herdr-insights/mods/insights/core/store.mjs"; +import { isProcessAlive, sleep } from "../../herdr-insights/mods/insights/core/util.mjs"; + +const STREAM_DELAY_MS = 200; +const RAW_BYTES = 10_000_000; +const HOST_FOREGROUND_LIMIT_MS = 30_000; +const sourceConfig = { + sources: [{ + id: "codex", + label: "Codex", + adapter: "codex", + enabled: true, + roots: ["~/.codex/sessions"], + }], +}; + +function codexContent() { + const shared = [ + { type: "session_meta", timestamp: "2026-04-03T10:00:00Z", payload: { id: "long-worker", cwd: "/repo/long", model_provider: "openai" } }, + { type: "turn_context", timestamp: "2026-04-03T10:00:01Z", payload: { turn_id: "turn1", model: "long-model", cwd: "/repo/long" } }, + { + type: "token_usage_record", + timestamp: "2026-04-03T10:00:02Z", + payload: { + response_id: "resp1", + turn_id: "turn1", + usage: { input_tokens: 11, cached_input_tokens: 0, cache_write_input_tokens: 0, output_tokens: 3, reasoning_output_tokens: 0, total_tokens: 14 }, + }, + }, + { type: "event_msg", timestamp: "2026-04-03T10:00:03Z", payload: { type: "task_complete", turn_id: "turn1", duration_ms: 4000 } }, + ].map((record) => JSON.stringify(record)).join("\n"); + const noise = `${JSON.stringify({ type: "synthetic_noise", payload: "x".repeat(1024) })}\n`; + const filler = noise.repeat(Math.ceil(RAW_BYTES / noise.length)); + return `${shared}\n${filler}`; +} + +async function env(prefix) { + const root = await mkdtemp(path.join(tmpdir(), prefix)); + const home = path.join(root, "home"); + const store = path.join(root, "data", "insights.json"); + await mkdir(path.join(home, ".codex", "sessions", "long"), { recursive: true }); + await mkdir(path.dirname(store), { recursive: true }); + return { root, home, store }; +} + +async function readStore(storePath) { + try { + return JSON.parse(await readFile(storePath, "utf8")); + } catch (error) { + if (error?.code === "ENOENT") return null; + throw error; + } +} + +async function main() { + const ctx = await env("insights-long-worker-"); + const oracle = await env("insights-long-worker-oracle-"); + let workerPid = null; + const result = { + label: "ARTIFICIAL synthetic stream throttling; not real-user performance or native Herdr proof", + artificialStreamDelayMs: STREAM_DELAY_MS, + rawDecompressedBytes: 0, + }; + try { + const content = codexContent(); + result.rawDecompressedBytes = Buffer.byteLength(content); + const compressed = zlib.zstdCompressSync(Buffer.from(content)); + const fileName = "long.jsonl.zst"; + await collectSnapshot("configure", { store: ctx.store, home: ctx.home, config: sourceConfig }); + await writeFile(path.join(ctx.home, ".codex", "sessions", "long", fileName), compressed); + await writeFile(path.join(oracle.home, ".codex", "sessions", "long", fileName), compressed); + const oracleSnapshot = await collectSnapshot("configure", { + store: oracle.store, + home: oracle.home, + config: sourceConfig, + }); + if (oracleSnapshot.scan.status !== "complete") throw new Error("oracle scan did not complete"); + const expectedRows = snapshotRows(JSON.parse(await readFile(oracle.store, "utf8"))); + + const foregroundStarted = Date.now(); + const launched = await collectSnapshot("snapshot", { + store: ctx.store, + home: ctx.home, + budgetMs: 0, + testStreamDelayMs: STREAM_DELAY_MS, + workerAckMs: 10_000, + }); + const foregroundMs = Date.now() - foregroundStarted; + workerPid = launched.scan.workerPid; + result.foregroundMs = foregroundMs; + result.foregroundUnderHostLimit = foregroundMs < HOST_FOREGROUND_LIMIT_MS; + result.foregroundAbortedBytes = launched.scan.decompressedBytes ?? 0; + result.workerPid = workerPid; + result.foregroundStatus = launched.scan.status; + if (!workerPid) throw new Error("foreground did not promote a worker"); + if (foregroundMs >= HOST_FOREGROUND_LIMIT_MS) { + throw new Error(`foreground took ${foregroundMs}ms, over the host limit`); + } + if (result.foregroundAbortedBytes !== 0) { + throw new Error("aborted foreground prefix contributed decompressed bytes"); + } + + const workerStarted = Date.now(); + const seenPids = new Set(); + const progress = []; + let finalStore = null; + for (;;) { + const store = await readStore(ctx.store); + if (store?.scan?.workerPid) seenPids.add(store.scan.workerPid); + progress.push({ + atMs: Date.now() - workerStarted, + status: store?.scan?.status ?? null, + workerPid: store?.scan?.workerPid ?? null, + processed: store?.scan?.processed ?? 0, + total: store?.scan?.total ?? 0, + }); + if (store?.scan?.status === "complete") { + finalStore = store; + break; + } + if (Date.now() - workerStarted > 120_000) throw new Error("worker did not finish within 120s"); + await sleep(2_000); + } + result.workerLifetimeMs = Date.now() - workerStarted; + result.workerExceeded30s = result.workerLifetimeMs > 30_000; + result.distinctWorkerPids = [...seenPids]; + result.singleWorkerPid = seenPids.size === 1 && seenPids.has(workerPid); + result.workerPolls = progress.length; + result.finalStatus = finalStore.scan.status; + result.finalDecompressedBytes = finalStore.scan.stats?.decompressedBytes ?? 0; + result.singleDecoderPass = result.finalDecompressedBytes === result.rawDecompressedBytes; + result.filesRead = finalStore.scan.stats?.filesRead ?? 0; + result.rows = snapshotRows(finalStore); + result.rowsMatchOracle = JSON.stringify(result.rows) === JSON.stringify(expectedRows); + for (let attempt = 0; attempt < 30; attempt += 1) { + if ((await readWorkerClaim(ctx.store)) === null && !isProcessAlive(workerPid)) break; + await sleep(100); + } + result.claimCleared = (await readWorkerClaim(ctx.store)) === null; + result.workerExited = !isProcessAlive(workerPid); + + const failures = []; + if (!result.workerExceeded30s) failures.push(`worker lifetime ${result.workerLifetimeMs}ms <= 30s`); + if (!result.singleWorkerPid) failures.push("worker pid changed across polls"); + if (!result.singleDecoderPass) failures.push("decompressed bytes do not equal one full pass"); + if (result.filesRead !== 1) failures.push(`filesRead ${result.filesRead} !== 1`); + if (!result.rowsMatchOracle) failures.push("final rows differ from one-shot oracle"); + if (!result.claimCleared) failures.push("worker claim was not cleared"); + if (!result.workerExited) failures.push("worker process did not exit"); + result.ok = failures.length === 0; + if (failures.length) result.failures = failures; + process.stdout.write(`${JSON.stringify(result, null, 2)}\n`); + if (!result.ok) process.exitCode = 1; + } finally { + if (workerPid && isProcessAlive(workerPid)) { + try { process.kill(workerPid, "SIGTERM"); } catch { /* already gone */ } + await sleep(500); + if (isProcessAlive(workerPid)) { + try { process.kill(workerPid, "SIGKILL"); } catch { /* already gone */ } + } + } + await rm(ctx.root, { recursive: true, force: true }); + await rm(oracle.root, { recursive: true, force: true }); + } +} + +await main(); From fb6e88ee755404b34b22c79945c9b383233b73c4 Mon Sep 17 00:00:00 2001 From: GaganSD Date: Mon, 2 Nov 2026 19:46:36 +0530 Subject: [PATCH 07/15] fix(insights): publish gated live worker progress metadata --- herdr-insights/INSTALL.md | 9 + herdr-insights/README.md | 22 ++ herdr-insights/RECIPE.md | 11 +- herdr-insights/mods/insights/collector.mjs | 115 ++++++++-- .../mods/insights/core/progress.mjs | 56 ++++- herdr-insights/mods/insights/indexer.mjs | 51 +++-- herdr-insights/mods/insights/main.js | 59 ++++- tests/herdr-insights/analytics-ui.test.mjs | 60 ++++++ .../collector-live-progress.test.mjs | 203 ++++++++++++++++++ 9 files changed, 541 insertions(+), 45 deletions(-) create mode 100644 tests/herdr-insights/collector-live-progress.test.mjs diff --git a/herdr-insights/INSTALL.md b/herdr-insights/INSTALL.md index a4691fcd..f50d68b2 100644 --- a/herdr-insights/INSTALL.md +++ b/herdr-insights/INSTALL.md @@ -27,3 +27,12 @@ only if requested: click **Insights**, or add `-- --open` to the install command Stop and report; do not force overwrites or rebuild to bypass errors. Preserve local edits and any failed layout clone. Remove the app home's `.insights-install.lock` directory only after confirming no installer/uninstaller is running. + +Long scans are expected. Foreground refreshes are bounded and auto-continue; +giant zstd archives may finish in one managed local Node worker. Do not raise +Herdr's exec timeout. Lock/claim files fail closed: never delete a live or +foreign owner by age. Remove stale lock, `lock.acquire-*`, or quarantine +leftovers only after proving no writer is running and rechecking first. + +Node.js **22.15+** is required. A partial save updates `savedAt`/`lastScanAt`, +not `lastCompletedScanAt`. diff --git a/herdr-insights/README.md b/herdr-insights/README.md index ac1dfcc9..d10baaf4 100644 --- a/herdr-insights/README.md +++ b/herdr-insights/README.md @@ -37,6 +37,28 @@ click-state rules. Set `HERDR_APP_HOME` to use a different app home. - Open **Settings** to choose sources, edit paths, toggle cached tokens, or change the palette and refresh interval. +## Long scans and recovery + +- A foreground refresh is a bounded slice. When work remains it returns + committed data, the dashboard keeps rendering, and the view automatically + continues. Herdr's exec timeout is unchanged. +- A giant non-seekable zstd archive that cannot finish in one slice continues + in one managed local detached Node worker. Polling never restarts it. There + is no fabricated compressed-offset resume; after a crash the unfinished + archive and any uncheckpointed batch are replayed, while completed files stay + committed. +- `prepare` and status reads use a metadata-only cache plus sanitized live progress + (current file, decoded bytes, counters). Only a matching live local worker + claim and writer lock at the same generation may publish it; rows, config, + and accounting always come from committed state. +- `savedAt`/`lastScanAt` is the last persisted write; `lastCompletedScanAt` is + the last fully completed scan. A saved partial scan is not a completed scan. +- Locks fail closed. Empty, foreign, or legacy lock state is never age-evicted + automatically. Clean up manually only after proving no writer is running; + `lock.acquire-*`/quarantine leftovers are inactive, but recheck before + deleting anything. +- Node.js 22.15+ is required. + ## Privacy No network requests or uploads. The mod runs a local Node helper, reads session diff --git a/herdr-insights/RECIPE.md b/herdr-insights/RECIPE.md index 3158a1e2..a3b8f0f9 100644 --- a/herdr-insights/RECIPE.md +++ b/herdr-insights/RECIPE.md @@ -162,8 +162,15 @@ including tool-result, standalone usage, compaction, and branch-summary events. Task duration uses exact boundaries or conservative capped spans, never idle session lifetime. Use local dates, not UTC days. Append-only incremental caches may require explicit rebuild after same-size rewrites. Use ownership-checked store locks; -never steal live/unknown owners' locks based on age. Preserve corrupt stores with -a backup before rebuilding; read-only prepare must not repair or create stores. +never steal live/unknown owners' locks based on age or evict empty/foreign locks +automatically. Preserve corrupt stores with a backup before rebuilding; read-only +prepare must not repair or create stores. Foreground slices are bounded and +auto-continue; a managed local worker may finish a giant non-seekable zstd archive. +There is no arbitrary compressed-offset resume: crash recovery replays the +unfinished archive and uncheckpointed batch, keeping completed commits. Live +progress is a metadata-only overlay gated by the matching live claim and writer +lock; rows, config, and accounting stay committed. `savedAt`/`lastScanAt` is not +`lastCompletedScanAt`. Require Node >=22.15; do not change Herdr's exec timeout. Never modify shipped layouts/mods, keys, source transcripts, or existing usage data. Write only the owned personal layout clone. Preserve collisions and local diff --git a/herdr-insights/mods/insights/collector.mjs b/herdr-insights/mods/insights/collector.mjs index f47b3537..3041aa86 100644 --- a/herdr-insights/mods/insights/collector.mjs +++ b/herdr-insights/mods/insights/collector.mjs @@ -9,7 +9,9 @@ import { migrateLoadedConfig, normalizeConfig, sourcesSignature } from "./core/c import { bumpFence, readEffectiveFence, + readWorkerClaim, restoreFence, + workerClaimActive, } from "./core/fence.mjs"; import { DEFAULT_WORKER_ACK_MS, @@ -17,7 +19,7 @@ import { waitForIndexerAck, } from "./core/indexer-control.mjs"; import { scanSources } from "./core/scan.mjs"; -import { progressWriter } from "./core/progress.mjs"; +import { progressWriter, readLiveProgress } from "./core/progress.mjs"; import { clearCollectedData, emptyScanState, @@ -171,23 +173,79 @@ function busyOverrides(owner) { }; } +function liveView(live) { + return { + file: live.file, + phase: live.phase, + processed: live.processed, + total: live.total, + filesRead: live.filesRead, + bytesRead: live.bytesRead, + decodedBytes: live.decodedBytes, + updatedAt: live.updatedAt, + }; +} + +/** + * Overlay metadata-only worker progress on committed rows/stats. The sidecar is + * not authoritative: it is accepted only while the matching local claim AND the + * store writer lock are owned by the same live pid at the committed generation. + * A stale, dead, replaced, or foreign worker is ignored, and only current file, + * counters, phase, and timestamps cross the boundary. Rows, config, accounting, + * and timestamps are never taken from progress. + */ +async function overlayLiveProgress(storePath, snapshot) { + const scan = snapshot?.scan; + if (!scan || !["running", "partial", "starting"].includes(scan.status)) return snapshot; + if (!Number.isInteger(scan.generation)) return snapshot; + const live = await readLiveProgress(storePath); + if (!live) return snapshot; + if (live.store !== path.resolve(storePath) || live.generation !== scan.generation) return snapshot; + const [claim, owner] = await Promise.all([ + readWorkerClaim(storePath), + readStoreLockInfo(storePath), + ]); + if (!claim || claim.host !== os.hostname() || !workerClaimActive(claim)) return snapshot; + if (claim.pid !== live.pid || claim.generation !== live.generation || claim.token !== live.token) { + return snapshot; + } + if (!owner || owner.role !== "worker" || owner.host !== os.hostname()) return snapshot; + if (owner.pid !== live.pid || Number(owner.generation) !== live.generation) return snapshot; + if (Number.isInteger(scan.workerPid) && scan.workerPid !== live.pid) return snapshot; + return { + ...snapshot, + scan: { + ...scan, + status: "running", + workerPid: live.pid, + live: liveView(live), + }, + }; +} + /** * Committed presentation for a reader. Never takes the writer lock, so a long * indexer cannot make status/prepare wait for a scan. The cache is used only - * when it is at least as new as the committed store. + * when it is at least as new as the committed store. Live worker progress is a + * sanitized metadata overlay; committed rows/stats remain authoritative. */ export async function readCommittedSnapshot(storePath, { now = Date.now } = {}) { const nowIso = new Date(now()).toISOString(); const cached = await readPresentationCache(storePath); - if (cached) return buildSnapshotFromCache(cached, nowIso); - const loaded = await loadStore(storePath, { readOnly: true }); - return buildSnapshot( - loaded.store, - scanResultFromStore(loaded.store), - nowIso, - loaded.warnings, - loaded.errors, - ); + let snapshot; + if (cached) { + snapshot = buildSnapshotFromCache(cached, nowIso); + } else { + const loaded = await loadStore(storePath, { readOnly: true }); + snapshot = buildSnapshot( + loaded.store, + scanResultFromStore(loaded.store), + nowIso, + loaded.warnings, + loaded.errors, + ); + } + return overlayLiveProgress(storePath, snapshot); } async function busySnapshot(storePath, owner, now) { @@ -322,6 +380,20 @@ async function scanAndPersist( const startedAt = now(); const deadline = startedAt + Math.max(0, Number(budgetMs) || 0); const progress = progressWriter(storePath); + // Foreground stream progress carries only the open file and decoded bytes. + // Keep the last accepted discovery counters so a mid-file update never + // replaces useful progress with 0/1, and label decoded bytes separately from + // the committed accepted stats. + let live = { + processed: 0, + total: 0, + file: null, + filesRead: 0, + bytesRead: 0, + decodedBytes: 0, + phase: "scan", + }; + const publish = (state) => progress.update({ ...live, ...state, mode, workerPid: null }); let result; try { result = await scanSources(store, { @@ -331,14 +403,19 @@ async function scanAndPersist( mode, streamDelayMs: testStreamDelayMs, shouldStop: () => false, - onProgress: (state) => progress.update({ ...state, workerPid: null }), - onStreamProgress: (state) => progress.update({ - ...state, - processed: 0, - total: 1, - mode, - workerPid: null, - }), + onProgress: (state) => { + live = { ...live, ...state, decodedBytes: 0 }; + publish(live); + }, + onStreamProgress: (state) => { + live = { + ...live, + file: state.file ?? live.file, + phase: state.phase ?? "decompress", + decodedBytes: Number.isInteger(state.bytes) ? state.bytes : live.decodedBytes, + }; + publish(live); + }, }); } finally { progress.clear(); diff --git a/herdr-insights/mods/insights/core/progress.mjs b/herdr-insights/mods/insights/core/progress.mjs index be37ce7a..099e3bcc 100644 --- a/herdr-insights/mods/insights/core/progress.mjs +++ b/herdr-insights/mods/insights/core/progress.mjs @@ -1,5 +1,58 @@ import { randomUUID } from "node:crypto"; import { renameSync, unlinkSync, writeFileSync } from "node:fs"; +import { readFile } from "node:fs/promises"; +import { isPlainObject } from "./util.mjs"; + +const MAX_FILE_CHARS = 4_096; +const ALLOWED_PHASES = new Set(["scan", "decompress"]); + +function nonNegInt(value) { + return Number.isInteger(value) && value >= 0 ? value : null; +} + +function shortString(value, max = 256) { + return typeof value === "string" && value.length > 0 && value.length <= max ? value : null; +} + +/** + * Metadata-only allowlist for the live progress sidecar. Unknown fields (rows, + * config, accounting, arbitrary diagnostics) are dropped before they can reach + * a presentation snapshot. Full identity is required: a foreground sidecar + * without a worker pid/token is deliberately not a live-worker overlay. + */ +export function liveProgressView(value) { + if (!isPlainObject(value)) return null; + const pid = nonNegInt(value.workerPid ?? value.pid); + const generation = nonNegInt(value.generation); + const token = shortString(value.token); + const store = shortString(value.store, 8_192); + if (!pid || pid <= 0 || generation === null || !token || !store) return null; + const file = shortString(value.file, MAX_FILE_CHARS); + const phase = ALLOWED_PHASES.has(value.phase) ? value.phase : null; + return { + pid, + generation, + token, + store, + file, + phase, + processed: nonNegInt(value.processed), + total: nonNegInt(value.total), + filesRead: nonNegInt(value.filesRead), + bytesRead: nonNegInt(value.bytesRead), + decodedBytes: nonNegInt(value.decodedBytes ?? value.bytes), + updatedAt: shortString(value.updatedAt, 64), + }; +} + +/** Read and sanitize the best-effort progress sidecar; missing/corrupt is null. */ +export async function readLiveProgress(storePath) { + try { + return liveProgressView(JSON.parse(await readFile(`${storePath}.progress.json`, "utf8"))); + } catch { + return null; + } +} /** * Best-effort scan progress sidecar next to the store. The native view polls @@ -18,7 +71,8 @@ export function progressWriter(storePath) { lastWrite = now; try { // Atomic replacement also avoids following an existing sidecar link. - writeFileSync(tempPath, `${JSON.stringify(state)}\n`, { mode: 0o600, flag: "wx" }); + const payload = { ...state, updatedAt: new Date(now).toISOString() }; + writeFileSync(tempPath, `${JSON.stringify(payload)}\n`, { mode: 0o600, flag: "wx" }); renameSync(tempPath, progressPath); } catch { /* progress is best-effort */ } finally { diff --git a/herdr-insights/mods/insights/indexer.mjs b/herdr-insights/mods/insights/indexer.mjs index 32bee306..b0a63a29 100644 --- a/herdr-insights/mods/insights/indexer.mjs +++ b/herdr-insights/mods/insights/indexer.mjs @@ -132,6 +132,37 @@ export async function runIndexer(argv) { process.on("SIGINT", onSignal); const progress = progressWriter(storePath); + // Metadata-only live view. Discovery counters survive stream updates so the + // UI never loses the accepted progress while a single giant file decodes. + let live = { + processed: 0, + total: 0, + file: null, + filesRead: 0, + bytesRead: 0, + decodedBytes: 0, + phase: "scan", + }; + const publishProgress = (state = {}) => { + if (Number.isInteger(state.processed)) live.processed = state.processed; + if (Number.isInteger(state.total)) live.total = state.total; + if (typeof state.file === "string") live.file = state.file; + if (typeof state.phase === "string") live.phase = state.phase; + if (Number.isInteger(state.decodedBytes)) live.decodedBytes = state.decodedBytes; + if (Number.isInteger(state.bytes)) live.decodedBytes = state.bytes; + if (Number.isInteger(state.filesRead)) live.filesRead = state.filesRead; + if (Number.isInteger(state.bytesRead)) live.bytesRead = state.bytesRead; + if (Number.isInteger(state.stats?.filesRead)) live.filesRead = state.stats.filesRead; + if (Number.isInteger(state.stats?.bytesRead)) live.bytesRead = state.stats.bytesRead; + progress.update({ + ...live, + mode: "worker", + workerPid: process.pid, + generation, + token: claimToken, + store: storePath, + }); + }; let lastSave = Date.now(); try { const result = await withStoreLock(storePath, async () => { @@ -191,20 +222,10 @@ export async function runIndexer(argv) { streamDelayMs: testStreamDelayMs, shouldStop: () => stopping || fenceAborted, onProgress: (state) => { - progress.update({ - ...state, - mode: "worker", - workerPid: process.pid, - generation, - }); + publishProgress({ ...state, decodedBytes: 0 }); }, onStreamProgress: (state) => { - progress.update({ - ...state, - mode: "worker", - workerPid: process.pid, - generation, - }); + publishProgress(state); }, onCheckpoint: async (state) => { if (testFailAfterCheckpointMs > 0) { @@ -216,15 +237,13 @@ export async function runIndexer(argv) { scanViewUpdate(store, state); store.scan.workerPid = process.pid; const finished = new Date().toISOString(); - progress.update({ + publishProgress({ processed: state.processed, total: state.total, file: state.file, filesRead: state.stats?.filesRead ?? 0, bytesRead: state.stats?.bytesRead ?? 0, - mode: "worker", - workerPid: process.pid, - generation, + decodedBytes: 0, phase: "scan", }); if (Date.now() - lastSave >= SAVE_INTERVAL_MS) { diff --git a/herdr-insights/mods/insights/main.js b/herdr-insights/mods/insights/main.js index ba44dd55..51d44ac7 100644 --- a/herdr-insights/mods/insights/main.js +++ b/herdr-insights/mods/insights/main.js @@ -51,6 +51,12 @@ const CELL_GAP = 2; const DAY_LABEL_WIDTH = 30; const TAB = { name: "Insights", mod: "insights" }; +function shortFileName(value) { + if (typeof value !== "string" || value.length === 0) return ""; + const index = Math.max(value.lastIndexOf("/"), value.lastIndexOf("\\")); + return index >= 0 ? value.slice(index + 1) : value; +} + /** One Insights tab per space: focus it if it exists, otherwise create it. */ function showInsights() { const active = spaces().find((space) => space.active); @@ -754,12 +760,35 @@ export default class Insights extends View { let next = null; try { const parsed = JSON.parse(await readTextFile(this.progressPath())); - if (parsed && Number.isFinite(Number(parsed.processed)) && Number.isFinite(Number(parsed.total))) next = parsed; + if (parsed && typeof parsed === "object" && !Array.isArray(parsed)) { + // Merge only allowlisted display metadata into the last foreground + // view: a mid-file stream update must not drop discovery counters. + const merged = { ...(this.scanProgress ?? {}) }; + let useful = false; + for (const key of ["processed", "total", "filesRead", "bytesRead", "bytes", "decodedBytes"]) { + const value = Number(parsed[key]); + if (Number.isFinite(value) && value >= 0) { + merged[key === "bytes" ? "decodedBytes" : key] = Math.floor(value); + useful = true; + } + } + if (typeof parsed.file === "string") { + merged.file = parsed.file; + useful = true; + } + if (typeof parsed.phase === "string") { + merged.phase = parsed.phase; + useful = true; + } + if (useful) next = merged; + } } catch { /* the progress file is best-effort */ } if (generation !== this.progressGeneration || !this.loading) return; this.progressTask = null; const changed = (next?.processed ?? null) !== (this.scanProgress?.processed ?? null) - || (next?.file ?? null) !== (this.scanProgress?.file ?? null); + || (next?.file ?? null) !== (this.scanProgress?.file ?? null) + || (next?.decodedBytes ?? null) !== (this.scanProgress?.decodedBytes ?? null) + || (next?.phase ?? null) !== (this.scanProgress?.phase ?? null); this.scanProgress = next; if (changed) progressCx.notify(); }); @@ -814,14 +843,26 @@ export default class Insights extends View { scanStatusLabel(scan) { if (!scan || typeof scan !== "object") return ""; if (scan.status === "reconfiguring") return "Applying configuration…"; + const live = scan.live && typeof scan.live === "object" ? scan.live : null; + const total = Number(live?.total ?? scan.total) || 0; + const processed = Number(live?.processed ?? scan.processed) || 0; + const file = shortFileName(live?.file); + const decodedBytes = Number(live?.decodedBytes) || 0; if (scan.status === "running" || scan.status === "starting") { - return Number(scan.total) > 0 - ? `Indexing ${formatInt(scan.processed)} of ${formatInt(scan.total)} files…` + if (live && (file || decodedBytes > 0)) { + const counts = total > 0 + ? `${formatInt(processed)} of ${formatInt(total)} files` + : "in the background"; + return [`Indexing ${counts}`, file, decodedBytes > 0 ? `${formatBytes(decodedBytes)} decoded` : null] + .filter(Boolean).join(" · ") + "…"; + } + return total > 0 + ? `Indexing ${formatInt(processed)} of ${formatInt(total)} files…` : "Indexing in the background…"; } if (scan.status === "partial") { - return Number(scan.total) > 0 - ? `Scanning ${formatInt(scan.processed)} of ${formatInt(scan.total)} files…` + return total > 0 + ? `Scanning ${formatInt(processed)} of ${formatInt(total)} files…` : "Scanning…"; } return ""; @@ -1453,11 +1494,15 @@ export default class Insights extends View { const total = Number(progress?.total) || 0; const processed = Number(progress?.processed) || 0; const file = typeof progress?.file === "string" ? progress.file : ""; + const decodedBytes = Number(progress?.decodedBytes ?? progress?.bytes) || 0; return stateCard(c, { tone: loading ? c.accent : c.destructive, title: loading ? "Loading local insights" : "Insights needs its collector", body: loading - ? (total > 0 ? `Scanning ${formatInt(processed)} of ${formatInt(total)} files` : null) + ? [ + total > 0 ? `Scanning ${formatInt(processed)} of ${formatInt(total)} files` : null, + decodedBytes > 0 ? `${formatBytes(decodedBytes)} decoded` : null, + ].filter(Boolean).join(" · ") || null : (this.error || "The collector did not return a snapshot yet."), detail: loading && file ? file : null, action: loading || this.busy() ? null : button("insights-initial-retry", "Retry", (_event, clickCx) => this.refresh(clickCx), c, { diff --git a/tests/herdr-insights/analytics-ui.test.mjs b/tests/herdr-insights/analytics-ui.test.mjs index 1684e72d..f919363b 100644 --- a/tests/herdr-insights/analytics-ui.test.mjs +++ b/tests/herdr-insights/analytics-ui.test.mjs @@ -1358,3 +1358,63 @@ test("scan progress awaits the host read and ignores results after cancellation" assert.equal(instance.scanProgress, null); assert.equal(instance.progressTask, null); }); + +test("progress poll merges mid-file stream state, keeps counts, and stops when settled", async () => { + const state = useHerdr({ + readTextFile: async () => JSON.stringify({ + file: "/tmp/open-giant.jsonl.zst", + decodedBytes: 4096, + rows: [{ injected: true }], + config: { injected: true }, + }), + }); + const instance = blankInstance(); + instance.sourceInputs = new Map(); + instance.loading = true; + const { cx, calls } = liveCx(); + instance.armProgress(cx); + instance.scanProgress = { processed: 7, total: 11, file: "/tmp/previous.jsonl" }; + calls.timers[0].fn(cx); + await flush(); + assert.equal(instance.scanProgress.processed, 7, "stream progress keeps accepted counts"); + assert.equal(instance.scanProgress.total, 11); + assert.equal(instance.scanProgress.decodedBytes, 4096); + assert.equal(instance.scanProgress.file, "/tmp/open-giant.jsonl.zst"); + assert.equal(instance.scanProgress.rows, undefined, "only display metadata is accepted"); + assert.equal(instance.scanProgress.config, undefined); + + instance.loading = false; + calls.timers[0].fn(cx); + await flush(); + assert.equal(instance.scanProgress, null, "completion stops and clears the poll"); + assert.equal(instance.progressTimer, null); + const reads = state.runtimeReads.length; + calls.timers[0].fn(cx); + await flush(); + assert.equal(state.runtimeReads.length, reads, "a settled poll never reads again"); + + instance.loading = true; + instance.armProgress(cx); + instance.close(); + assert.equal(instance.scanProgress, null, "close cancels in-flight progress"); + assert.equal(instance.progressTimer, null); +}); + +test("running worker status shows current file and decoded bytes", () => { + const instance = blankInstance(); + const label = instance.scanStatusLabel({ + status: "running", + processed: 2, + total: 10, + live: { + file: "/home/u/.codex/sessions/open-giant.jsonl.zst", + processed: 3, + total: 10, + decodedBytes: 123_456, + phase: "decompress", + }, + }); + assert.match(label, /Indexing 3 of 10 files/); + assert.match(label, /open-giant\.jsonl\.zst/, "the live file is visible"); + assert.match(label, /decoded/, "in-progress decoded bytes are visible"); +}); diff --git a/tests/herdr-insights/collector-live-progress.test.mjs b/tests/herdr-insights/collector-live-progress.test.mjs new file mode 100644 index 00000000..0a116608 --- /dev/null +++ b/tests/herdr-insights/collector-live-progress.test.mjs @@ -0,0 +1,203 @@ +import assert from "node:assert/strict"; +import { mkdir, mkdtemp, rm, writeFile } from "node:fs/promises"; +import { hostname } from "node:os"; +import { tmpdir } from "node:os"; +import path from "node:path"; +import test from "node:test"; + +import { + collectSnapshot, + readCommittedSnapshot, +} from "../../herdr-insights/mods/insights/collector.mjs"; +import { normalizeConfig } from "../../herdr-insights/mods/insights/core/config.mjs"; +import { + atomicWriteJson, + withStoreLock, +} from "../../herdr-insights/mods/insights/core/util.mjs"; +import { + workerClaimPath, +} from "../../herdr-insights/mods/insights/core/fence.mjs"; +import { + emptyStore, + newFileEntry, + saveStore, +} from "../../herdr-insights/mods/insights/core/store.mjs"; +import { run as runPresenter } from "../../herdr-insights/mods/insights/presenter.mjs"; + +const WORKER_TOKEN = "live-progress-job-token"; + +async function fixture() { + const root = await mkdtemp(path.join(tmpdir(), "insights-live-progress-")); + const storePath = path.join(root, "data", "insights.json"); + await mkdir(path.dirname(storePath), { recursive: true }); + const store = emptyStore(normalizeConfig({ sources: [] })); + store.generation = 4; + store.lastScanAt = "2026-04-01T12:00:00.000Z"; + store.lastCompletedScanAt = "2026-04-01T11:00:00.000Z"; + store.scan = { + status: "running", + generation: 4, + startedAt: "2026-04-01T12:00:00.000Z", + updatedAt: "2026-04-01T12:00:00.000Z", + completedAt: null, + processed: 2, + total: 10, + stats: { + files: 2, + filesRead: 2, + bytesRead: 800, + unchangedFiles: 0, + errors: 0, + decompressedBytes: 500, + }, + currentFile: null, + pendingFile: null, + workerPid: null, + error: null, + }; + const entry = newFileEntry(store, { + sourceId: "pi", + path: "/tmp/committed.jsonl", + adapter: "pi", + size: 10, + mtimeMs: 1, + }); + entry.rows.push({ + day: "2026-04-01", + session: "committed-session", + harness: "pi", + provider: "provider", + model: "model", + project: "project", + input: 12, + output: 3, + cacheRead: 0, + cacheWrite: 0, + reasoning: 0, + messages: 1, + }); + await saveStore(storePath, store, { status: "running", now: "2026-04-01T12:00:00.000Z" }); + await atomicWriteJson(workerClaimPath(storePath), { + version: 1, + token: WORKER_TOKEN, + pid: process.pid, + host: hostname(), + generation: 4, + startedAt: Date.now(), + at: Date.now(), + }); + return { root, storePath }; +} + +function progress(storePath, overrides = {}) { + return { + workerPid: process.pid, + pid: process.pid, + generation: 4, + token: WORKER_TOKEN, + store: path.resolve(storePath), + file: "/tmp/open-giant.jsonl.zst", + processed: 3, + total: 10, + filesRead: 2, + bytesRead: 900, + decodedBytes: 123_456, + phase: "decompress", + rows: [{ injected: true }], + config: { injected: true }, + stats: { input: 999_999 }, + accounting: { input: 999_999 }, + ...overrides, + }; +} + +async function writeProgress(storePath, overrides) { + await writeFile(`${storePath}.progress.json`, `${JSON.stringify(progress(storePath, overrides))}\n`, { mode: 0o600 }); +} + +test("prepare/status/read overlay only allowlisted live worker metadata", async () => { + const ctx = await fixture(); + try { + const before = await readCommittedSnapshot(ctx.storePath); + await withStoreLock(ctx.storePath, async () => { + await writeProgress(ctx.storePath); + const snapshot = await readCommittedSnapshot(ctx.storePath); + assert.equal(snapshot.scan.status, "running"); + assert.equal(snapshot.scan.workerPid, process.pid); + assert.deepEqual(snapshot.scan.live, { + file: "/tmp/open-giant.jsonl.zst", + phase: "decompress", + processed: 3, + total: 10, + filesRead: 2, + bytesRead: 900, + decodedBytes: 123_456, + updatedAt: null, + }); + assert.deepEqual(snapshot.rows, before.rows, "committed rows are unchanged"); + assert.deepEqual(snapshot.config, before.config, "committed config is unchanged"); + assert.deepEqual(snapshot.stats, before.stats, "committed accepted stats are unchanged"); + assert.equal(snapshot.stats.decompressedBytes, 500); + assert.equal(snapshot.lastCompletedScanAt, before.lastCompletedScanAt); + assert.equal(snapshot.scan.live.rows, undefined); + assert.equal(snapshot.scan.live.config, undefined); + assert.equal(snapshot.scan.live.stats, undefined); + + const status = await collectSnapshot("status", { store: ctx.storePath }); + assert.equal(status.scan.live.decodedBytes, 123_456); + assert.deepEqual(status.rows, before.rows); + + const prepared = await runPresenter(["prepare", "--store", ctx.storePath]); + assert.equal(prepared.exitCode, 0); + const payload = JSON.parse(prepared.stdout); + assert.equal(payload.snapshot.scan.live.file, "/tmp/open-giant.jsonl.zst"); + assert.equal(payload.snapshot.scan.live.decodedBytes, 123_456); + assert.equal(payload.snapshot.rowCount, before.rows.length); + }, { timeoutMs: 2_000, meta: { role: "worker", generation: 4 } }); + } finally { + await rm(ctx.root, { recursive: true, force: true }); + } +}); + +test("stale job pid, generation, token, store, or non-worker lock reject live progress", async () => { + const ctx = await fixture(); + try { + await withStoreLock(ctx.storePath, async () => { + const cases = [ + ["dead pid", { workerPid: 999_999_999, pid: 999_999_999 }], + ["old generation", { generation: 3 }], + ["replaced token", { token: "replaced-job-token" }], + ["foreign store", { store: "/tmp/other-insights.json" }], + ]; + for (const [label, overrides] of cases) { + await writeProgress(ctx.storePath, overrides); + const snapshot = await readCommittedSnapshot(ctx.storePath); + assert.equal(snapshot.scan.live, undefined, label); + assert.equal(snapshot.scan.workerPid, null, label); + } + }, { timeoutMs: 2_000, meta: { role: "worker", generation: 4 } }); + + await withStoreLock(ctx.storePath, async () => { + await writeProgress(ctx.storePath); + const snapshot = await readCommittedSnapshot(ctx.storePath); + assert.equal(snapshot.scan.live, undefined, "a non-worker lock owner rejects progress"); + }, { timeoutMs: 2_000, meta: { role: "collector", generation: 4 } }); + } finally { + await rm(ctx.root, { recursive: true, force: true }); + } +}); + +test("completed snapshots never expose residual worker progress", async () => { + const ctx = await fixture(); + try { + await withStoreLock(ctx.storePath, async () => { + await saveStore(ctx.storePath, emptyStore(normalizeConfig({ sources: [] })), { completed: true }); + await writeProgress(ctx.storePath); + const snapshot = await readCommittedSnapshot(ctx.storePath); + assert.equal(snapshot.scan.status, "complete"); + assert.equal(snapshot.scan.live, undefined); + }, { timeoutMs: 2_000, meta: { role: "worker", generation: 4 } }); + } finally { + await rm(ctx.root, { recursive: true, force: true }); + } +}); From 55b7d459b1bb51ebe2ae5de79307c7818df64ac2 Mon Sep 17 00:00:00 2001 From: GaganSD Date: Mon, 2 Nov 2026 19:59:26 +0530 Subject: [PATCH 08/15] bench(insights): measure large-history performance and RSS --- herdr-insights/README.md | 18 + scripts/herdr-insights/benchmark.mjs | 811 +++++++++++++++++++++++++ scripts/herdr-insights/clone-probe.mjs | 32 + 3 files changed, 861 insertions(+) create mode 100644 scripts/herdr-insights/benchmark.mjs create mode 100644 scripts/herdr-insights/clone-probe.mjs diff --git a/herdr-insights/README.md b/herdr-insights/README.md index d10baaf4..f2316ce9 100644 --- a/herdr-insights/README.md +++ b/herdr-insights/README.md @@ -79,3 +79,21 @@ without modifying them (see [INSTALL.md](INSTALL.md)). Developer tests, fixtures, and release validation live outside this distribution; see the repo's [development guide](../docs/herdr-insights-development.md). They are not installation steps. + +### Reproducible performance benchmark + +The opt-in benchmark uses only synthetic sessions and never reads a real home: + +```sh +node scripts/herdr-insights/benchmark.mjs +``` + +It starts three fresh presenter processes per case and records medians plus peak +RSS, source/store/payload bytes, filesRead, bytesRead, structuredClone counts, +and every gate in `research/issue-109/round2-perf-benchmark.json` (gitignored). +Cases cover 1,000 and 10,008 unique compressed Codex sessions cold and warm, a +single 250,000-event session, and 50MiB/500MiB plain and zstd payloads with fixed +usage cardinality. It also checks cached `status` latency while a stream-throttled +worker indexes. The benchmark is a development measurement, not native Herdr +proof and not a Node 22 compatibility certification; see the artifact's pending +gates for that separate work. diff --git a/scripts/herdr-insights/benchmark.mjs b/scripts/herdr-insights/benchmark.mjs new file mode 100644 index 00000000..e715b230 --- /dev/null +++ b/scripts/herdr-insights/benchmark.mjs @@ -0,0 +1,811 @@ +#!/usr/bin/env node +// Opt-in reproducible performance/RSS benchmark for Insights. +// +// Everything is synthetic and lives in a private temp directory. Fixtures are +// generated in this parent process BEFORE any measured child; the measured +// children are fresh presenter CLI processes. On macOS/Linux their peak RSS is +// read from /usr/bin/time. The 500MiB input is removed in finally. +// +// Usage: +// node scripts/herdr-insights/benchmark.mjs [--keep] [--json=] +// +// This is a development benchmark, not native Herdr proof and never reads the +// operator's real home directories. +import assert from "node:assert/strict"; +import { spawn } from "node:child_process"; +import { createHash } from "node:crypto"; +import { createReadStream, createWriteStream } from "node:fs"; +import { + copyFile, + link, + mkdir, + mkdtemp, + readFile, + rm, + stat, + unlink, + writeFile, +} from "node:fs/promises"; +import os from "node:os"; +import path from "node:path"; +import { pipeline } from "node:stream/promises"; +import { fileURLToPath, pathToFileURL } from "node:url"; +import * as zlib from "node:zlib"; + +import { collectSnapshot } from "../../herdr-insights/mods/insights/collector.mjs"; +import { clearWorkerClaim, readWorkerClaim } from "../../herdr-insights/mods/insights/core/fence.mjs"; +import { isProcessAlive, sleep } from "../../herdr-insights/mods/insights/core/util.mjs"; + +const MiB = 1024 * 1024; +const REPO_ROOT = fileURLToPath(new URL("../..", import.meta.url)); +const PRESENTER = path.join(REPO_ROOT, "herdr-insights/mods/insights/presenter.mjs"); +const PROBE = path.join(REPO_ROOT, "scripts/herdr-insights/clone-probe.mjs"); +const FIXTURE = path.join(REPO_ROOT, "tests/herdr-insights/fixtures/codex-modern.jsonl.zst"); +const DEFAULT_JSON = path.join(REPO_ROOT, "research/issue-109/round2-perf-benchmark.json"); +const COLD_CODEX_COUNTS = [1_000, 10_008]; +const SAMPLES = 3; +const STATUS_SAMPLES = 5; +const USAGE_EVENTS = 250_000; +const GIANT_SIZES = [50 * MiB, 500 * MiB]; + +// One checked-in codex-modern fixture contributes these committed values +// (token_count twins are deduped against their token_usage_record). +const CODEX_FILE_ORACLE = { + tokens: { input: 55, output: 30, cacheRead: 90, cacheWrite: 5, reasoning: 7, total: 180 }, + messages: 2, + tasks: 2, + activeMs: 7_000, + longestTaskMs: 4_000, + sessions: 1, +}; + +const args = process.argv.slice(2); +const keep = args.includes("--keep"); +const jsonArg = args.find((arg) => arg.startsWith("--json=")); +const outputPath = jsonArg ? path.resolve(REPO_ROOT, jsonArg.slice("--json=".length)) : DEFAULT_JSON; + +function median(values) { + const sorted = [...values].sort((left, right) => left - right); + const middle = Math.floor(sorted.length / 2); + return sorted.length % 2 === 1 + ? sorted[middle] + : (sorted[middle - 1] + sorted[middle]) / 2; +} + +function sum(values) { + return values.reduce((total, value) => total + value, 0); +} + +async function fileSha256(file) { + const hash = createHash("sha256"); + await pipeline(createReadStream(file), hash); + return hash.digest("hex"); +} + +function timeArgs() { + if (process.platform === "darwin") return ["-l"]; + if (process.platform === "linux") return ["-v"]; + return null; +} + +function parsePeakRss(stderr, platform) { + const text = stderr.toString("utf8"); + const match = platform === "darwin" + ? /^\s*(\d+)\s+maximum resident set size$/im.exec(text) + : /Maximum resident set size \(kbytes\):\s*(\d+)/i.exec(text); + if (!match) return null; + return Number(match[1]) * (platform === "darwin" ? 1 : 1024); +} + +async function readCloneCounts(logPath) { + try { + const lines = (await readFile(logPath, "utf8")).trim().split("\n").filter(Boolean); + const counts = lines.map((line) => JSON.parse(line)); + return { + processes: counts.length, + calls: sum(counts.map((entry) => Number(entry.total) || 0)), + wholeStore: sum(counts.map((entry) => Number(entry.wholeStore) || 0)), + }; + } catch (error) { + if (error?.code === "ENOENT") return { processes: 0, calls: 0, wholeStore: 0, missing: true }; + throw error; + } +} + +async function resetLog(file) { + await unlink(file).catch((error) => { + if (error?.code !== "ENOENT") throw error; + }); +} + +function runPresenter(commandArgs, cloneLog) { + const time = timeArgs(); + const executable = time ? "/usr/bin/time" : process.execPath; + const spawnArgs = time + ? [...time, process.execPath, PRESENTER, ...commandArgs] + : [PRESENTER, ...commandArgs]; + const existingOptions = process.env.NODE_OPTIONS ? `${process.env.NODE_OPTIONS} ` : ""; + const child = spawn(executable, spawnArgs, { + cwd: REPO_ROOT, + env: { + ...process.env, + TZ: "UTC", + INSIGHTS_CLONE_LOG: cloneLog, + NODE_OPTIONS: `${existingOptions}--import ${pathToFileURL(PROBE).href}`.trim(), + }, + stdio: ["ignore", "pipe", "pipe"], + }); + const started = performance.now(); + const stdout = []; + const stderr = []; + child.stdout.on("data", (chunk) => stdout.push(chunk)); + child.stderr.on("data", (chunk) => stderr.push(chunk)); + const timeout = setTimeout(() => child.kill("SIGKILL"), 180_000); + return new Promise((resolve, reject) => { + child.on("error", (error) => { + clearTimeout(timeout); + reject(error); + }); + child.on("close", (code, signal) => { + clearTimeout(timeout); + const out = Buffer.concat(stdout); + const err = Buffer.concat(stderr); + resolve({ + code, + signal, + wallMs: performance.now() - started, + stdout: out, + stderr: err, + payloadBytes: out.length, + peakRssBytes: parsePeakRss(err, process.platform), + peakRssMethod: time ? `/usr/bin/time ${time.join(" ")}` : "unavailable", + }); + }); + }); +} + +function parsePresentation(result, label) { + assert.equal(result.code, 0, `${label}: presenter exited ${result.code}${result.stderr.length ? `: ${result.stderr.toString("utf8")}` : ""}`); + const text = result.stdout.toString("utf8"); + let parsed; + try { + parsed = JSON.parse(text); + } catch (error) { + throw new Error(`${label}: invalid presenter JSON: ${error.message}`, { cause: error }); + } + assert.equal(parsed.kind, "insights-presentation", `${label}: unexpected payload kind`); + return parsed; +} + +async function snapshotStore(storePath) { + try { + return JSON.parse(await readFile(storePath, "utf8")); + } catch (error) { + if (error?.code === "ENOENT") return null; + throw error; + } +} + +async function storeMetrics(storePath) { + const store = await snapshotStore(storePath); + if (!store) return null; + const rows = Object.values(store.files ?? {}).reduce( + (total, file) => total + (Array.isArray(file.rows) ? file.rows.length : 0), + 0, + ); + const text = await readFile(storePath, "utf8"); + return { + files: Object.keys(store.files ?? {}).length, + rows, + dedup: Object.keys(store.dedup ?? {}).length, + tasks: Object.keys(store.tasks ?? {}).length, + bytes: Buffer.byteLength(text), + payloadMarkerLeaked: text.includes("BENCH_PAYLOAD_MARKER"), + scanStatus: store.scan?.status ?? null, + }; +} + +async function cleanupWorker(storePath) { + const claim = await readWorkerClaim(storePath).catch(() => null); + if (!claim) return; + if (claim.host === os.hostname() && Number.isInteger(claim.pid) && claim.pid > 0) { + try { + process.kill(claim.pid, "SIGTERM"); + } catch { + // Already gone. + } + for (let index = 0; index < 50 && isProcessAlive(claim.pid); index += 1) { + await sleep(100); + } + if (isProcessAlive(claim.pid)) { + try { + process.kill(claim.pid, "SIGKILL"); + } catch { + // Already gone. + } + } + } + await clearWorkerClaim(storePath, claim.token).catch(() => {}); +} + +async function coldSample({ home, store, label }) { + await mkdir(path.dirname(store), { recursive: true }); + await rm(store, { force: true }); + const cloneLog = `${store}.clones`; + await resetLog(cloneLog); + const started = performance.now(); + const peaks = []; + const payloads = []; + const invocations = []; + let final = null; + let finalResult = null; + try { + for (let attempt = 0; attempt < 12; attempt += 1) { + const result = await runPresenter( + ["snapshot", "--store", store, "--home", home], + cloneLog, + ); + invocations.push(Math.round(result.wallMs)); + if (result.peakRssBytes) peaks.push(result.peakRssBytes); + payloads.push(result.payloadBytes); + finalResult = result; + final = parsePresentation(result, `${label} run ${attempt + 1}`); + if (final.snapshot?.scan?.status === "complete") break; + await sleep(150); + } + if (final?.snapshot?.scan?.status !== "complete") { + throw new Error(`${label}: scan did not complete (${final?.snapshot?.scan?.status ?? "no snapshot"})`); + } + await sleep(250); + return { + wallMs: performance.now() - started, + invocations, + peakRssBytes: peaks.length > 0 ? Math.max(...peaks) : null, + payloadBytes: finalResult?.payloadBytes ?? null, + maxPayloadBytes: payloads.length > 0 ? Math.max(...payloads) : null, + clones: await readCloneCounts(cloneLog), + snapshot: final.snapshot, + dashboard: final.dashboard, + store: await storeMetrics(store), + }; + } finally { + const scanStatus = final?.snapshot?.scan?.status; + if (scanStatus !== "complete") await cleanupWorker(store); + } +} + +function assertScanOracle(sample, expected, label) { + const { snapshot, dashboard } = sample; + assert.equal(snapshot.scan.status, "complete", `${label}: scan status`); + assert.equal(snapshot.scan.processed, snapshot.scan.total, `${label}: scan processed total`); + assert.ok(snapshot.scan.total >= expected.files, `${label}: discovered ${snapshot.scan.total} < ${expected.files}`); + assert.equal(snapshot.stats.files, expected.files, `${label}: stats.files`); + assert.equal(snapshot.stats.filesRead, expected.files, `${label}: cold stats.filesRead`); + assert.equal(snapshot.stats.errors, 0, `${label}: cold scan errors`); + assert.equal(dashboard?.totals?.input, expected.tokens.input, `${label}: input tokens`); + assert.equal(dashboard?.totals?.output, expected.tokens.output, `${label}: output tokens`); + assert.equal(dashboard?.totals?.cacheRead, expected.tokens.cacheRead, `${label}: cacheRead tokens`); + assert.equal(dashboard?.totals?.cacheWrite, expected.tokens.cacheWrite, `${label}: cacheWrite tokens`); + assert.equal(dashboard?.totals?.reasoning, expected.tokens.reasoning, `${label}: reasoning tokens`); + assert.equal(dashboard?.totals?.total, expected.tokens.total, `${label}: total tokens`); + assert.equal(dashboard?.totals?.messages, expected.messages, `${label}: messages`); + assert.equal(dashboard?.totals?.activeMs, expected.activeMs, `${label}: activeMs`); + assert.equal(dashboard?.totals?.longestTaskMs, expected.longestTaskMs, `${label}: longestTaskMs`); + assert.equal(dashboard?.totals?.sessions, expected.sessions, `${label}: sessions`); + assert.equal(sample.store?.tasks, expected.tasks, `${label}: tasks`); + assert.equal(sample.store?.files, expected.files, `${label}: store files`); + assert.equal(sample.clones.wholeStore, 0, `${label}: whole-store clone count`); + assert.equal(sample.clones.missing, undefined, `${label}: clone probe did not report`); +} + +function codexExpected(count) { + return { + files: count, + tokens: { + input: CODEX_FILE_ORACLE.tokens.input * count, + output: CODEX_FILE_ORACLE.tokens.output * count, + cacheRead: CODEX_FILE_ORACLE.tokens.cacheRead * count, + cacheWrite: CODEX_FILE_ORACLE.tokens.cacheWrite * count, + reasoning: CODEX_FILE_ORACLE.tokens.reasoning * count, + total: CODEX_FILE_ORACLE.tokens.total * count, + }, + messages: CODEX_FILE_ORACLE.messages * count, + tasks: CODEX_FILE_ORACLE.tasks * count, + activeMs: CODEX_FILE_ORACLE.activeMs * count, + longestTaskMs: CODEX_FILE_ORACLE.longestTaskMs, + sessions: CODEX_FILE_ORACLE.sessions * count, + }; +} + +async function generateCodexHome(home, count) { + const directory = path.join(home, ".codex/sessions/bench"); + await mkdir(directory, { recursive: true }); + const base = zlib.zstdDecompressSync(await readFile(FIXTURE)).toString("utf8"); + const lines = base.trimEnd().split("\n"); + let bytes = 0; + for (let index = 0; index < count; index += 1) { + const session = `bench-session-${index}`; + const turn1 = `bench-${index}-turn1`; + const turn2 = `bench-${index}-turn2`; + const resp1 = `bench-${index}-resp1`; + const resp2 = `bench-${index}-resp2`; + const content = `${lines.map((line) => line + .replaceAll("codex-fixture", session) + .replaceAll("turn1", turn1) + .replaceAll("turn2", turn2) + .replaceAll("resp1", resp1) + .replaceAll("resp2", resp2)).join("\n")}\n`; + const compressed = zlib.zstdCompressSync(Buffer.from(content)); + const file = path.join(directory, `bench-${String(index).padStart(5, "0")}.jsonl.zst`); + await writeFile(file, compressed); + bytes += compressed.length; + } + return { files: count, sourceBytes: bytes }; +} + +async function writeStreamLine(stream, line) { + if (!stream.write(line)) { + await new Promise((resolve) => stream.once("drain", resolve)); + } +} + +async function generatePiUsageHome(home, events) { + const directory = path.join(home, ".pi/agent/sessions/bench"); + await mkdir(directory, { recursive: true }); + const file = path.join(directory, "usage-250k.jsonl"); + const stream = createWriteStream(file, { encoding: "utf8" }); + const header = { type: "session", version: 3, id: "bench-usage", timestamp: "2026-03-01T00:00:00Z", cwd: "/bench/usage" }; + await writeStreamLine(stream, `${JSON.stringify(header)}\n`); + for (let index = 0; index < events; index += 1) { + const record = { + type: "message", + id: `usage-${index}`, + timestamp: "2026-03-01T00:00:00Z", + message: { + role: "assistant", + model: "bench-model", + provider: "bench-provider", + usage: { input: 1, output: 1, cacheRead: 0, cacheWrite: 0, reasoning: 0, totalTokens: 2 }, + stopReason: "toolUse", + content: [], + }, + }; + await writeStreamLine(stream, `${JSON.stringify(record)}\n`); + } + stream.end(); + await new Promise((resolve, reject) => { + stream.on("finish", resolve); + stream.on("error", reject); + }); + return { files: 1, sourceBytes: (await stat(file)).size }; +} + +async function generateGiantSource(work, size) { + const directory = path.join(work, "giant-src"); + await mkdir(directory, { recursive: true }); + const file = path.join(directory, `giant-${size}.jsonl`); + const stream = createWriteStream(file, { encoding: "utf8" }); + const header = { type: "session_meta", timestamp: "2026-03-02T00:00:00Z", payload: { id: "bench-giant", cwd: "/bench/giant", model_provider: "bench-provider" } }; + await writeStreamLine(stream, `${JSON.stringify(header)}\n`); + const context = { type: "turn_context", timestamp: "2026-03-02T00:00:00Z", payload: { turn_id: "giant-turn", model: "bench-model", cwd: "/bench/giant" } }; + await writeStreamLine(stream, `${JSON.stringify(context)}\n`); + for (let index = 0; index < 1_000; index += 1) { + const record = { + type: "token_usage_record", + timestamp: "2026-03-02T00:00:00Z", + payload: { + response_id: `giant-${index}`, + turn_id: "giant-turn", + usage: { input_tokens: 3, cached_input_tokens: 1, cache_write_input_tokens: 0, output_tokens: 2, reasoning_output_tokens: 0, total_tokens: 5 }, + }, + }; + await writeStreamLine(stream, `${JSON.stringify(record)}\n`); + } + const block = Buffer.alloc(192 * 1024); + let state = 0x9e3779b9; + for (let index = 0; index < block.length; index += 1) { + state ^= state << 13; + state ^= state >>> 17; + state ^= state << 5; + block[index] = state & 0xff; + } + const filler = `${JSON.stringify({ type: "bench_payload", marker: "BENCH_PAYLOAD_MARKER", payload: block.toString("base64") })}\n`; + const lineBytes = Buffer.byteLength(filler); + const count = Math.ceil(size / lineBytes); + for (let index = 0; index < count; index += 1) { + await writeStreamLine(stream, filler); + } + stream.end(); + await new Promise((resolve, reject) => { + stream.on("finish", resolve); + stream.on("error", reject); + }); + return { file, bytes: (await stat(file)).size }; +} + +async function giantVariant(work, source, size, format) { + const home = path.join(work, `giant-${size}-${format}`, "home"); + const directory = path.join(home, ".codex/sessions/bench"); + await mkdir(directory, { recursive: true }); + const file = path.join(directory, format === "zstd" ? "giant.jsonl.zst" : "giant.jsonl"); + if (format === "zstd") { + await pipeline( + createReadStream(source), + zlib.createZstdCompress(), + createWriteStream(file), + ); + } else { + try { + await link(source, file); + } catch { + await copyFile(source, file); + } + } + return { home, file, sourceBytes: (await stat(file)).size }; +} + +async function measureStatusWhileIndexing(work) { + const home = path.join(work, "status", "home"); + const store = path.join(work, "status", "data", "insights.json"); + const directory = path.join(home, ".codex/sessions/bench"); + await mkdir(directory, { recursive: true }); + await mkdir(path.dirname(store), { recursive: true }); + const base = zlib.zstdDecompressSync(await readFile(FIXTURE)).toString("utf8"); + const filler = `${JSON.stringify({ type: "synthetic_noise", payload: "s".repeat(1024) })}\n`; + const content = `${base}${filler.repeat(Math.ceil((8 * MiB) / filler.length))}`; + await writeFile(path.join(directory, "slow.jsonl.zst"), zlib.zstdCompressSync(Buffer.from(content))); + try { + const launched = await collectSnapshot("snapshot", { + store, + home, + budgetMs: 1, + workerAckMs: 5_000, + testStreamDelayMs: 100, + }); + const workerPid = launched.scan?.workerPid ?? null; + if (launched.scan?.status !== "running" && launched.scan?.status !== "complete") { + throw new Error(`status bench: expected a promoted worker, got ${launched.scan?.status}`); + } + for (let index = 0; index < 20; index += 1) { + const storeState = await snapshotStore(store); + if (storeState?.scan?.status === "running" || storeState?.scan?.status === "partial") break; + await sleep(100); + } + const cloneLog = path.join(work, "status", "clones.log"); + const times = []; + const statuses = []; + const payloads = []; + for (let index = 0; index < STATUS_SAMPLES; index += 1) { + const result = await runPresenter(["status", "--store", store], cloneLog); + const parsed = parsePresentation(result, `status ${index + 1}`); + times.push(result.wallMs); + payloads.push(result.payloadBytes); + statuses.push(parsed.snapshot?.scan?.status ?? null); + } + for (let index = 0; index < 300; index += 1) { + const storeState = await snapshotStore(store); + if (storeState?.scan?.status === "complete") break; + await sleep(100); + } + const finalStore = await snapshotStore(store); + return { + label: "ARTIFICIAL stream throttling; cached status while one detached indexer runs", + workerPid, + timesMs: times, + medianMs: median(times), + payloadBytes: Math.max(...payloads), + statuses, + finalScanStatus: finalStore?.scan?.status ?? null, + complete: finalStore?.scan?.status === "complete", + clones: await readCloneCounts(cloneLog), + }; + } finally { + await cleanupWorker(store); + } +} + +function buildGates(results) { + const gates = []; + const gate = (id, target, actual, pass, note) => gates.push({ id, target, actual, pass, ...(note ? { note } : {}) }); + const cold10k = results.cold.find((entry) => entry.files === 10_008); + gate("cold10kMedianMs", "<=8000 (target <=5000)", cold10k.medianMs, cold10k.medianMs <= 8_000, + cold10k.medianMs <= 5_000 ? "target met" : "hard limit met, target missed"); + gate("warm10kMedianMs", "<=1500 (target <=1000)", results.warm.medianMs, results.warm.medianMs <= 1_500, + results.warm.medianMs <= 1_000 ? "target met" : "hard limit met, target missed"); + gate("warm10kFilesRead", "0", results.warm.samples[0].snapshot.stats.filesRead, + results.warm.samples.every((sample) => sample.snapshot.stats.filesRead === 0)); + gate("warm10kBytesRead", "0", results.warm.samples[0].snapshot.stats.bytesRead, + results.warm.samples.every((sample) => sample.snapshot.stats.bytesRead === 0)); + gate("warm10kStoreRewrite", "none (mtimeNs + sha256 unchanged)", results.warm.storeUnchanged, + results.warm.storeUnchanged); + gate("coldGrowth1kTo10k", "<=15x", results.growth, results.growth <= 15); + gate("usage250kMedianMs", "<=5000", results.usage.medianMs, results.usage.medianMs <= 5_000); + gate("usage250kExactTotals", "exact", results.usage.exact, results.usage.exact); + gate("giantPlainRssDeltaMiB", "<=64", results.giant.plain.deltaMiB, results.giant.plain.deltaMiB <= 64); + gate("giantZstdRssDeltaMiB", "<=64", results.giant.zstd.deltaMiB, results.giant.zstd.deltaMiB <= 64); + gate("presentationPayloadMaxKiB", "<512", results.payloadMaxBytes / 1024, results.payloadMaxBytes < 512 * 1024); + gate("wholeStoreClones", "0", results.wholeStoreClones, results.wholeStoreClones === 0); + gate("cachedStatusMedianMs", "<=300 (hard <=1000)", results.status.medianMs, results.status.medianMs <= 300); + gate("cachedStatusCompletes", "complete", results.status.finalScanStatus, results.status.complete); + gate("node22FullSuite", "PENDING parent-owned blocker", "not evaluated by this benchmark", null, + "Node 22.23.4 and 22.15.0 full suites fail the expired-budget zstd budget test; the committed cache identity blocker is also deferred to B6. This benchmark does not certify Node 22."); + return gates; +} + +async function main() { + const work = await mkdtemp(path.join(os.tmpdir(), "insights-bench-")); + const machine = { + platform: process.platform, + release: os.release(), + arch: process.arch, + cpu: os.cpus()[0]?.model ?? "unknown", + cores: os.cpus().length, + totalMemBytes: os.totalmem(), + node: process.version, + loadAvg: os.loadavg(), + }; + const startedAt = new Date().toISOString(); + const results = { + schemaVersion: 1, + label: "Synthetic, opt-in Insights performance/RSS benchmark. Not native Herdr proof.", + startedAt, + machine, + work, + commands: { + benchmark: "node scripts/herdr-insights/benchmark.mjs", + coldAndWarm: "node herdr-insights/mods/insights/presenter.mjs snapshot --store --home ", + status: "node herdr-insights/mods/insights/presenter.mjs status --store ", + peakRss: process.platform === "darwin" ? "/usr/bin/time -l ..." : "/usr/bin/time -v ...", + cloneProbe: "NODE_OPTIONS=--import scripts/herdr-insights/clone-probe.mjs (benchmark sets INSIGHTS_CLONE_LOG)", + }, + generation: {}, + cold: [], + warm: {}, + usage: {}, + giant: {}, + status: {}, + payloadMaxBytes: 0, + wholeStoreClones: 0, + }; + try { + console.log(`work=${work}`); + console.log(`node=${process.version} platform=${process.platform} cpu=${machine.cpu}`); + + const codexHomes = {}; + for (const count of COLD_CODEX_COUNTS) { + const home = path.join(work, `codex-${count}`, "home"); + const generationStarted = performance.now(); + const generated = await generateCodexHome(home, count); + results.generation[`codex${count}`] = { + ...generated, + ms: performance.now() - generationStarted, + }; + codexHomes[count] = home; + console.log(`generated ${count} unique codex archives in ${Math.round(results.generation[`codex${count}`].ms)}ms`); + } + + for (const count of COLD_CODEX_COUNTS) { + const home = codexHomes[count]; + const samples = []; + for (let index = 0; index < SAMPLES; index += 1) { + const store = path.join(work, `cold-${count}`, `sample-${index}`, "insights.json"); + const sample = await coldSample({ home, store, label: `cold-${count}-${index + 1}` }); + assertScanOracle(sample, codexExpected(count), `cold-${count}-${index + 1}`); + sample.wallMs = Math.round(sample.wallMs); + samples.push(sample); + results.payloadMaxBytes = Math.max(results.payloadMaxBytes, sample.maxPayloadBytes ?? 0); + results.wholeStoreClones += sample.clones.wholeStore; + console.log(`cold ${count} #${index + 1}: ${sample.wallMs}ms peak=${sample.peakRssBytes} filesRead=${sample.snapshot.stats.filesRead} payload=${sample.payloadBytes}B`); + } + results.cold.push({ + files: count, + sourceBytes: results.generation[`codex${count}`].sourceBytes, + samples, + medianMs: median(samples.map((sample) => sample.wallMs)), + peakRssBytes: median(samples.map((sample) => sample.peakRssBytes)), + storeBytes: samples[0].store.bytes, + payloadBytes: samples[0].payloadBytes, + filesRead: samples[0].snapshot.stats.filesRead, + bytesRead: samples[0].snapshot.stats.bytesRead, + clones: samples.reduce((total, sample) => total + sample.clones.calls, 0), + wholeStoreClones: samples.reduce((total, sample) => total + sample.clones.wholeStore, 0), + }); + } + + const warmHome = codexHomes[10_008]; + const warmStore = path.join(work, "cold-10008", "sample-0", "insights.json"); + await mkdir(path.dirname(warmStore), { recursive: true }); + const beforeStat = await stat(warmStore, { bigint: true }); + assert.ok(beforeStat, "warm baseline store must exist after a completed cold scan"); + const beforeHash = await fileSha256(warmStore); + let afterMtimeNs = beforeStat.mtimeNs; + let afterHash = beforeHash; + const warmSamples = []; + for (let index = 0; index < SAMPLES; index += 1) { + const cloneLog = path.join(path.dirname(warmStore), `warm-clones-${index}.log`); + const result = await runPresenter(["snapshot", "--store", warmStore, "--home", warmHome], cloneLog); + const parsed = parsePresentation(result, `warm-${index + 1}`); + const sample = { + wallMs: result.wallMs, + peakRssBytes: result.peakRssBytes, + payloadBytes: result.payloadBytes, + snapshot: parsed.snapshot, + dashboard: parsed.dashboard, + clones: await readCloneCounts(cloneLog), + }; + warmSamples.push(sample); + results.payloadMaxBytes = Math.max(results.payloadMaxBytes, result.payloadBytes); + results.wholeStoreClones += sample.clones.wholeStore; + assert.equal(sample.snapshot.scan.status, "complete", `warm-${index + 1}: scan status`); + assert.equal(sample.snapshot.stats.filesRead, 0, `warm-${index + 1}: filesRead`); + assert.equal(sample.snapshot.stats.bytesRead, 0, `warm-${index + 1}: bytesRead`); + assert.equal(sample.clones.wholeStore, 0, `warm-${index + 1}: whole-store clone`); + const afterStat = await stat(warmStore, { bigint: true }); + afterMtimeNs = afterStat.mtimeNs; + afterHash = await fileSha256(warmStore); + assert.equal(afterStat.mtimeNs, beforeStat.mtimeNs, `warm-${index + 1}: store mtime changed`); + assert.equal(afterHash, beforeHash, `warm-${index + 1}: store content changed`); + console.log(`warm 10008 #${index + 1}: ${Math.round(result.wallMs)}ms peak=${result.peakRssBytes} payload=${result.payloadBytes}B`); + } + results.warm = { + files: 10_008, + samples: warmSamples, + medianMs: median(warmSamples.map((sample) => sample.wallMs)), + peakRssBytes: median(warmSamples.map((sample) => sample.peakRssBytes)), + storeUnchanged: true, + storeMtimeNsBefore: beforeStat.mtimeNs.toString(), + storeMtimeNsAfter: afterMtimeNs.toString(), + storeSha256: beforeHash, + storeSha256After: afterHash, + storeBytes: (await stat(warmStore)).size, + filesRead: 0, + bytesRead: 0, + clones: warmSamples.reduce((total, sample) => total + sample.clones.calls, 0), + wholeStoreClones: warmSamples.reduce((total, sample) => total + sample.clones.wholeStore, 0), + }; + const growth = results.cold.find((entry) => entry.files === 10_008).medianMs + / results.cold.find((entry) => entry.files === 1_000).medianMs; + results.growth = growth; + console.log(`growth 1k->10k: ${growth.toFixed(2)}x`); + + const usageHome = path.join(work, "usage-250k", "home"); + const usageGenerated = performance.now(); + const usageFixture = await generatePiUsageHome(usageHome, USAGE_EVENTS); + results.generation.usage250k = { + ...usageFixture, + ms: performance.now() - usageGenerated, + }; + const usageSamples = []; + for (let index = 0; index < SAMPLES; index += 1) { + const store = path.join(work, "usage-250k", `sample-${index}`, "insights.json"); + const sample = await coldSample({ home: usageHome, store, label: `usage-250k-${index + 1}` }); + const expected = { + files: 1, + tokens: { input: 250_000, output: 250_000, cacheRead: 0, cacheWrite: 0, reasoning: 0, total: 500_000 }, + messages: 250_000, + tasks: 0, + activeMs: 0, + longestTaskMs: 0, + sessions: 1, + }; + assertScanOracle(sample, expected, `usage-250k-${index + 1}`); + sample.wallMs = Math.round(sample.wallMs); + usageSamples.push(sample); + results.payloadMaxBytes = Math.max(results.payloadMaxBytes, sample.maxPayloadBytes ?? 0); + results.wholeStoreClones += sample.clones.wholeStore; + console.log(`usage 250k #${index + 1}: ${sample.wallMs}ms peak=${sample.peakRssBytes} filesRead=${sample.snapshot.stats.filesRead} bytesRead=${sample.snapshot.stats.bytesRead}`); + } + const usageCloneLog = path.join(work, "usage-warm", "clones.log"); + await mkdir(path.dirname(usageCloneLog), { recursive: true }); + const usageWarm = await runPresenter( + ["snapshot", "--store", path.join(work, "usage-250k", "sample-0", "insights.json"), "--home", usageHome], + usageCloneLog, + ); + const usageWarmParsed = parsePresentation(usageWarm, "usage-warm"); + assert.equal(usageWarmParsed.snapshot.stats.filesRead, 0, "usage warm: filesRead"); + assert.equal(usageWarmParsed.snapshot.stats.bytesRead, 0, "usage warm: bytesRead"); + results.usage = { + events: USAGE_EVENTS, + sourceBytes: usageFixture.sourceBytes, + samples: usageSamples, + medianMs: median(usageSamples.map((sample) => sample.wallMs)), + peakRssBytes: median(usageSamples.map((sample) => sample.peakRssBytes)), + storeBytes: usageSamples[0].store.bytes, + payloadBytes: usageSamples[0].payloadBytes, + filesRead: usageSamples[0].snapshot.stats.filesRead, + bytesRead: usageSamples[0].snapshot.stats.bytesRead, + exact: true, + warmFilesRead: usageWarmParsed.snapshot.stats.filesRead, + warmBytesRead: usageWarmParsed.snapshot.stats.bytesRead, + }; + + results.giant = { formats: {} }; + for (const size of GIANT_SIZES) { + const giantStarted = performance.now(); + results.generation[`giantSource${size}`] = { + ...(await generateGiantSource(work, size)), + ms: performance.now() - giantStarted, + }; + } + for (const format of ["plain", "zstd"]) { + const bySize = []; + for (const size of GIANT_SIZES) { + const variant = await giantVariant(work, results.generation[`giantSource${size}`].file, size, format); + const samples = []; + for (let index = 0; index < SAMPLES; index += 1) { + const store = path.join(work, `giant-${size}-${format}`, `sample-${index}`, "insights.json"); + const sample = await coldSample({ home: variant.home, store, label: `giant-${size}-${format}-${index + 1}` }); + const expected = { + files: 1, + tokens: { input: 2_000, output: 2_000, cacheRead: 1_000, cacheWrite: 0, reasoning: 0, total: 5_000 }, + messages: 1_000, + tasks: 0, + activeMs: 0, + longestTaskMs: 0, + sessions: 1, + }; + assertScanOracle(sample, expected, `giant-${size}-${format}-${index + 1}`); + assert.equal(sample.store.payloadMarkerLeaked, false, `giant-${size}-${format}: raw payload leaked into store`); + sample.wallMs = Math.round(sample.wallMs); + samples.push(sample); + results.payloadMaxBytes = Math.max(results.payloadMaxBytes, sample.maxPayloadBytes ?? 0); + results.wholeStoreClones += sample.clones.wholeStore; + console.log(`giant ${format} ${Math.round(size / MiB)}MiB #${index + 1}: ${sample.wallMs}ms peak=${sample.peakRssBytes} bytesRead=${sample.snapshot.stats.bytesRead}`); + } + bySize.push({ + sizeMiB: size / MiB, + sourceBytes: variant.sourceBytes, + rawSourceBytes: results.generation[`giantSource${size}`].bytes, + samples, + medianMs: median(samples.map((sample) => sample.wallMs)), + peakRssBytes: median(samples.map((sample) => sample.peakRssBytes)), + storeBytes: samples[0].store.bytes, + payloadBytes: samples[0].payloadBytes, + filesRead: samples[0].snapshot.stats.filesRead, + bytesRead: samples[0].snapshot.stats.bytesRead, + decompressedBytes: samples[0].snapshot.stats.decompressedBytes, + clones: samples.reduce((total, sample) => total + sample.clones.calls, 0), + wholeStoreClones: samples.reduce((total, sample) => total + sample.clones.wholeStore, 0), + }); + } + results.giant.formats[format] = { + sizes: bySize, + deltaMiB: (bySize[1].peakRssBytes - bySize[0].peakRssBytes) / MiB, + }; + } + results.giant.plain = results.giant.formats.plain; + results.giant.zstd = results.giant.formats.zstd; + + results.status = await measureStatusWhileIndexing(work); + results.payloadMaxBytes = Math.max(results.payloadMaxBytes, results.status.payloadBytes); + results.wholeStoreClones += results.status.clones.wholeStore; + console.log(`status while indexing median: ${results.status.medianMs.toFixed(1)}ms (${results.status.statuses.join(",")})`); + + results.gates = buildGates(results); + results.finishedAt = new Date().toISOString(); + results.workKept = keep; + const failed = results.gates.filter((gate) => gate.pass === false); + const pending = results.gates.filter((gate) => gate.pass === null); + results.ok = failed.length === 0; + results.pending = pending.map((gate) => gate.id); + await mkdir(path.dirname(outputPath), { recursive: true }); + await writeFile(outputPath, `${JSON.stringify(results, null, 2)}\n`); + console.log(`\nJSON artifact: ${outputPath}`); + for (const gate of results.gates) { + const state = gate.pass === null ? "PENDING" : gate.pass ? "PASS" : "FAIL"; + console.log(`${state} ${gate.id}: ${gate.actual} (${gate.target})${gate.note ? ` — ${gate.note}` : ""}`); + } + if (failed.length > 0) process.exitCode = 1; + } finally { + if (!keep) { + await rm(work, { recursive: true, force: true, maxRetries: 3 }); + } else { + console.log(`kept work directory: ${work}`); + } + } +} + +main().catch((error) => { + console.error(error?.stack ?? String(error)); + process.exitCode = 1; +}); diff --git a/scripts/herdr-insights/clone-probe.mjs b/scripts/herdr-insights/clone-probe.mjs new file mode 100644 index 00000000..d646254b --- /dev/null +++ b/scripts/herdr-insights/clone-probe.mjs @@ -0,0 +1,32 @@ +// Benchmark-only instrumentation. Wraps structuredClone without changing its +// result so the benchmark can prove the collector never clones the accumulated +// store. Loaded only when INSIGHTS_CLONE_LOG is set by the benchmark runner. +import { appendFileSync } from "node:fs"; + +const target = process.env.INSIGHTS_CLONE_LOG; +if (target) { + const original = globalThis.structuredClone; + let total = 0; + let wholeStore = 0; + globalThis.structuredClone = function countedStructuredClone(value, options) { + total += 1; + if ( + value && + typeof value === "object" && + value.files && value.dedup && value.tasks && value.config && value.scan + ) { + wholeStore += 1; + } + return original.call(this, value, options); + }; + process.on("exit", () => { + try { + appendFileSync( + target, + `${JSON.stringify({ pid: process.pid, total, wholeStore })}\n`, + ); + } catch { + // A benchmark child must never fail because its probe log is gone. + } + }); +} From 48ba9bc4f304f31e4ceb6453adab1102ebba7ae6 Mon Sep 17 00:00:00 2001 From: GaganSD Date: Mon, 2 Nov 2026 20:12:18 +0530 Subject: [PATCH 09/15] fix(insights): preserve deliberate zstd budget stops Node 22 pipeline can wrap the destination ScanStopError as AbortError, which was misclassified as archive corruption and permanently fingerprinted a valid zstd file. Capture the exact stop at the destination and keep genuine stream failures distinct. Also validate zstd frame structure because Node 22 can silently accept truncated trailing frames, making existing corruption rollback tests runtime-dependent. --- herdr-insights/mods/insights/core/scan.mjs | 150 +++++++++++++++++- .../herdr-insights/collector-bounded.test.mjs | 16 ++ 2 files changed, 161 insertions(+), 5 deletions(-) diff --git a/herdr-insights/mods/insights/core/scan.mjs b/herdr-insights/mods/insights/core/scan.mjs index fe0d8bd7..42f2bfda 100644 --- a/herdr-insights/mods/insights/core/scan.mjs +++ b/herdr-insights/mods/insights/core/scan.mjs @@ -2,6 +2,7 @@ import { createHash } from "node:crypto"; import { createReadStream } from "node:fs"; import { open, readdir, realpath, stat } from "node:fs/promises"; import path from "node:path"; +import { Transform } from "node:stream"; import { pipeline } from "node:stream/promises"; import * as zlib from "node:zlib"; import { parseAdapterRecord } from "./adapters.mjs"; @@ -343,6 +344,127 @@ async function streamRange(filePath, start, end, onChunk, { shouldStop } = {}) { return { bytes, stopped, position }; } +const ZSTD_FRAME_MAGIC = 0xfd2fb528; +const ZSTD_SKIPPABLE_MASK = 0xfffffff0; +const ZSTD_SKIPPABLE_MAGIC = 0x184d2a50; +const ZSTD_DICT_ID_BYTES = [0, 1, 2, 4]; +const ZSTD_FCS_BYTES = [0, 2, 4, 8]; + +/** + * Structural zstd frame validator. Some Node 22 runtimes silently accept a + * complete frame followed by a truncated extra frame or arbitrary trailing + * bytes. The decoder still streams the payload; this tap only walks frame, + * header, block and checksum lengths so corruption is detected consistently + * instead of depending on runtime-specific end-of-stream behaviour. + */ +function createZstdFrameValidator() { + let buffer = Buffer.alloc(0); + let state = "magic"; + let frames = 0; + let remaining = 0; + let lastBlock = false; + let checksum = false; + + const fail = (message) => { + throw new Error(`invalid zstd stream: ${message}`); + }; + + const parse = () => { + for (;;) { + if (state === "magic") { + if (buffer.length < 4) return; + const magic = buffer.readUInt32LE(0); + buffer = buffer.subarray(4); + if (magic === ZSTD_FRAME_MAGIC) { + frames += 1; + state = "frameHeader"; + } else if ((magic & ZSTD_SKIPPABLE_MASK) === ZSTD_SKIPPABLE_MAGIC) { + frames += 1; + state = "skippableSize"; + } else { + fail("unknown frame magic"); + } + } else if (state === "skippableSize") { + if (buffer.length < 4) return; + remaining = buffer.readUInt32LE(0); + buffer = buffer.subarray(4); + state = "skip"; + } else if (state === "skip") { + if (remaining > 0) { + if (buffer.length === 0) return; + const take = Math.min(remaining, buffer.length); + buffer = buffer.subarray(take); + remaining -= take; + continue; + } + state = "magic"; + } else if (state === "frameHeader") { + if (buffer.length < 1) return; + const descriptor = buffer[0]; + if (descriptor & 0x18) fail("reserved frame header bits"); + const fcsFlag = descriptor >> 6; + const singleSegment = (descriptor >> 5) & 1; + checksum = Boolean((descriptor >> 2) & 1); + const dictBytes = ZSTD_DICT_ID_BYTES[descriptor & 0x03]; + const fcsBytes = fcsFlag === 0 && singleSegment ? 1 : ZSTD_FCS_BYTES[fcsFlag]; + const headerBytes = 1 + (singleSegment ? 0 : 1) + dictBytes + fcsBytes; + if (buffer.length < headerBytes) return; + buffer = buffer.subarray(headerBytes); + state = "blockHeader"; + } else if (state === "blockHeader") { + if (buffer.length < 3) return; + const header = buffer.readUIntLE(0, 3); + buffer = buffer.subarray(3); + lastBlock = Boolean(header & 1); + const type = (header >> 1) & 0x03; + if (type === 3) fail("reserved block type"); + remaining = type === 1 ? 1 : header >> 3; + state = "blockData"; + } else if (state === "blockData") { + if (remaining > 0) { + if (buffer.length === 0) return; + const take = Math.min(remaining, buffer.length); + buffer = buffer.subarray(take); + remaining -= take; + continue; + } + if (lastBlock) { + state = checksum ? "checksum" : "magic"; + } else { + state = "blockHeader"; + } + } else if (state === "checksum") { + if (buffer.length < 4) return; + buffer = buffer.subarray(4); + state = "magic"; + } + } + }; + + return { + push(chunk) { + buffer = buffer.length ? Buffer.concat([buffer, chunk]) : chunk; + parse(); + }, + get complete() { + return frames > 0 && state === "magic" && buffer.length === 0; + }, + }; +} + +function zstdValidationTap(validator) { + return new Transform({ + transform(chunk, _encoding, callback) { + try { + validator.push(chunk); + callback(null, chunk); + } catch (error) { + callback(error); + } + }, + }); +} + function hasZstdSupport() { return typeof zlib.createZstdDecompress === "function"; } @@ -556,19 +678,37 @@ async function scanCompressedFile( // The caller owns a per-file transaction. Any decompression failure or // deadline stop discards this open archive without exposing its rows. Raw // transcript bytes are never retained: the line consumer is bounded. + // Capture the deliberate stop at the destination because some Node 22 + // pipeline implementations re-wrap a destination error as a generic + // AbortError. Only this exact captured ScanStopError is a budget stop; + // every other stream failure remains a corruption/IO failure. + let deliberateStop = null; + const validator = createZstdFrameValidator(); let result; try { if (entry.offset || entry.rows.length || Object.keys(entry.state).length) resetFileEntry(store, entry); result = await pipeline( createReadStream(filePath), + zstdValidationTap(validator), zlib.createZstdDecompress(), - async (chunks) => processStringChunks( - store, entry, source, chunks, 0, undefined, sink, options, - ), + async (chunks) => { + try { + return await processStringChunks( + store, entry, source, chunks, 0, undefined, sink, options, + ); + } catch (error) { + if (error instanceof ScanStopError) deliberateStop = error; + throw error; + } + }, ); + if (!validator.complete) { + throw new Error("invalid zstd stream: truncated or trailing data"); + } } catch (error) { - if (error instanceof ScanStopError) { - return { status: "stopped", reason: error.reason, dirty: false }; + const stop = deliberateStop ?? (error instanceof ScanStopError ? error : null); + if (stop) { + return { status: "stopped", reason: stop.reason, dirty: false }; } return { status: "corrupt", error, dirty: false }; } diff --git a/tests/herdr-insights/collector-bounded.test.mjs b/tests/herdr-insights/collector-bounded.test.mjs index cd5259a4..b28d110c 100644 --- a/tests/herdr-insights/collector-bounded.test.mjs +++ b/tests/herdr-insights/collector-bounded.test.mjs @@ -306,4 +306,20 @@ test("a bounded scan never exposes rows from an unfinished archive", async () => assert.equal(result.abortedFile?.path?.endsWith("big.jsonl.zst"), true); assert.equal(Object.keys(store.files).length, 0, "an aborted archive contributes no file entry"); assert.deepEqual(snapshotRows(store), []); + + // A deliberate budget stop is not corruption: retrying without a budget-- + // or the promoted worker--must index the same valid archive normally. + const retry = await scanSources(store, { + home, + deadline: Date.now() + 60_000, + now: Date.now, + }); + assert.equal(retry.complete, true); + assert.equal(retry.abortedFile, null); + assert.equal(retry.stats.errors, 0); + assert.equal(Object.keys(store.files).length, 1); + const [entry] = Object.values(store.files); + assert.equal(entry.state.zstdError, undefined, "a budget stop leaves no corrupt fingerprint"); + assert.ok(entry.rows.length > 0, "the valid archive is indexed on retry"); + assert.equal(snapshotRows(store).length > 0, true); }); From aec93e421ad4035d5db4e33867b88e3e570dc1bb Mon Sep 17 00:00:00 2001 From: GaganSD Date: Mon, 2 Nov 2026 20:12:18 +0530 Subject: [PATCH 10/15] fix(insights): bind cache to committed store revision Replace the 2ms mtime tolerance with an exact dev/ino/size/mtimeNs fingerprint captured after the atomic store rename while writer authority is held. Validate the current store before and after reading the cache, so a crash before the cache update cannot serve old rows for a newer commit. Legacy un-fingerprinted caches fall back to the store. --- herdr-insights/mods/insights/core/cache.mjs | 66 +++++++++--- herdr-insights/mods/insights/core/store.mjs | 11 +- .../collector-cache-identity.test.mjs | 102 ++++++++++++++++++ 3 files changed, 162 insertions(+), 17 deletions(-) create mode 100644 tests/herdr-insights/collector-cache-identity.test.mjs diff --git a/herdr-insights/mods/insights/core/cache.mjs b/herdr-insights/mods/insights/core/cache.mjs index ba606a76..25f97f88 100644 --- a/herdr-insights/mods/insights/core/cache.mjs +++ b/herdr-insights/mods/insights/core/cache.mjs @@ -5,14 +5,47 @@ import { atomicWriteJson, isPlainObject } from "./util.mjs"; // writer lock. This sidecar is a compact, metadata-only copy of the committed // aggregate rows plus the truthful scan state. It is always derived from the // committed store and is never read as a source for mutation. -const CACHE_VERSION = 1; -const STALE_MTIME_MS = 2; +const CACHE_VERSION = 2; +const FINGERPRINT_FIELDS = ["dev", "ino", "size", "mtimeNs"]; export function presentationCachePath(storePath) { return `${storePath}.cache.json`; } -export async function writePresentationCache(storePath, { savedAt, store, rows }) { +/** + * Exact identity of one committed atomic store revision. The publisher calls + * this only after its store rename while still holding the writer lock, so a + * cache can never inherit a newer revision's fingerprint by racing a commit. + */ +export async function storeRevisionFingerprint(storePath) { + const info = await stat(storePath, { bigint: true }); + if (!info.isFile()) return null; + return { + dev: String(info.dev), + ino: String(info.ino), + size: String(info.size), + mtimeNs: String(info.mtimeNs), + }; +} + +function validFingerprint(value) { + return isPlainObject(value) + && FINGERPRINT_FIELDS.every((field) => typeof value[field] === "string" && value[field]); +} + +function sameFingerprint(left, right) { + return validFingerprint(left) + && validFingerprint(right) + && FINGERPRINT_FIELDS.every((field) => left[field] === right[field]); +} + +export async function writePresentationCache( + storePath, + { savedAt, store, rows, fingerprint }, +) { + if (!validFingerprint(fingerprint)) { + throw new Error("presentation cache requires the committed store fingerprint"); + } const scan = isPlainObject(store?.scan) ? store.scan : {}; const value = { version: CACHE_VERSION, @@ -23,6 +56,7 @@ export async function writePresentationCache(storePath, { savedAt, store, rows } config: store?.config ?? {}, rows: Array.isArray(rows) ? rows : [], scan, + storeFingerprint: fingerprint, }; await atomicWriteJson(presentationCachePath(storePath), value); return value; @@ -38,25 +72,29 @@ function validCache(value) { && (value.lastScanAt === null || typeof value.lastScanAt === "string") && (value.lastCompletedScanAt === null || typeof value.lastCompletedScanAt === "string") && Number.isInteger(value.generation) - && value.generation >= 0, + && value.generation >= 0 + && validFingerprint(value.storeFingerprint), ); } /** - * Returns the cached presentation data only when it is at least as new as the - * committed store file. A missing, corrupt, or older cache falls back to the - * full store so prepare never invents data. + * Returns the cached presentation data only when it was derived from the exact + * committed store revision that is visible now. A missing, legacy (un- + * fingerprinted), corrupt, mismatched, or concurrently replaced cache falls + * back to the full store so prepare never invents or serves old data. */ export async function readPresentationCache(storePath) { try { - const [storeStat, cacheStat] = await Promise.all([ - stat(storePath), - stat(presentationCachePath(storePath)), - ]); - if (!storeStat.isFile() || !cacheStat.isFile()) return null; - if (cacheStat.mtimeMs + STALE_MTIME_MS < storeStat.mtimeMs) return null; - const value = JSON.parse(await readFile(presentationCachePath(storePath), "utf8")); + const before = await storeRevisionFingerprint(storePath); + if (!before) return null; + const text = await readFile(presentationCachePath(storePath), "utf8"); + const after = await storeRevisionFingerprint(storePath); + // A commit that landed while the cache was being read makes the cache + // ambiguous; use the committed store instead of guessing. + if (!sameFingerprint(before, after)) return null; + const value = JSON.parse(text); if (!validCache(value)) return null; + if (!sameFingerprint(value.storeFingerprint, after)) return null; return value; } catch { return null; diff --git a/herdr-insights/mods/insights/core/store.mjs b/herdr-insights/mods/insights/core/store.mjs index 9f9f585c..26a6ce49 100644 --- a/herdr-insights/mods/insights/core/store.mjs +++ b/herdr-insights/mods/insights/core/store.mjs @@ -1,5 +1,5 @@ import { readFile, stat, unlink } from "node:fs/promises"; -import { writePresentationCache } from "./cache.mjs"; +import { storeRevisionFingerprint, writePresentationCache } from "./cache.mjs"; import { defaultConfig, migrateLoadedConfig, normalizeConfig } from "./config.mjs"; import { STORE_VERSION, @@ -976,10 +976,15 @@ export async function saveStore( store.lastCompletedScanAt = savedAt; } await atomicWriteJson(storePath, store); - // The presentation cache is derived from this exact committed store. A + // The presentation cache is derived from this exact committed store. Capture + // the post-rename identity while the caller still holds writer authority so + // an old cache can never be accepted for a newer atomic store revision. A // failure here must not fail the commit; prepare falls back to the store. try { - await writePresentationCache(storePath, { savedAt, store, rows: snapshotRows(store) }); + const fingerprint = await storeRevisionFingerprint(storePath); + await writePresentationCache(storePath, { + savedAt, store, rows: snapshotRows(store), fingerprint, + }); } catch { /* the cache is an optional read accelerator */ } return savedAt; } diff --git a/tests/herdr-insights/collector-cache-identity.test.mjs b/tests/herdr-insights/collector-cache-identity.test.mjs new file mode 100644 index 00000000..4cb92570 --- /dev/null +++ b/tests/herdr-insights/collector-cache-identity.test.mjs @@ -0,0 +1,102 @@ +import assert from "node:assert/strict"; +import { mkdir, mkdtemp, readFile, rm, stat, symlink, utimes } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import path from "node:path"; +import test from "node:test"; + +import { readCommittedSnapshot } from "../../herdr-insights/mods/insights/collector.mjs"; +import { + presentationCachePath, + readPresentationCache, + storeRevisionFingerprint, +} from "../../herdr-insights/mods/insights/core/cache.mjs"; +import { normalizeConfig } from "../../herdr-insights/mods/insights/core/config.mjs"; +import { emptyStore, saveStore } from "../../herdr-insights/mods/insights/core/store.mjs"; +import { atomicWriteJson } from "../../herdr-insights/mods/insights/core/util.mjs"; + +function storeWith(name, generation = 0) { + const store = emptyStore(normalizeConfig({ + profile: { name }, + sources: [], + })); + store.generation = generation; + store.scan.generation = generation; + return store; +} + +async function context(prefix) { + const root = await mkdtemp(path.join(tmpdir(), prefix)); + return { root, store: path.join(root, "data", "insights.json") }; +} + +test("an old cache is rejected after a new atomic store commit, even with the same mtime", async () => { + const ctx = await context("insights-cache-identity-"); + const oldStore = storeWith("OLD"); + await saveStore(ctx.store, oldStore); + + const oldStat = await stat(ctx.store); + const staleCache = JSON.parse(await readFile(presentationCachePath(ctx.store), "utf8")); + assert.equal(staleCache.config.profile.name, "OLD"); + + // Simulate a crash after the new store rename but before the cache rename. + const newStore = JSON.parse(JSON.stringify(oldStore)); + newStore.config.profile.name = "NEW"; + newStore.generation = 1; + newStore.scan.generation = 1; + await atomicWriteJson(ctx.store, newStore); + // Deliberately keep the old store mtime inside the former 2ms tolerance. + await utimes(ctx.store, oldStat.atime, oldStat.mtime); + + const newStat = await stat(ctx.store); + const cacheStat = await stat(presentationCachePath(ctx.store)); + assert.ok( + cacheStat.mtimeMs + 2 >= newStat.mtimeMs, + "the stale cache would satisfy the old 2ms mtime tolerance", + ); + const newFingerprint = await storeRevisionFingerprint(ctx.store); + assert.notDeepEqual(staleCache.storeFingerprint, newFingerprint); + + assert.equal(await readPresentationCache(ctx.store), null); + const snapshot = await readCommittedSnapshot(ctx.store); + assert.equal(snapshot.config.profile.name, "NEW", "prepare reads the committed NEW store"); + assert.equal(snapshot.scan.generation, 1); +}); + +test("a valid cache is a fast-path hit and legacy or failed caches fall back", async () => { + const ctx = await context("insights-cache-fallback-"); + await saveStore(ctx.store, storeWith("CURRENT")); + const cached = await readPresentationCache(ctx.store); + assert.equal(cached?.config.profile.name, "CURRENT"); + assert.deepEqual(cached.storeFingerprint, await storeRevisionFingerprint(ctx.store)); + + // Legacy cache without a store fingerprint must not be trusted. + const legacy = { ...cached }; + delete legacy.storeFingerprint; + legacy.version = 1; + await atomicWriteJson(presentationCachePath(ctx.store), legacy); + assert.equal(await readPresentationCache(ctx.store), null); + const fallback = await readCommittedSnapshot(ctx.store); + assert.equal(fallback.config.profile.name, "CURRENT"); + + // A cache write failure must not fail the store commit; prepare uses it. + await rm(presentationCachePath(ctx.store)); + await mkdir(presentationCachePath(ctx.store)); + await saveStore(ctx.store, storeWith("AFTER-FAILURE", 1)); + assert.equal(await readPresentationCache(ctx.store), null); + const committed = await readCommittedSnapshot(ctx.store); + assert.equal(committed.config.profile.name, "AFTER-FAILURE"); +}); + +test("cache identity works through a symlinked store directory", async () => { + const ctx = await context("insights-cache-symlink-"); + const real = path.join(ctx.root, "real"); + const linked = path.join(ctx.root, "linked"); + await mkdir(real, { recursive: true }); + await symlink(real, linked); + const storePath = path.join(linked, "insights.json"); + await saveStore(storePath, storeWith("LINKED")); + const cached = await readPresentationCache(storePath); + assert.equal(cached?.config.profile.name, "LINKED"); + const snapshot = await readCommittedSnapshot(storePath); + assert.equal(snapshot.config.profile.name, "LINKED"); +}); From 93e24372ec3b09ea2ea6e91628269ee33daad276 Mon Sep 17 00:00:00 2001 From: GaganSD Date: Mon, 2 Nov 2026 20:12:18 +0530 Subject: [PATCH 11/15] ci(insights): cover minimum and latest Node 22 Run the Herdr Insights suite on Node 22.15.0, latest Node 22, and Node 24 so the zstd budget compatibility bug cannot recur. --- .github/workflows/ci.yml | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 713c0a50..bc45d28c 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -144,13 +144,17 @@ jobs: name: Herdr Insights runs-on: ubuntu-latest timeout-minutes: 10 + strategy: + fail-fast: false + matrix: + node: ['22.15.0', '22', '24'] steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: persist-credentials: false - uses: actions/setup-node@949feb2413d6458794dcd2491c4babbbce0c15c1 # v7.1.0 with: - node-version: '24' + node-version: ${{ matrix.node }} package-manager-cache: false # Dependency-free; native Herdr checks run locally, not on headless CI. - run: npm run test:insights From 4b752bc11ffe53b075087846ea8e0b77150f836b Mon Sep 17 00:00:00 2001 From: GaganSD Date: Mon, 2 Nov 2026 20:45:41 +0530 Subject: [PATCH 12/15] fix(insights): stream-decode concatenated zstd frames Node 22 stops a zstd decoder after the first frame of a concatenated stream. Split frames structurally and run a fresh decoder per data frame, skipping skippable frames, while feeding one shared line consumer. Truncated frames still roll back and fingerprint; budget stops stay retriable. --- herdr-insights/mods/insights/core/scan.mjs | 320 +++++++++++------- scripts/herdr-insights/benchmark.mjs | 4 +- .../herdr-insights/collector-bounded.test.mjs | 44 +++ .../collector-transactions.test.mjs | 81 +++++ 4 files changed, 322 insertions(+), 127 deletions(-) diff --git a/herdr-insights/mods/insights/core/scan.mjs b/herdr-insights/mods/insights/core/scan.mjs index 42f2bfda..0daaeabe 100644 --- a/herdr-insights/mods/insights/core/scan.mjs +++ b/herdr-insights/mods/insights/core/scan.mjs @@ -1,8 +1,9 @@ import { createHash } from "node:crypto"; +import { once } from "node:events"; import { createReadStream } from "node:fs"; import { open, readdir, realpath, stat } from "node:fs/promises"; import path from "node:path"; -import { Transform } from "node:stream"; +import { PassThrough } from "node:stream"; import { pipeline } from "node:stream/promises"; import * as zlib from "node:zlib"; import { parseAdapterRecord } from "./adapters.mjs"; @@ -351,120 +352,127 @@ const ZSTD_DICT_ID_BYTES = [0, 1, 2, 4]; const ZSTD_FCS_BYTES = [0, 2, 4, 8]; /** - * Structural zstd frame validator. Some Node 22 runtimes silently accept a - * complete frame followed by a truncated extra frame or arbitrary trailing - * bytes. The decoder still streams the payload; this tap only walks frame, - * header, block and checksum lengths so corruption is detected consistently - * instead of depending on runtime-specific end-of-stream behaviour. + * Structural zstd stream splitter. Some Node 22 runtimes stop a decoder at + * the first frame of a concatenated stream, so each data frame is decoded by + * its own fresh decoder. This parser walks frame, header, block and checksum + * lengths to delimit frames; skippable frames are discarded without decoding. */ -function createZstdFrameValidator() { - let buffer = Buffer.alloc(0); +function createZstdFrameSplitter({ onFrameStart, onFrameData, onFrameEnd }) { + let head = Buffer.alloc(0); let state = "magic"; - let frames = 0; let remaining = 0; let lastBlock = false; let checksum = false; + let frames = 0; + let dataFrame = false; const fail = (message) => { throw new Error(`invalid zstd stream: ${message}`); }; - const parse = () => { - for (;;) { - if (state === "magic") { - if (buffer.length < 4) return; - const magic = buffer.readUInt32LE(0); - buffer = buffer.subarray(4); - if (magic === ZSTD_FRAME_MAGIC) { - frames += 1; - state = "frameHeader"; - } else if ((magic & ZSTD_SKIPPABLE_MASK) === ZSTD_SKIPPABLE_MAGIC) { - frames += 1; - state = "skippableSize"; - } else { - fail("unknown frame magic"); - } - } else if (state === "skippableSize") { - if (buffer.length < 4) return; - remaining = buffer.readUInt32LE(0); - buffer = buffer.subarray(4); - state = "skip"; - } else if (state === "skip") { - if (remaining > 0) { - if (buffer.length === 0) return; - const take = Math.min(remaining, buffer.length); - buffer = buffer.subarray(take); + const need = () => { + if (state === "magic" || state === "skippableSize") return 4; + if (state === "blockHeader") return 3; + if (state === "checksum") return 4; + if (state === "frameHeader") { + if (head.length < 1) return 1; + const descriptor = head[0]; + if (descriptor & 0x18) fail("reserved frame header bits"); + const fcsFlag = descriptor >> 6; + const singleSegment = (descriptor >> 5) & 1; + const dictBytes = ZSTD_DICT_ID_BYTES[descriptor & 0x03]; + const fcsBytes = fcsFlag === 0 && singleSegment ? 1 : ZSTD_FCS_BYTES[fcsFlag]; + return 1 + (singleSegment ? 0 : 1) + dictBytes + fcsBytes; + } + return 0; + }; + + const endFrame = async () => { + dataFrame = false; + await onFrameEnd(); + state = "magic"; + }; + + return { + async push(chunk) { + let offset = 0; + while (offset < chunk.length) { + if (state === "skip" || state === "blockData") { + const take = Math.min(remaining, chunk.length - offset); + if (take > 0 && state === "blockData") { + await onFrameData(chunk.subarray(offset, offset + take)); + } + offset += take; remaining -= take; + if (remaining === 0) { + if (state === "skip") { + state = "magic"; + } else if (lastBlock) { + if (checksum) state = "checksum"; + else await endFrame(); + } else { + state = "blockHeader"; + } + } continue; } - state = "magic"; - } else if (state === "frameHeader") { - if (buffer.length < 1) return; - const descriptor = buffer[0]; - if (descriptor & 0x18) fail("reserved frame header bits"); - const fcsFlag = descriptor >> 6; - const singleSegment = (descriptor >> 5) & 1; - checksum = Boolean((descriptor >> 2) & 1); - const dictBytes = ZSTD_DICT_ID_BYTES[descriptor & 0x03]; - const fcsBytes = fcsFlag === 0 && singleSegment ? 1 : ZSTD_FCS_BYTES[fcsFlag]; - const headerBytes = 1 + (singleSegment ? 0 : 1) + dictBytes + fcsBytes; - if (buffer.length < headerBytes) return; - buffer = buffer.subarray(headerBytes); - state = "blockHeader"; - } else if (state === "blockHeader") { - if (buffer.length < 3) return; - const header = buffer.readUIntLE(0, 3); - buffer = buffer.subarray(3); - lastBlock = Boolean(header & 1); - const type = (header >> 1) & 0x03; - if (type === 3) fail("reserved block type"); - remaining = type === 1 ? 1 : header >> 3; - state = "blockData"; - } else if (state === "blockData") { - if (remaining > 0) { - if (buffer.length === 0) return; - const take = Math.min(remaining, buffer.length); - buffer = buffer.subarray(take); - remaining -= take; - continue; + const size = need(); + const take = Math.min(size - head.length, chunk.length - offset); + if (take > 0) { + const piece = chunk.subarray(offset, offset + take); + head = Buffer.concat([head, piece]); + offset += take; + if (dataFrame) await onFrameData(piece); } - if (lastBlock) { - state = checksum ? "checksum" : "magic"; - } else { + if (head.length < size) return; + if (state === "magic") { + const magicBytes = head; + const magic = magicBytes.readUInt32LE(0); + head = Buffer.alloc(0); + if (magic === ZSTD_FRAME_MAGIC) { + frames += 1; + dataFrame = true; + state = "frameHeader"; + await onFrameStart(); + await onFrameData(magicBytes); + } else if ((magic & ZSTD_SKIPPABLE_MASK) === ZSTD_SKIPPABLE_MAGIC) { + frames += 1; + dataFrame = false; + state = "skippableSize"; + } else { + fail("unknown frame magic"); + } + } else if (state === "skippableSize") { + remaining = head.readUInt32LE(0); + head = Buffer.alloc(0); + state = "skip"; + } else if (state === "frameHeader") { + // The descriptor may arrive before the rest of the header; wait for + // the full window/dictionary/content-size fields. + if (head.length < need()) continue; + checksum = Boolean((head[0] >> 2) & 1); + head = Buffer.alloc(0); state = "blockHeader"; + } else if (state === "blockHeader") { + const header = head.readUIntLE(0, 3); + head = Buffer.alloc(0); + lastBlock = Boolean(header & 1); + const type = (header >> 1) & 0x03; + if (type === 3) fail("reserved block type"); + remaining = type === 1 ? 1 : header >> 3; + state = "blockData"; + } else if (state === "checksum") { + head = Buffer.alloc(0); + await endFrame(); } - } else if (state === "checksum") { - if (buffer.length < 4) return; - buffer = buffer.subarray(4); - state = "magic"; } - } - }; - - return { - push(chunk) { - buffer = buffer.length ? Buffer.concat([buffer, chunk]) : chunk; - parse(); }, get complete() { - return frames > 0 && state === "magic" && buffer.length === 0; + return frames > 0 && state === "magic" && head.length === 0; }, }; } -function zstdValidationTap(validator) { - return new Transform({ - transform(chunk, _encoding, callback) { - try { - validator.push(chunk); - callback(null, chunk); - } catch (error) { - callback(error); - } - }, - }); -} - function hasZstdSupport() { return typeof zlib.createZstdDecompress === "function"; } @@ -499,18 +507,9 @@ function lineContext(store, entry, source, sink) { }; } -async function parseBuffers( - store, - entry, - source, - chunks, - startOffset, - endOffset, - sink, - options = {}, -) { +function createRecordConsumer(store, entry, source, sink) { const context = lineContext(store, entry, source, sink); - const consumer = createLineConsumer((record, info) => { + return createLineConsumer((record, info) => { if (info.parseError) { sink.warn( `${source.adapter} ${entry.path}: line ${info.lineNo} is not valid JSON; skipped.`, @@ -527,6 +526,19 @@ async function parseBuffers( ); } }); +} + +async function parseBuffers( + store, + entry, + source, + chunks, + startOffset, + endOffset, + sink, + options = {}, +) { + const consumer = createRecordConsumer(store, entry, source, sink); consumer.offset = startOffset; consumer.lineNo = entry.lineNo ?? 0; const now = nowFn(options); @@ -677,35 +689,93 @@ async function scanCompressedFile( } // The caller owns a per-file transaction. Any decompression failure or // deadline stop discards this open archive without exposing its rows. Raw - // transcript bytes are never retained: the line consumer is bounded. - // Capture the deliberate stop at the destination because some Node 22 - // pipeline implementations re-wrap a destination error as a generic - // AbortError. Only this exact captured ScanStopError is a budget stop; - // every other stream failure remains a corruption/IO failure. + // transcript bytes are never retained: each frame streams through its own + // decoder into one shared line consumer, so a line may span frames. + // Capture the deliberate stop here because some Node 22 pipeline + // implementations re-wrap a destination error as a generic AbortError. + // Only this exact captured ScanStopError is a budget stop; every structural + // or CRC failure on a complete frame remains a corruption failure. let deliberateStop = null; - const validator = createZstdFrameValidator(); - let result; - try { - if (entry.offset || entry.rows.length || Object.keys(entry.state).length) resetFileEntry(store, entry); - result = await pipeline( - createReadStream(filePath), - zstdValidationTap(validator), - zlib.createZstdDecompress(), - async (chunks) => { + let decompressedBytes = 0; + let chunkStart = 0; + let chunksSeen = 0; + let decoderInput = null; + let decoderDone = null; + const consumer = createRecordConsumer(store, entry, source, sink); + consumer.offset = 0; + consumer.lineNo = entry.lineNo ?? 0; + const now = nowFn(options); + const delay = pausableDelay(options.streamDelayMs); + const splitter = createZstdFrameSplitter({ + async onFrameStart() { + decoderInput = new PassThrough(); + const input = decoderInput; + decoderDone = pipeline(input, zlib.createZstdDecompress(), async (chunks) => { try { - return await processStringChunks( - store, entry, source, chunks, 0, undefined, sink, options, - ); + for await (const chunk of chunks) { + if (chunksSeen > 0) { + if (stopRequested(options)) throw new ScanStopError("stop"); + if (deadlineReached(options, now())) throw new ScanStopError("budget"); + } + if (delay) await delay(); + consumer.feedAt(chunk, chunkStart); + chunkStart += chunk.length; + decompressedBytes += chunk.length; + chunksSeen += 1; + if (chunksSeen % PROGRESS_CHUNK_INTERVAL === 0) { + options.onStreamProgress?.({ + file: entry.path, + bytes: decompressedBytes, + phase: "decompress", + }); + } + } } catch (error) { if (error instanceof ScanStopError) deliberateStop = error; throw error; } - }, - ); - if (!validator.complete) { + }); + // The parser awaits the pipeline; this guard only prevents a transient + // unhandled rejection between a destination failure and the next write. + decoderDone.catch(() => {}); + }, + async onFrameData(data) { + if (deliberateStop) throw deliberateStop; + const input = decoderInput; + if (!input.write(data)) { + await Promise.race([once(input, "drain"), decoderDone]); + if (deliberateStop) throw deliberateStop; + } + }, + async onFrameEnd() { + const input = decoderInput; + const done = decoderDone; + decoderInput = null; + decoderDone = null; + input.end(); + await done; + }, + }); + try { + if (entry.offset || entry.rows.length || Object.keys(entry.state).length) resetFileEntry(store, entry); + for await (const chunk of createReadStream(filePath)) { + if (deliberateStop) break; + await splitter.push(chunk); + } + if (deliberateStop) throw deliberateStop; + if (!splitter.complete) { throw new Error("invalid zstd stream: truncated or trailing data"); } + // Finish the shared consumer only at the true end of the stream. Budget + // stops must not commit a partial tail that may still continue. + consumer.finish(chunkStart); + if (consumer.skippedLargeLine) { + sink.warn( + `${source.adapter} ${entry.path}: a line larger than ${MAX_LINE_BYTES} bytes was skipped.`, + ); + } } catch (error) { + decoderInput?.destroy(); const stop = deliberateStop ?? (error instanceof ScanStopError ? error : null); if (stop) { return { status: "stopped", reason: stop.reason, dirty: false }; @@ -714,9 +784,9 @@ async function scanCompressedFile( } stats.bytesRead += fileStat.size; if (fileStat.size > 0) stats.filesRead += 1; - stats.decompressedBytes = (stats.decompressedBytes ?? 0) + (result.bytes ?? 0); + stats.decompressedBytes = (stats.decompressedBytes ?? 0) + decompressedBytes; entry.offset = fileStat.size; - entry.lineNo = result.lineNo; + entry.lineNo = consumer.lineNo; delete entry.state.zstdError; return { status: "ok", dirty: true }; } diff --git a/scripts/herdr-insights/benchmark.mjs b/scripts/herdr-insights/benchmark.mjs index e715b230..c8fdb730 100644 --- a/scripts/herdr-insights/benchmark.mjs +++ b/scripts/herdr-insights/benchmark.mjs @@ -527,8 +527,8 @@ function buildGates(results) { gate("wholeStoreClones", "0", results.wholeStoreClones, results.wholeStoreClones === 0); gate("cachedStatusMedianMs", "<=300 (hard <=1000)", results.status.medianMs, results.status.medianMs <= 300); gate("cachedStatusCompletes", "complete", results.status.finalScanStatus, results.status.complete); - gate("node22FullSuite", "PENDING parent-owned blocker", "not evaluated by this benchmark", null, - "Node 22.23.4 and 22.15.0 full suites fail the expired-budget zstd budget test; the committed cache identity blocker is also deferred to B6. This benchmark does not certify Node 22."); + gate("node22FullSuite", "CI-owned cross-version check", "not evaluated by this benchmark", null, + "This benchmark runs on the invoking Node runtime only and does not compare Node versions. Full-suite and lint results for the supported Node 22 and 24 runtimes are recorded by CI, not certified here."); return gates; } diff --git a/tests/herdr-insights/collector-bounded.test.mjs b/tests/herdr-insights/collector-bounded.test.mjs index b28d110c..6bf2d6cf 100644 --- a/tests/herdr-insights/collector-bounded.test.mjs +++ b/tests/herdr-insights/collector-bounded.test.mjs @@ -323,3 +323,47 @@ test("a bounded scan never exposes rows from an unfinished archive", async () => assert.ok(entry.rows.length > 0, "the valid archive is indexed on retry"); assert.equal(snapshotRows(store).length > 0, true); }); + +test("a budget stop inside a multi-frame zstd archive is retried without a fingerprint", async () => { + const { zstdCompressSync } = await import("node:zlib"); + const bounded = await context("insights-multiframe-budget-"); + const sessions = path.join(bounded.home, ".codex", "sessions"); + await mkdir(sessions, { recursive: true }); + const plain = await readFile(new URL("./fixtures/codex-modern.jsonl", import.meta.url), "utf8"); + const lines = plain.split("\n").filter(Boolean); + const mid = Math.floor(lines.length / 2); + await writeFile(path.join(sessions, "multi.jsonl.zst"), Buffer.concat([ + zstdCompressSync(`${lines.slice(0, mid).join("\n")}\n`), + zstdCompressSync(`${lines.slice(mid).join("\n")}\n`), + ])); + const store = emptyStore(normalizeConfig({ + sources: [{ + id: "codex", + label: "Codex", + adapter: "codex", + enabled: true, + roots: ["~/.codex/sessions"], + }], + })); + const stopped = await scanSources(store, { + home: bounded.home, + deadline: Date.now(), + now: Date.now, + }); + assert.equal(stopped.abortedFile?.path?.endsWith("multi.jsonl.zst"), true); + assert.equal(stopped.stats.errors, 0); + assert.equal(Object.keys(store.files).length, 0, "an aborted archive contributes no file entry"); + assert.deepEqual(snapshotRows(store), []); + + const retry = await scanSources(store, { + home: bounded.home, + deadline: Date.now() + 60_000, + now: Date.now, + }); + assert.equal(retry.complete, true); + assert.equal(retry.stats.errors, 0); + const [entry] = Object.values(store.files); + assert.equal(entry.state.zstdError, undefined, "a budget stop leaves no corrupt fingerprint"); + assert.equal(entry.rows.length, 2, "both frames are indexed on retry"); + assert.equal(snapshotRows(store).reduce((sum, row) => sum + row.messages, 0), 2); +}); diff --git a/tests/herdr-insights/collector-transactions.test.mjs b/tests/herdr-insights/collector-transactions.test.mjs index badd3331..6871896a 100644 --- a/tests/herdr-insights/collector-transactions.test.mjs +++ b/tests/herdr-insights/collector-transactions.test.mjs @@ -218,6 +218,87 @@ test("a first-seen corrupt archive persists a rowless fingerprint and recovers", assert.ok(entry.rows.length > 0, "a valid replacement is indexed"); }); +function archiveStore() { + const store = emptyStore(); + store.config = { + ...store.config, + sources: [{ id: "codex", label: "Codex", adapter: "codex", enabled: true, roots: ["~/.codex/sessions"] }], + }; + return store; +} + +async function codexFrames() { + const plain = await readFile(path.join(here, "fixtures", "codex-modern.jsonl"), "utf8"); + const lines = plain.split("\n").filter(Boolean); + const mid = Math.floor(lines.length / 2); + return [ + zstdCompressSync(`${lines.slice(0, mid).join("\n")}\n`), + zstdCompressSync(`${lines.slice(mid).join("\n")}\n`), + ]; +} + +async function scanArchive(frames) { + const root = await mkdtemp(path.join(tmpdir(), "insights-multiframe-")); + const home = path.join(root, "home"); + const sessions = path.join(home, ".codex", "sessions"); + await mkdir(sessions, { recursive: true }); + const file = path.join(sessions, "archive.jsonl.zst"); + await writeFile(file, Buffer.concat(frames)); + const store = archiveStore(); + const first = await scanSources(store, { home }); + return { file, store, first }; +} + +test("two concatenated zstd frames both contribute rows", async () => { + const [firstFrame, secondFrame] = await codexFrames(); + const { store, first } = await scanArchive([firstFrame, secondFrame]); + assert.equal(first.stats.errors, 0); + assert.equal(first.stats.filesRead, 1); + const [entry] = Object.values(store.files); + assert.equal(entry.state.zstdError, undefined); + assert.equal(entry.rows.length, 2, "both frames were decoded"); + const rows = snapshotRows(store); + assert.deepEqual( + rows.map((row) => row.model).sort(), + ["gpt-6.1", "gpt-6.2"], + ); + assert.equal(rows.reduce((sum, row) => sum + row.messages, 0), 2); +}); + +test("a skippable zstd frame between data frames is ignored, not decoded", async () => { + const [firstFrame, secondFrame] = await codexFrames(); + const skippable = Buffer.from([0x50, 0x2a, 0x4d, 0x18, 3, 0, 0, 0, 0xaa, 0xbb, 0xcc]); + const { store, first } = await scanArchive([firstFrame, skippable, secondFrame]); + assert.equal(first.stats.errors, 0); + const rows = snapshotRows(store); + assert.deepEqual( + rows.map((row) => row.model).sort(), + ["gpt-6.1", "gpt-6.2"], + ); + assert.equal(rows.reduce((sum, row) => sum + row.messages, 0), 2); +}); + +test("a truncated second zstd frame rolls back and stays fingerprinted", async () => { + const [firstFrame, secondFrame] = await codexFrames(); + const { file, store, first } = await scanArchive([ + firstFrame, + secondFrame.subarray(0, secondFrame.length - 20), + ]); + assert.equal(first.stats.errors, 1, "the truncated frame is corrupt"); + assert.equal(snapshotRows(store).length, 0, "no frame rows are committed"); + const [entry] = Object.values(store.files); + assert.equal(entry.rows.length, 0); + const badStat = await stat(file); + assert.equal(entry.state.zstdError, `${badStat.mtimeMs}:${badStat.size}`); + assert.equal(entry.offset, badStat.size); + + const repeated = await scanSources(store, { home: path.dirname(path.dirname(path.dirname(file))) }); + assert.equal(repeated.stats.errors, 0); + assert.equal(repeated.stats.filesRead, 0, "the corrupt fingerprint suppresses rereads"); + assert.equal(repeated.stats.bytesRead, 0); + assert.equal(snapshotRows(store).length, 0); +}); + test("production scanning does not clone the accumulated store", async () => { const scanSource = await readFile( path.join(here, "../../herdr-insights/mods/insights/core/scan.mjs"), From cec09d279ebc92a1ed5766bfd939627fed03e2c1 Mon Sep 17 00:00:00 2001 From: GaganSD Date: Mon, 2 Nov 2026 20:52:04 +0530 Subject: [PATCH 13/15] fix(insights): reset archive parser context and accept empty frames --- herdr-insights/mods/insights/core/scan.mjs | 15 +++++-- .../collector-transactions.test.mjs | 40 +++++++++++++++++++ 2 files changed, 51 insertions(+), 4 deletions(-) diff --git a/herdr-insights/mods/insights/core/scan.mjs b/herdr-insights/mods/insights/core/scan.mjs index 0daaeabe..ed82c868 100644 --- a/herdr-insights/mods/insights/core/scan.mjs +++ b/herdr-insights/mods/insights/core/scan.mjs @@ -445,7 +445,7 @@ function createZstdFrameSplitter({ onFrameStart, onFrameData, onFrameEnd }) { } else if (state === "skippableSize") { remaining = head.readUInt32LE(0); head = Buffer.alloc(0); - state = "skip"; + state = remaining === 0 ? "magic" : "skip"; } else if (state === "frameHeader") { // The descriptor may arrive before the rest of the header; wait for // the full window/dictionary/content-size fields. @@ -461,6 +461,11 @@ function createZstdFrameSplitter({ onFrameStart, onFrameData, onFrameEnd }) { if (type === 3) fail("reserved block type"); remaining = type === 1 ? 1 : header >> 3; state = "blockData"; + if (remaining === 0) { + if (!lastBlock) state = "blockHeader"; + else if (checksum) state = "checksum"; + else await endFrame(); + } } else if (state === "checksum") { head = Buffer.alloc(0); await endFrame(); @@ -701,9 +706,7 @@ async function scanCompressedFile( let chunksSeen = 0; let decoderInput = null; let decoderDone = null; - const consumer = createRecordConsumer(store, entry, source, sink); - consumer.offset = 0; - consumer.lineNo = entry.lineNo ?? 0; + let consumer; const now = nowFn(options); const delay = pausableDelay(options.streamDelayMs); const splitter = createZstdFrameSplitter({ @@ -758,6 +761,10 @@ async function scanCompressedFile( }); try { if (entry.offset || entry.rows.length || Object.keys(entry.state).length) resetFileEntry(store, entry); + // Bind parser context only after resetting the archive's previous state. + consumer = createRecordConsumer(store, entry, source, sink); + consumer.offset = 0; + consumer.lineNo = entry.lineNo ?? 0; for await (const chunk of createReadStream(filePath)) { if (deliberateStop) break; await splitter.push(chunk); diff --git a/tests/herdr-insights/collector-transactions.test.mjs b/tests/herdr-insights/collector-transactions.test.mjs index 6871896a..98d561d4 100644 --- a/tests/herdr-insights/collector-transactions.test.mjs +++ b/tests/herdr-insights/collector-transactions.test.mjs @@ -278,6 +278,19 @@ test("a skippable zstd frame between data frames is ignored, not decoded", async assert.equal(rows.reduce((sum, row) => sum + row.messages, 0), 2); }); +test("empty trailing data and skippable frames preserve preceding archive rows", async () => { + const [firstFrame, secondFrame] = await codexFrames(); + const emptySkip = Buffer.from([0x50, 0x2a, 0x4d, 0x18, 0, 0, 0, 0]); + const { store, first } = await scanArchive([ + firstFrame, secondFrame, zstdCompressSync(Buffer.alloc(0)), emptySkip, + ]); + assert.equal(first.stats.errors, 0); + assert.equal(snapshotRows(store).reduce((sum, row) => sum + row.messages, 0), 2); + const empty = await scanArchive([zstdCompressSync(Buffer.alloc(0))]); + assert.equal(empty.first.stats.errors, 0); + assert.deepEqual(snapshotRows(empty.store), []); +}); + test("a truncated second zstd frame rolls back and stays fingerprinted", async () => { const [firstFrame, secondFrame] = await codexFrames(); const { file, store, first } = await scanArchive([ @@ -299,6 +312,33 @@ test("a truncated second zstd frame rolls back and stays fingerprinted", async ( assert.equal(snapshotRows(store).length, 0); }); +test("replacing an archive binds the parser to fresh session state", async (t) => { + const plain = await readFile(path.join(here, "fixtures", "codex-modern.jsonl"), "utf8"); + const { file, store, first } = await scanArchive([ + zstdCompressSync(plain.replaceAll("codex-fixture", "before-replacement")), + ]); + const home = path.dirname(path.dirname(path.dirname(file))); + t.after(async () => { + const { rm } = await import("node:fs/promises"); + await rm(path.dirname(home), { recursive: true, force: true }); + }); + assert.equal(first.stats.errors, 0); + assert.deepEqual([...new Set(snapshotRows(store).map((row) => row.session))], ["codex:before-replacement"]); + // Grow the same inode so truncation handling cannot hide a stale parser context. + const skip = Buffer.alloc(1024); + skip.writeUInt32LE(0x184d2a50, 0); + skip.writeUInt32LE(skip.length - 8, 4); + await writeFile(file, Buffer.concat([ + zstdCompressSync(plain.replaceAll("codex-fixture", "after-replacement")), + skip, + ])); + const updated = await scanSources(store, { home }); + assert.equal(updated.stats.errors, 0); + const rows = snapshotRows(store); + assert.deepEqual([...new Set(rows.map((row) => row.session))], ["codex:after-replacement"]); + assert.equal(rows.reduce((sum, row) => sum + row.messages, 0), 2); +}); + test("production scanning does not clone the accumulated store", async () => { const scanSource = await readFile( path.join(here, "../../herdr-insights/mods/insights/core/scan.mjs"), From 28dd083e5bc99bf99e56701e6405241a703b2c8d Mon Sep 17 00:00:00 2001 From: GaganSD Date: Mon, 2 Nov 2026 21:38:49 +0530 Subject: [PATCH 14/15] fix(insights): stop review stalls in scan and worker control Unchanged settings no longer leave a live fence ahead of the committed generation. Signal only a same-host worker, publish claim pids without an empty file, read plain lines through to a boundary, and honor budget and cancel stops on empty and skippable zstd frames. --- herdr-insights/mods/insights/collector.mjs | 187 ++++++++++-------- herdr-insights/mods/insights/core/fence.mjs | 81 +++++++- herdr-insights/mods/insights/core/scan.mjs | 41 +++- herdr-insights/mods/insights/indexer.mjs | 12 +- .../herdr-insights/collector-bounded.test.mjs | 57 +++++- .../collector-transactions.test.mjs | 44 +++++ .../collector-worker-lifecycle.test.mjs | 147 +++++++++++++- 7 files changed, 473 insertions(+), 96 deletions(-) diff --git a/herdr-insights/mods/insights/collector.mjs b/herdr-insights/mods/insights/collector.mjs index 3041aa86..71db7758 100644 --- a/herdr-insights/mods/insights/collector.mjs +++ b/herdr-insights/mods/insights/collector.mjs @@ -10,6 +10,7 @@ import { bumpFence, readEffectiveFence, readWorkerClaim, + relaxFence, restoreFence, workerClaimActive, } from "./core/fence.mjs"; @@ -296,11 +297,13 @@ async function runWithStoreLock( if ( signalWorker && !signaled + && owner.host === os.hostname() && Number.isInteger(owner.pid) && owner.pid > 0 && (fenceGeneration === null || Number(owner.generation ?? 0) < fenceGeneration) ) { - // Only the pid recorded by the current writer lease is signalled. + // Same host only. A shared store directory must not signal whatever + // local process happens to reuse the other machine's worker pid. try { process.kill(owner.pid, "SIGTERM"); } catch { /* already exiting */ } signaled = true; } @@ -437,14 +440,14 @@ async function scanAndPersist( if (completed && !mustSave && store.lastCompletedScanAt === null) mustSave = true; const guarded = guard ? await guard() : null; - if (guarded) return { fenced: true }; + if (guarded) return { fenced: true, saved: false }; if (result.abortedFile) { // Persist the abandoned file marker under the writer lock; the worker is // spawned by the caller only AFTER this lock is released so it can take // ownership immediately instead of exiting on a busy lock. await saveStore(storePath, store, { status: "partial", now: finishedIso }); const snapshot = buildSnapshot(store, result, finishedIso, loaded?.warnings, loaded?.errors); - return { snapshot, worker: null, promote: true }; + return { snapshot, worker: null, promote: true, saved: true }; } if (mustSave) { @@ -460,7 +463,7 @@ async function scanAndPersist( store.lastCompletedScanAt = loaded?.store?.lastCompletedScanAt ?? store.lastCompletedScanAt; } const snapshot = buildSnapshot(store, result, finishedIso, loaded?.warnings, loaded?.errors); - return { snapshot, worker: null }; + return { snapshot, worker: null, saved: mustSave }; } async function promoteIndexer(storePath, home, snapshot, testStreamDelayMs = 0) { @@ -630,82 +633,112 @@ async function configureStore( const loaded = await loadStore(storePath); const store = loaded.store; let dirty = loaded.dirty; - store.generation = Math.max(storeGeneration(store) + 1, fence.generation); - const previousStatus = store.scan.status; - const configChanged = JSON.stringify(store.config) !== JSON.stringify(config); - const mappingsChanged = sourcesSignature(store.config) !== sourcesSignature(config); - const cleared = mappingsChanged || rebuild; - if (cleared) { - clearCollectedData(store); - store.scan = emptyScanState(); - store.scan.generation = store.generation; - store.scan.startedAt = new Date(now()).toISOString(); - store.lastCompletedScanAt = null; - dirty = true; - } - if (configChanged) { - store.config = config; - dirty = true; - } - applied = true; - // Persist accepted config first. Config acceptance is never reported as a - // completed scan; retained committed data keeps its completed timestamp. - const acceptedComplete = !cleared && previousStatus === "complete"; - const acceptedIso = new Date(now()).toISOString(); - if (dirty) { - store.scan.generation = store.generation; - store.scan.updatedAt = acceptedIso; - await saveStore(storePath, store, { - completed: false, - status: acceptedComplete ? "complete" : "partial", - now: acceptedIso, - }); - dirty = false; - } + const previousGeneration = storeGeneration(store); + store.generation = Math.max(previousGeneration + 1, fence.generation); + let persistedGeneration = previousGeneration; + let outcome = null; + try { + const previousStatus = store.scan.status; + const configChanged = JSON.stringify(store.config) !== JSON.stringify(config); + const mappingsChanged = sourcesSignature(store.config) !== sourcesSignature(config); + const cleared = mappingsChanged || rebuild; + if (cleared) { + clearCollectedData(store); + store.scan = emptyScanState(); + store.scan.generation = store.generation; + store.scan.startedAt = new Date(now()).toISOString(); + store.lastCompletedScanAt = null; + dirty = true; + } + if (configChanged) { + store.config = config; + dirty = true; + } + applied = true; + // Persist accepted config first. Config acceptance is never reported as a + // completed scan; retained committed data keeps its completed timestamp. + const acceptedComplete = !cleared && previousStatus === "complete"; + const acceptedIso = new Date(now()).toISOString(); + if (dirty) { + store.scan.generation = store.generation; + store.scan.updatedAt = acceptedIso; + await saveStore(storePath, store, { + completed: false, + status: acceptedComplete ? "complete" : "partial", + now: acceptedIso, + }); + persistedGeneration = store.generation; + dirty = false; + } - if (await removeStalePendingFile(storePath, store, now)) dirty = true; - if (store.scan.pendingFile) { - const nowIso = new Date(now()).toISOString(); - if (dirty) await saveStore(storePath, store, { status: "partial", now: nowIso }); - const snapshot = buildSnapshot( - store, - scanResultFromStore(store), - nowIso, - loaded.warnings, - loaded.errors, - ); - snapshot.scan = { ...snapshot.scan, status: "partial" }; - return { snapshot, worker: null, promote: true }; - } + if (await removeStalePendingFile(storePath, store, now)) dirty = true; + if (store.scan.pendingFile) { + const nowIso = new Date(now()).toISOString(); + if (dirty) { + await saveStore(storePath, store, { status: "partial", now: nowIso }); + persistedGeneration = store.generation; + } + const snapshot = buildSnapshot( + store, + scanResultFromStore(store), + nowIso, + loaded.warnings, + loaded.errors, + ); + snapshot.scan = { ...snapshot.scan, status: "partial" }; + outcome = { snapshot, worker: null, promote: true }; + return outcome; + } - const scanned = await scanAndPersist(storePath, home, store, { - loaded, - dirty, - budgetMs, - now, - testStreamDelayMs, - mode: "configure", - guard: async () => { - const fenceAfter = await readEffectiveFence(storePath, { now: now() }); - return Boolean(fenceAfter && !fenceAfter.stale && fenceAfter.generation > store.generation); - }, - }); - if (scanned.fenced) { - // A newer configure is already queued; its generation fence owns the - // next commit. Return committed data without writing this generation. - const committed = await loadStore(storePath, { readOnly: true }); - const nowIso = new Date(now()).toISOString(); - const snapshot = buildSnapshot( - committed.store, - scanResultFromStore(committed.store), - nowIso, - loaded.warnings, - loaded.errors, - ); - snapshot.scan = { ...snapshot.scan, status: "reconfiguring" }; - return { snapshot, worker: null }; + const scanned = await scanAndPersist(storePath, home, store, { + loaded, + dirty, + budgetMs, + now, + testStreamDelayMs, + mode: "configure", + guard: async () => { + const fenceAfter = await readEffectiveFence(storePath, { now: now() }); + return Boolean(fenceAfter && !fenceAfter.stale && fenceAfter.generation > store.generation); + }, + }); + if (scanned.fenced) { + // A newer configure is already queued; its generation fence owns the + // next commit. Return committed data without writing this generation. + const committed = await loadStore(storePath, { readOnly: true }); + const nowIso = new Date(now()).toISOString(); + const snapshot = buildSnapshot( + committed.store, + scanResultFromStore(committed.store), + nowIso, + loaded.warnings, + loaded.errors, + ); + snapshot.scan = { ...snapshot.scan, status: "reconfiguring" }; + outcome = { snapshot, worker: null }; + return outcome; + } + if (scanned.saved) persistedGeneration = store.generation; + outcome = scanned; + return outcome; + } finally { + // An unchanged settings save bumps the fence before it knows whether + // a write is required. If this generation was not committed, put the + // fence back so a live configure process cannot pin later snapshots + // in "reconfiguring" until the fence expires. + if (persistedGeneration < fence.generation) { + await relaxFence(storePath, fence.generation, persistedGeneration); + store.generation = persistedGeneration; + if (store.scan) store.scan.generation = persistedGeneration; + if (Number.isInteger(outcome?.snapshot?.scan?.generation) + && outcome.snapshot.scan.generation !== persistedGeneration) { + outcome.snapshot.scan = { + ...outcome.snapshot.scan, + generation: persistedGeneration, + }; + } + } } - return scanned; }, { waitMs: lockWaitMs, meta: { role: "configure", generation: fence.generation }, diff --git a/herdr-insights/mods/insights/core/fence.mjs b/herdr-insights/mods/insights/core/fence.mjs index ff054b7a..62269946 100644 --- a/herdr-insights/mods/insights/core/fence.mjs +++ b/herdr-insights/mods/insights/core/fence.mjs @@ -122,16 +122,77 @@ export async function restoreFence(storePath, generation) { return true; } -export async function readWorkerClaim(storePath) { +/** + * Drop a fence this process still owns back to the committed generation. + * A newer configure (different generation or pid) is left untouched. Generation + * 0 removes the file; readers treat a missing fence as "no configure queued". + */ +export async function relaxFence(storePath, ownedGeneration, committedGeneration) { + const current = await readFence(storePath); + if (!current || current.pid !== process.pid) return false; + if (current.generation !== ownedGeneration) return false; + if (!Number.isInteger(committedGeneration) || committedGeneration >= ownedGeneration) return false; + if (committedGeneration < 1) { + await unlink(fencePath(storePath)).catch((error) => { + if (error?.code !== "ENOENT") throw error; + }); + return true; + } + await writeFence(storePath, committedGeneration); + return true; +} + +/** + * Startup readers must tell "reaped" apart from "publisher is mid-write". + * A missing file is replacement. An empty or torn file is still the same claim + * until the startup deadline; treating it as replacement kills a live worker. + */ +export async function readWorkerClaimResult(storePath) { + let text; try { - const value = await readJsonFile(workerClaimPath(storePath)); - if (!isPlainObject(value)) return null; - if (typeof value.token !== "string" || value.token.length === 0) return null; - if (!Number.isInteger(value.generation) || value.generation < 0) return null; - return value; + text = await readFile(workerClaimPath(storePath), "utf8"); + } catch (error) { + if (error?.code === "ENOENT") return { status: "missing" }; + return { status: "pending" }; + } + let value; + try { + value = JSON.parse(text); } catch { - return null; + return { status: "pending" }; + } + if (!isPlainObject(value) || typeof value.token !== "string" || value.token.length === 0) { + return { status: "pending" }; } + if (!Number.isInteger(value.generation) || value.generation < 0) return { status: "pending" }; + return { status: "ok", claim: value }; +} + +export async function readWorkerClaim(storePath) { + const result = await readWorkerClaimResult(storePath); + return result.status === "ok" ? result.claim : null; +} + +/** Token checks stay strict. Only an unreadable in-place publish is retried. */ +export function classifyWorkerClaim(result, token, pid) { + if (!result || result.status === "missing") return "replaced"; + if (result.status !== "ok") return "pending"; + if (result.claim?.token !== token) return "replaced"; + if (result.claim.pid === pid) return "ready"; + if (Number.isInteger(result.claim.pid) && result.claim.pid > 0) return "other"; + return "pending"; +} + +/** + * Pid publication bytes. The record is never shorter than the previous file, + * so the publisher does not have to truncate first. Trailing spaces are JSON + * whitespace; a concurrent reader never observes an empty claim. + */ +export function claimPidRecord(currentText, next) { + const body = JSON.stringify(next); + const previousBytes = Buffer.byteLength(String(currentText ?? "")); + const pad = Math.max(0, previousBytes - Buffer.byteLength(body) - 1); + return `${body}${" ".repeat(pad)}\n`; } export async function writeWorkerClaim(storePath, claim) { @@ -173,8 +234,10 @@ export async function updateWorkerClaimPid(storePath, token, pid) { try { current = JSON.parse(text); } catch { current = null; } if (!isPlainObject(current) || current.token !== token) return false; const next = { ...current, pid, at: Date.now() }; - await handle.truncate(0); - await handle.write(`${JSON.stringify(next)}\n`, 0, "utf8"); + // Overwrite in place at offset 0. Truncating first publishes an empty + // claim, and the worker treats that as claim-replaced and exits. + const content = claimPidRecord(text, next); + await handle.write(content, 0, "utf8"); await handle.sync(); return true; } catch (error) { diff --git a/herdr-insights/mods/insights/core/scan.mjs b/herdr-insights/mods/insights/core/scan.mjs index ed82c868..fd7abd7a 100644 --- a/herdr-insights/mods/insights/core/scan.mjs +++ b/herdr-insights/mods/insights/core/scan.mjs @@ -357,7 +357,13 @@ const ZSTD_FCS_BYTES = [0, 2, 4, 8]; * its own fresh decoder. This parser walks frame, header, block and checksum * lengths to delimit frames; skippable frames are discarded without decoding. */ -function createZstdFrameSplitter({ onFrameStart, onFrameData, onFrameEnd }) { +function createZstdFrameSplitter({ + onFrameStart, + onFrameData, + onFrameEnd, + onFrameBoundary, + onSkippedBytes, +}) { let head = Buffer.alloc(0); let state = "magic"; let remaining = 0; @@ -401,6 +407,8 @@ function createZstdFrameSplitter({ onFrameStart, onFrameData, onFrameEnd }) { const take = Math.min(remaining, chunk.length - offset); if (take > 0 && state === "blockData") { await onFrameData(chunk.subarray(offset, offset + take)); + } else if (take > 0 && state === "skip") { + await onSkippedBytes?.(take); } offset += take; remaining -= take; @@ -430,12 +438,14 @@ function createZstdFrameSplitter({ onFrameStart, onFrameData, onFrameEnd }) { const magic = magicBytes.readUInt32LE(0); head = Buffer.alloc(0); if (magic === ZSTD_FRAME_MAGIC) { + await onFrameBoundary?.(); frames += 1; dataFrame = true; state = "frameHeader"; await onFrameStart(); await onFrameData(magicBytes); } else if ((magic & ZSTD_SKIPPABLE_MASK) === ZSTD_SKIPPABLE_MAGIC) { + await onFrameBoundary?.(); frames += 1; dataFrame = false; state = "skippableSize"; @@ -641,7 +651,14 @@ async function scanPlainFile( bytes += chunk.length; consumer.feedAt(chunk, chunkStart); }, { - shouldStop: () => stopRequested(options) || deadlineReached(options, now()), + // Honor cancellation immediately. A deadline stop must still reach a + // newline or the large-line cap: one 256 KiB chunk with no newline + // leaves the safe offset unchanged, and every later slice repeats it. + shouldStop: () => { + if (stopRequested(options)) return true; + if (!deadlineReached(options, now())) return false; + return consumer.offset > start || consumer.discarding; + }, }); bytes = result.bytes; stopped = result.stopped; @@ -709,7 +726,24 @@ async function scanCompressedFile( let consumer; const now = nowFn(options); const delay = pausableDelay(options.streamDelayMs); + let frameBoundaries = 0; + let skippedBytes = 0; + const enforceCompressedLimit = () => { + if (stopRequested(options)) throw new ScanStopError("stop"); + if (deadlineReached(options, now())) throw new ScanStopError("budget"); + }; const splitter = createZstdFrameSplitter({ + async onFrameBoundary() { + frameBoundaries += 1; + // The first frame may start after the foreground budget is already + // gone. Every later frame, including empty and skippable frames that + // produce no decoder output, has to observe the same stop. + if (frameBoundaries > 1) enforceCompressedLimit(); + }, + async onSkippedBytes(count) { + skippedBytes += count; + if (skippedBytes >= CHUNK_BYTES) enforceCompressedLimit(); + }, async onFrameStart() { decoderInput = new PassThrough(); const input = decoderInput; @@ -765,9 +799,12 @@ async function scanCompressedFile( consumer = createRecordConsumer(store, entry, source, sink); consumer.offset = 0; consumer.lineNo = entry.lineNo ?? 0; + let sawCompressedChunk = false; for await (const chunk of createReadStream(filePath)) { + if (sawCompressedChunk) enforceCompressedLimit(); if (deliberateStop) break; await splitter.push(chunk); + sawCompressedChunk = true; } if (deliberateStop) throw deliberateStop; if (!splitter.complete) { diff --git a/herdr-insights/mods/insights/indexer.mjs b/herdr-insights/mods/insights/indexer.mjs index b0a63a29..88920676 100644 --- a/herdr-insights/mods/insights/indexer.mjs +++ b/herdr-insights/mods/insights/indexer.mjs @@ -12,10 +12,11 @@ import { stat } from "node:fs/promises"; import path from "node:path"; import { fileURLToPath } from "node:url"; import { + classifyWorkerClaim, clearWorkerClaim, readEffectiveFence, readFenceStrict, - readWorkerClaim, + readWorkerClaimResult, storeGeneration, } from "./core/fence.mjs"; import { progressWriter } from "./core/progress.mjs"; @@ -95,10 +96,11 @@ async function recordWorkerFailure(storePath, generation, error) { async function waitForClaim(storePath, token) { const deadline = Date.now() + CLAIM_WAIT_MS; for (;;) { - const claim = await readWorkerClaim(storePath); - if (!claim || claim.token !== token) return { ok: false, reason: "claim-replaced" }; - if (claim.pid === process.pid) return { ok: true, claim }; - if (Number.isInteger(claim.pid) && claim.pid > 0) return { ok: false, reason: "claim-other-pid" }; + const result = await readWorkerClaimResult(storePath); + const decision = classifyWorkerClaim(result, token, process.pid); + if (decision === "ready") return { ok: true, claim: result.claim }; + if (decision === "replaced") return { ok: false, reason: "claim-replaced" }; + if (decision === "other") return { ok: false, reason: "claim-other-pid" }; if (Date.now() >= deadline) return { ok: false, reason: "claim-timeout" }; await new Promise((resolve) => setTimeout(resolve, 40)); } diff --git a/tests/herdr-insights/collector-bounded.test.mjs b/tests/herdr-insights/collector-bounded.test.mjs index 6bf2d6cf..5130f4de 100644 --- a/tests/herdr-insights/collector-bounded.test.mjs +++ b/tests/herdr-insights/collector-bounded.test.mjs @@ -146,7 +146,7 @@ test("plain JSONL continuation reads only the unfinished tail and is not fingerp assert.equal(third.stats.filesRead, 0); }); -test("a budget stop before a newline never commits a parseable mid-line prefix", async () => { +test("a stop before a newline never commits a parseable mid-line prefix", async () => { const contextInfo = await context("insights-plain-boundary-"); const file = path.join(contextInfo.home, ".pi", "agent", "sessions", "boundary.jsonl"); const record = { @@ -167,7 +167,9 @@ test("a budget stop before a newline never commits a parseable mid-line prefix", assert.ok(head.length < chunkBytes, "the valid prefix must fit in one chunk"); // The first chunk is complete JSON followed by whitespace, so a premature // finish() would accept it. The non-whitespace suffix makes the whole line - // invalid, and a budget stop must leave it uncommitted. + // invalid, and a cancellation inside that line must leave it uncommitted. + // Budget exhaustion is not this case: a slice keeps reading until a newline + // so the next refresh cannot repeat the same chunk forever. const prefix = Buffer.concat([head, Buffer.alloc(chunkBytes - head.length, 0x20)]); const suffix = Buffer.from("trailing-junk\n"); await writeFile(file, Buffer.concat([prefix, suffix])); @@ -178,10 +180,17 @@ test("a budget stop before a newline never commits a parseable mid-line prefix", assert.deepEqual(snapshotRows(oracleStore), [], "one-shot rejects the whole line"); const store = emptyStore(normalizeConfig(sourceConfig())); + let polls = 0; const first = await scanSources(store, { home: contextInfo.home, deadline: Date.now(), now: Date.now, + // The first poll is the scan loop. Later polls are inside the open file, + // so cancellation lands after the first chunk and before the newline. + shouldStop: () => { + polls += 1; + return polls > 1; + }, }); assert.equal(first.complete, false); assert.equal(first.stopped, true); @@ -202,6 +211,50 @@ test("a budget stop before a newline never commits a parseable mid-line prefix", assert.equal(entry.state.scanPending, undefined); }); +test("an exhausted budget finishes a plain line longer than one read chunk", async () => { + const contextInfo = await context("insights-long-line-"); + const file = path.join(contextInfo.home, ".pi", "agent", "sessions", "long.jsonl"); + const record = { + type: "message", + id: "assistant-long-line", + timestamp: "2026-04-04T00:00:00Z", + message: { + role: "assistant", + model: "long-model", + provider: "long-provider", + usage: { input: 9, output: 1, cacheRead: 0, cacheWrite: 0, reasoning: 0 }, + stopReason: "stop", + content: [{ type: "text", text: "z".repeat(300 * 1024) }], + }, + }; + const line = `${JSON.stringify(record)}\n`; + assert.ok(Buffer.byteLength(line) > 256 * 1024, "the line must span more than one chunk"); + await writeFile(file, line); + const totalSize = (await stat(file)).size; + const store = emptyStore(normalizeConfig(sourceConfig())); + let result; + for (let attempt = 0; attempt < 3; attempt += 1) { + result = await scanSources(store, { + home: contextInfo.home, + deadline: Date.now(), + now: Date.now, + }); + if (result.complete) break; + } + assert.equal(result.complete, true, "a long line must not be retried from the same offset"); + const entry = Object.values(store.files)[0]; + assert.equal(entry.offset, totalSize); + assert.equal(entry.state.scanPending, undefined); + assert.equal(snapshotRows(store).reduce((sum, row) => sum + row.tokens.input, 0), 9); + const again = await scanSources(store, { + home: contextInfo.home, + deadline: Date.now(), + now: Date.now, + }); + assert.equal(again.stats.bytesRead, 0, "the finished line is not read again"); + assert.equal(again.stats.filesRead, 0); +}); + test("partial scans expose savedAt separately from the last completed scan", async () => { const contextInfo = await context("insights-timestamps-"); for (let index = 0; index < 3; index += 1) { diff --git a/tests/herdr-insights/collector-transactions.test.mjs b/tests/herdr-insights/collector-transactions.test.mjs index 98d561d4..c32697d4 100644 --- a/tests/herdr-insights/collector-transactions.test.mjs +++ b/tests/herdr-insights/collector-transactions.test.mjs @@ -278,6 +278,50 @@ test("a skippable zstd frame between data frames is ignored, not decoded", async assert.equal(rows.reduce((sum, row) => sum + row.messages, 0), 2); }); +test("empty and skippable frames still honor a scan deadline", async () => { + const emptySkip = Buffer.from([0x50, 0x2a, 0x4d, 0x18, 0, 0, 0, 0]); + const empties = Buffer.concat(Array.from({ length: 8 }, () => emptySkip)); + const payload = 256 * 1024; + const header = Buffer.alloc(8); + header.writeUInt32LE(0x184d2a50, 0); + header.writeUInt32LE(payload, 4); + const hugeSkip = Buffer.concat([header, Buffer.alloc(payload)]); + const root = await mkdtemp(path.join(tmpdir(), "insights-empty-limit-")); + const home = path.join(root, "home"); + const sessions = path.join(home, ".codex", "sessions"); + await mkdir(sessions, { recursive: true }); + await writeFile(path.join(sessions, "empty.jsonl.zst"), Buffer.concat([empties, hugeSkip])); + const store = archiveStore(); + const stopped = await scanSources(store, { + home, + deadline: Date.now(), + now: Date.now, + }); + assert.equal(stopped.complete, false); + assert.equal(stopped.abortedFile?.reason, "budget"); + assert.equal(stopped.stats.errors, 0, "a budget stop is not archive corruption"); + assert.equal(Object.keys(store.files).length, 0, "the unfinished archive is rolled back"); + + const cancelledRoot = await mkdtemp(path.join(tmpdir(), "insights-empty-cancel-")); + const cancelledHome = path.join(cancelledRoot, "home"); + const cancelledSessions = path.join(cancelledHome, ".codex", "sessions"); + await mkdir(cancelledSessions, { recursive: true }); + await writeFile(path.join(cancelledSessions, "empty.jsonl.zst"), Buffer.concat([empties, hugeSkip])); + const cancelledStore = archiveStore(); + let polls = 0; + const cancelled = await scanSources(cancelledStore, { + home: cancelledHome, + // Let the scan enter the archive, then cancel on the next empty frame. + shouldStop: () => { + polls += 1; + return polls > 1; + }, + }); + assert.equal(cancelled.complete, false); + assert.equal(cancelled.abortedFile?.reason, "stop"); + assert.equal(Object.keys(cancelledStore.files).length, 0); +}); + test("empty trailing data and skippable frames preserve preceding archive rows", async () => { const [firstFrame, secondFrame] = await codexFrames(); const emptySkip = Buffer.from([0x50, 0x2a, 0x4d, 0x18, 0, 0, 0, 0]); diff --git a/tests/herdr-insights/collector-worker-lifecycle.test.mjs b/tests/herdr-insights/collector-worker-lifecycle.test.mjs index 5f7349d9..6be5b79e 100644 --- a/tests/herdr-insights/collector-worker-lifecycle.test.mjs +++ b/tests/herdr-insights/collector-worker-lifecycle.test.mjs @@ -7,7 +7,14 @@ import test from "node:test"; import { collectSnapshot } from "../../herdr-insights/mods/insights/collector.mjs"; import { normalizeConfig } from "../../herdr-insights/mods/insights/core/config.mjs"; -import { readWorkerClaim, workerClaimPath } from "../../herdr-insights/mods/insights/core/fence.mjs"; +import { + claimPidRecord, + classifyWorkerClaim, + readFence, + readWorkerClaim, + updateWorkerClaimPid, + workerClaimPath, +} from "../../herdr-insights/mods/insights/core/fence.mjs"; import { ensureIndexer, indexerPath, @@ -191,6 +198,144 @@ test("worker failure is recorded under the lock and never clobbers a queued gene } }); +test("unchanged configure does not pin later snapshots on a live fence", async () => { + const ctx = await context("insights-unchanged-config-"); + await writeFile( + path.join(ctx.home, ".pi", "agent", "sessions", "one.jsonl"), + piSession("one", 4), + ); + const config = { sources: piSource }; + const first = await collectSnapshot("configure", { + store: ctx.store, + home: ctx.home, + config, + }); + assert.equal(first.scan.status, "complete"); + const inputOf = (snapshot) => snapshot.rows.reduce((sum, row) => sum + row.tokens.input, 0); + assert.equal(inputOf(first), 4); + const generation = JSON.parse(await readFile(ctx.store, "utf8")).generation; + + const second = await collectSnapshot("configure", { + store: ctx.store, + home: ctx.home, + config, + }); + assert.notEqual(second.scan.status, "reconfiguring"); + const committed = JSON.parse(await readFile(ctx.store, "utf8")); + assert.equal(committed.generation, generation, "an unchanged save does not bump the generation"); + const fence = await readFence(ctx.store); + if (fence) assert.ok(fence.generation <= committed.generation); + + await writeFile( + path.join(ctx.home, ".pi", "agent", "sessions", "two.jsonl"), + piSession("two", 8), + ); + const third = await collectSnapshot("snapshot", { store: ctx.store, home: ctx.home }); + assert.equal(third.scan.status, "complete"); + assert.equal(inputOf(third), 12, "new usage is scanned instead of staying reconfiguring"); + await rm(ctx.root, { recursive: true, force: true }); +}); + +test("configure signals only a same-host worker pid", async () => { + const ctx = await context("insights-foreign-signal-"); + let signaled = false; + const onSignal = () => { signaled = true; }; + process.on("SIGTERM", onSignal); + const foreignLock = `${ctx.store}.lock`; + await mkdir(foreignLock, { recursive: true }); + await writeFile(path.join(foreignLock, "owner-foreign.json"), `${JSON.stringify({ + role: "worker", + pid: process.pid, + host: "foreign-host", + generation: 0, + token: "foreign-token", + startedAt: Date.now(), + })}\n`); + try { + await assert.rejects( + collectSnapshot("configure", { + store: ctx.store, + home: ctx.home, + config: { sources: piSource }, + lockWaitMs: 150, + }), + /busy/, + ); + assert.equal(signaled, false, "a foreign host lock must not be signaled on this machine"); + } finally { + process.off("SIGTERM", onSignal); + await rm(ctx.root, { recursive: true, force: true }); + } + + const local = await context("insights-local-signal-"); + const child = spawn(process.execPath, ["-e", "setInterval(() => {}, 1000)"], { stdio: "ignore" }); + const exited = new Promise((resolve) => child.once("exit", () => resolve(true))); + const localLock = `${local.store}.lock`; + await mkdir(localLock, { recursive: true }); + const { hostname } = await import("node:os"); + await writeFile(path.join(localLock, "owner-local.json"), `${JSON.stringify({ + role: "worker", + pid: child.pid, + host: hostname(), + generation: 0, + token: "local-token", + startedAt: Date.now(), + })}\n`); + try { + // A dead same-host lock is reclaimed, so configure may proceed after the + // signal. The assertion is that this machine's worker pid was signaled. + await collectSnapshot("configure", { + store: local.store, + home: local.home, + config: { sources: piSource }, + lockWaitMs: 200, + }).catch((error) => { + assert.match(String(error?.message), /busy/); + }); + const didExit = await Promise.race([ + exited, + new Promise((resolve) => setTimeout(() => resolve(false), 1_000)), + ]); + assert.equal(didExit, true, "the same-host worker pid is signaled"); + } finally { + if (child.pid && isProcessAlive(child.pid)) { + try { process.kill(child.pid, "SIGKILL"); } catch { /* already gone */ } + } + await rm(local.root, { recursive: true, force: true }); + } +}); + +test("pid publication keeps the worker claim parseable", async () => { + const ctx = await context("insights-claim-pid-"); + const claim = { + version: 1, + token: "token-a", + pid: null, + host: "local", + generation: 3, + startedAt: 1, + at: 10, + }; + const original = `${JSON.stringify({ ...claim, note: "x".repeat(80) })}\n`; + await writeFile(workerClaimPath(ctx.store), original); + const record = claimPidRecord(original, { ...JSON.parse(original), pid: 42, at: 2 }); + assert.ok(Buffer.byteLength(record) >= Buffer.byteLength(original)); + assert.equal(JSON.parse(record).pid, 42); + assert.equal(JSON.parse(record).token, "token-a"); + assert.equal(await updateWorkerClaimPid(ctx.store, "token-a", 42), true); + const text = await readFile(workerClaimPath(ctx.store), "utf8"); + assert.ok(text.trim().length > 0, "the claim is never published empty"); + assert.equal(JSON.parse(text).pid, 42); + assert.equal(JSON.parse(text).token, "token-a"); + assert.equal(classifyWorkerClaim({ status: "pending" }, "token-a", 42), "pending"); + assert.equal(classifyWorkerClaim({ status: "missing" }, "token-a", 42), "replaced"); + assert.equal( + classifyWorkerClaim({ status: "ok", claim: { ...claim, token: "other" } }, "token-a", 42), + "replaced", + ); + await rm(ctx.root, { recursive: true, force: true }); +}); + test("promotion spawns one worker that waits out the launcher lock handoff", async () => { const ctx = await context("insights-worker-promotion-"); const big = path.join(ctx.home, ".pi", "agent", "sessions", "big.jsonl"); From 82c53c9bb7a87cb35a6428221de8038ad2325f9b Mon Sep 17 00:00:00 2001 From: GaganSD Date: Tue, 3 Nov 2026 08:53:52 +0530 Subject: [PATCH 15/15] refactor(insights): drop unused scan and worker wrappers --- herdr-insights/mods/insights/collector.mjs | 2 - herdr-insights/mods/insights/core/cache.mjs | 7 ---- herdr-insights/mods/insights/core/fence.mjs | 16 +------- .../mods/insights/core/indexer-control.mjs | 17 --------- herdr-insights/mods/insights/core/scan.mjs | 38 +------------------ herdr-insights/mods/insights/indexer.mjs | 2 +- tests/herdr-insights/ship-readiness.test.mjs | 2 +- 7 files changed, 4 insertions(+), 80 deletions(-) diff --git a/herdr-insights/mods/insights/collector.mjs b/herdr-insights/mods/insights/collector.mjs index 71db7758..767e2518 100644 --- a/herdr-insights/mods/insights/collector.mjs +++ b/herdr-insights/mods/insights/collector.mjs @@ -37,8 +37,6 @@ import { withStoreLock, } from "./core/util.mjs"; -export { progressWriter }; - // Leave room for startup, store load, save, and presentation derivation inside // Herdr's fixed 30s exec budget. A partial slice returns committed data and the // UI continues automatically. diff --git a/herdr-insights/mods/insights/core/cache.mjs b/herdr-insights/mods/insights/core/cache.mjs index 25f97f88..1738a135 100644 --- a/herdr-insights/mods/insights/core/cache.mjs +++ b/herdr-insights/mods/insights/core/cache.mjs @@ -100,10 +100,3 @@ export async function readPresentationCache(storePath) { return null; } } - -export async function removePresentationCache(storePath) { - const { unlink } = await import("node:fs/promises"); - await unlink(presentationCachePath(storePath)).catch((error) => { - if (error?.code !== "ENOENT") throw error; - }); -} diff --git a/herdr-insights/mods/insights/core/fence.mjs b/herdr-insights/mods/insights/core/fence.mjs index 62269946..bbdf00f3 100644 --- a/herdr-insights/mods/insights/core/fence.mjs +++ b/herdr-insights/mods/insights/core/fence.mjs @@ -27,11 +27,6 @@ export function workerClaimPath(storePath) { return `${storePath}.worker.json`; } -export function storeGeneration(store) { - const value = Number(store?.generation); - return Number.isInteger(value) && value >= 0 ? value : 0; -} - export function nextGeneration(...values) { let highest = 0; for (const value of values) { @@ -195,11 +190,6 @@ export function claimPidRecord(currentText, next) { return `${body}${" ".repeat(pad)}\n`; } -export async function writeWorkerClaim(storePath, claim) { - await atomicWriteJson(workerClaimPath(storePath), claim); - return claim; -} - /** * Exclusive claim creation (O_CREAT|O_EXCL); never last-write-wins. Concurrent * launchers either create the claim or observe EEXIST and coalesce. @@ -254,7 +244,7 @@ export async function updateWorkerClaimPid(storePath, token, pid) { * claim that was replaced between observation and rename is restored with an * exclusive link so it is never destroyed by a stale reaper. */ -async function reapWorkerClaim(storePath, token) { +export async function clearWorkerClaim(storePath, token) { const file = workerClaimPath(storePath); const quarantine = `${file}.${process.pid}-${randomBytes(6).toString("hex")}.reap`; try { @@ -276,10 +266,6 @@ async function reapWorkerClaim(storePath, token) { return true; } -export async function clearWorkerClaim(storePath, token) { - return reapWorkerClaim(storePath, token); -} - /** * A claim is live while its recorded pid is alive. A claim without a pid is * only a spawn in progress; it expires quickly so a crashed launcher cannot diff --git a/herdr-insights/mods/insights/core/indexer-control.mjs b/herdr-insights/mods/insights/core/indexer-control.mjs index 10c91189..510c82e8 100644 --- a/herdr-insights/mods/insights/core/indexer-control.mjs +++ b/herdr-insights/mods/insights/core/indexer-control.mjs @@ -6,7 +6,6 @@ import { clearWorkerClaim, readEffectiveFence, readWorkerClaim, - storeGeneration, tryCreateWorkerClaim, updateWorkerClaimPid, workerClaimActive, @@ -137,19 +136,3 @@ export async function waitForIndexerAck(storePath, claim, { timeoutMs = DEFAULT_ await sleep(40); } } - -export async function indexerGeneration(storePath) { - const fence = await readEffectiveFence(storePath); - return { - fence, - generation: fence && !fence.stale ? fence.generation : null, - }; -} - -export function indexerScriptPath() { - return indexerPath(); -} - -export function storeGenerationOrZero(store) { - return storeGeneration(store); -} diff --git a/herdr-insights/mods/insights/core/scan.mjs b/herdr-insights/mods/insights/core/scan.mjs index fd7abd7a..80641ff1 100644 --- a/herdr-insights/mods/insights/core/scan.mjs +++ b/herdr-insights/mods/insights/core/scan.mjs @@ -588,19 +588,6 @@ async function parseBuffers( return { offset: consumer.offset, lineNo: consumer.lineNo, bytes }; } -async function processStringChunks( - store, - entry, - source, - chunks, - startOffset, - endOffset, - sink, - options = {}, -) { - return parseBuffers(store, entry, source, chunks, startOffset, endOffset, sink, options); -} - async function scanPlainFile( store, entry, @@ -622,24 +609,7 @@ async function scanPlainFile( return { dirty: false, pending: false }; } - const context = lineContext(store, entry, source, sink); - const consumer = createLineConsumer((record, info) => { - if (info.parseError) { - sink.warn( - `${adapter} ${entry.path}: line ${info.lineNo} is not valid JSON; skipped.`, - ); - return; - } - if (!isPlainObject(record)) return; - context.lineHash = info.hash; - try { - parseAdapterRecord(adapter, record, context); - } catch (error) { - sink.warn( - `${adapter} ${entry.path}: line ${info.lineNo} could not be parsed: ${error.message}`, - ); - } - }); + const consumer = createRecordConsumer(store, entry, source, sink); consumer.offset = start; consumer.lineNo = entry.lineNo ?? 0; consumer.discarding = entry.state.discardingLargeLine; @@ -1386,9 +1356,3 @@ export async function scanSources(store, { total: discovered.length, }; } - -export async function rebuildScan(store, options = {}) { - return scanSources(store, options); -} - -export { createLineConsumer, processStringChunks, scanOneFile, walkRoot }; diff --git a/herdr-insights/mods/insights/indexer.mjs b/herdr-insights/mods/insights/indexer.mjs index 88920676..0cd8d0c4 100644 --- a/herdr-insights/mods/insights/indexer.mjs +++ b/herdr-insights/mods/insights/indexer.mjs @@ -17,7 +17,6 @@ import { readEffectiveFence, readFenceStrict, readWorkerClaimResult, - storeGeneration, } from "./core/fence.mjs"; import { progressWriter } from "./core/progress.mjs"; import { scanSources } from "./core/scan.mjs"; @@ -26,6 +25,7 @@ import { ensureStoreMeta, loadStore, saveStore, + storeGeneration, } from "./core/store.mjs"; import { isLockBusyError, withStoreLock } from "./core/util.mjs"; diff --git a/tests/herdr-insights/ship-readiness.test.mjs b/tests/herdr-insights/ship-readiness.test.mjs index 183b27a4..3e86dfcc 100644 --- a/tests/herdr-insights/ship-readiness.test.mjs +++ b/tests/herdr-insights/ship-readiness.test.mjs @@ -5,7 +5,7 @@ import { hostname, tmpdir } from "node:os"; import path from "node:path"; import { registerHooks } from "node:module"; import test from "node:test"; -import { progressWriter } from "../../herdr-insights/mods/insights/collector.mjs"; +import { progressWriter } from "../../herdr-insights/mods/insights/core/progress.mjs"; import { compactDashboard, deriveDashboard } from "../../herdr-insights/mods/insights/analytics.js"; import { MAX_LINE_BYTES, withStoreLock } from "../../herdr-insights/mods/insights/core/util.mjs"; import { normalizeConfig } from "../../herdr-insights/mods/insights/core/config.mjs";