diff --git a/src/codex/inject-coordination.ts b/src/codex/inject-coordination.ts index c3ced10ba2..e91e9f339b 100644 --- a/src/codex/inject-coordination.ts +++ b/src/codex/inject-coordination.ts @@ -16,6 +16,7 @@ import { CODEX_CONFIG_PATH, CODEX_PROFILE_PATH } from "./paths"; import type { CodexArtifactId, CodexProvenanceEntry, + CodexProvenanceLedger, } from "./convergence-types"; import { codexWriteCoordination, @@ -196,6 +197,41 @@ export function captureCodexPreImages(): CodexPreImages { */ export const CODEX_PROVENANCE_MAX_TRANSACTIONS = 16; +/** + * How many serialized bytes of evidence the ledger keeps. + * + * The transaction window alone bounds the ledger only if each transaction is small, and a + * baseline embeds the artifact's exact bytes. Those artifacts sit outside this proxy's trust + * boundary: a native `config.toml` grown to hundreds of megabytes is copied into the record as + * base64 and re-serialized on every append, so a single oversized file turns a 16-transaction + * window into a multi-gigabyte write. + * + * The size is derived from the window above rather than picked: the 25 KB `config.toml` that + * comment describes measures 100.8 KiB per transaction, so a full 16-transaction window is + * 1.58 MiB. 4 MiB leaves that ordinary window untouched with room to spare while still refusing + * the pathological case, and a regression test pins the ordinary window so the two cannot drift + * apart silently. + */ +export const CODEX_PROVENANCE_MAX_BYTES = 4 * 1024 * 1024; + +/** + * Measure the representation the integration-record writer actually emits. + * + * The minimal wrapper reproduces the final indentation depth of `provenance.entries`, including + * structured unknown extension fields preserved from older records. Its fixed wrapper makes this + * slightly conservative as a ledger-only ceiling, which is preferable to admitting an entry that + * expands past the limit when `JSON.stringify(record, null, 2)` writes it. + */ +function serializedProvenanceBytes( + entries: readonly CodexProvenanceEntry[], + ledger: CodexProvenanceLedger | undefined, +): number { + return Buffer.byteLength(`${JSON.stringify({ + version: 1, + provenance: { ...ledger, entries }, + }, null, 2)}\n`, "utf-8"); +} + function provenanceBaseline(bytes: string | null): CodexProvenanceEntry["baseline"] { if (bytes === null) return { kind: "absent" }; return { @@ -214,21 +250,66 @@ function provenancePostImage(path: string): string | null { } /** - * Keep the newest `CODEX_PROVENANCE_MAX_TRANSACTIONS` transactions, whole. + * Keep the newest transactions that fit both the transaction window and the byte ceiling, whole. * * Trimming by ENTRY count would cut a transaction in half and leave evidence that says a * transaction touched two artifacts when it touched three — worse than dropping it outright, - * because a partial record still reads as complete. Order is preserved; only whole leading - * transactions are removed. + * because a partial record still reads as complete. Order is preserved; only whole transactions + * are removed. + * + * The byte ceiling applies the same rule: a transaction that does not fit is omitted whole, + * including the newest one — a record silently truncated to fit would be read as a faithful + * pre-image. An oversized transaction is skipped rather than ending the scan, so one pathological + * artifact cannot erase the smaller transactions that still fit and are still diagnosable. */ export function boundProvenanceEntries( entries: readonly CodexProvenanceEntry[], maxTransactions = CODEX_PROVENANCE_MAX_TRANSACTIONS, + maxBytes = CODEX_PROVENANCE_MAX_BYTES, + ledger?: CodexProvenanceLedger, ): readonly CodexProvenanceEntry[] { - const order: string[] = []; - for (const entry of entries) if (!order.includes(entry.txId)) order.push(entry.txId); - if (order.length <= maxTransactions) return entries; - const keep = new Set(order.slice(order.length - maxTransactions)); + const transactions = new Map(); + for (const entry of entries) { + const transaction = transactions.get(entry.txId); + if (transaction) transaction.push(entry); + else transactions.set(entry.txId, [entry]); + } + const order = [...transactions.keys()]; + if (order.length <= maxTransactions) { + const baselineBytes = entries.reduce( + (total, entry) => total + (entry.baseline.kind === "present" ? entry.baseline.bytesBase64.length : 0), + 0, + ); + if ( + baselineBytes <= maxBytes + && serializedProvenanceBytes(entries, ledger) <= maxBytes + ) return entries; + } + // Newest first, so the transactions anyone diagnoses against are the ones that fit. + const keep = new Set(); + let baselineBytes = 0; + for (let i = order.length - 1; i >= 0 && keep.size < maxTransactions; i--) { + const txId = order[i]!; + const txEntries = transactions.get(txId)!; + // Measure the embedded pre-images first: a pathological baseline is refused without + // serializing it, so the ceiling does not itself allocate the payload it exists to reject. + const transactionBaselineBytes = txEntries.reduce( + (total, entry) => total + (entry.baseline.kind === "present" ? entry.baseline.bytesBase64.length : 0), + 0, + ); + if (baselineBytes + transactionBaselineBytes > maxBytes) continue; + const candidateKeep = new Set(keep); + candidateKeep.add(txId); + // Reuse the already-grouped transactions: this avoids another full-ledger filter on each + // iteration while preserving the original transaction and entry order. + const candidateEntries = order.flatMap(id => + candidateKeep.has(id) ? transactions.get(id)! : [] + ); + if (serializedProvenanceBytes(candidateEntries, ledger) > maxBytes) continue; + baselineBytes += transactionBaselineBytes; + keep.add(txId); + } + if (keep.size === order.length) return entries; return entries.filter(entry => keep.has(entry.txId)); } @@ -250,13 +331,29 @@ export function recordCodexNativeTransactionProvenance( txId, at, })); - return updateIntegrationRecord(record => ({ - ...record, - provenance: { - ...record.provenance, - entries: boundProvenanceEntries([...(record.provenance?.entries ?? []), ...entries]), - }, - })); + return updateIntegrationRecord(record => { + const previousLedger = record.provenance; + // Unknown ledger extensions are forward-compatible and must be preserved. If those fixed + // fields alone exceed the ceiling, no entry selection can make the write compliant. Keep the + // existing record byte-for-byte instead of deleting all known evidence and rewriting the + // same oversized extension on every native transaction. + if ( + previousLedger + && serializedProvenanceBytes([], previousLedger) > CODEX_PROVENANCE_MAX_BYTES + ) return record; + return { + ...record, + provenance: { + ...previousLedger, + entries: boundProvenanceEntries( + [...(previousLedger?.entries ?? []), ...entries], + CODEX_PROVENANCE_MAX_TRANSACTIONS, + CODEX_PROVENANCE_MAX_BYTES, + previousLedger, + ), + }, + }; + }); } /** diff --git a/src/codex/integration-record.ts b/src/codex/integration-record.ts index e595d81a58..177d4dd1a4 100644 --- a/src/codex/integration-record.ts +++ b/src/codex/integration-record.ts @@ -204,12 +204,17 @@ function mergeEntry(previous: CodexProvenanceEntry, next: CodexProvenanceEntry): function mergeLedger(previous: CodexProvenanceLedger, next: CodexProvenanceLedger): CodexProvenanceLedger { const unused = new Set(previous.entries.map((_, index) => index)); const entries = next.entries.map((entry, nextIndex) => { - let previousIndex = previous.entries.findIndex((candidate, index) => + // Extensions follow provenance identity, never position. A bounded ledger may drop an + // oversized transaction, shifting an unrelated entry into that slot, and even within one + // transaction and timestamp the artifact may differ. Since this search already requires + // txId, timestamp, and artifact identity, any remaining positional match is by definition + // a different artifact, so preserving its unknown entry/artifact/baseline fields would + // attach false evidence and could regrow the record past the measured byte ceiling. + const previousIndex = previous.entries.findIndex((candidate, index) => unused.has(index) && candidate.txId === entry.txId && candidate.at === entry.at && knownArtifactIdentity(candidate.artifact) === knownArtifactIdentity(entry.artifact)); - if (previousIndex < 0 && unused.has(nextIndex)) previousIndex = nextIndex; if (previousIndex < 0) return entry; unused.delete(previousIndex); return mergeEntry(previous.entries[previousIndex]!, entry); @@ -245,6 +250,11 @@ export const updateIntegrationRecord = ( if (read.kind === "invalid") return read; const previous: CodexIntegrationRecord = read.kind === "ready" ? read.record : { version: 1 }; const proposed = mutate(previous); + // The mutator can explicitly decline an update by returning the exact record it received. + // This matters when a preserved forward-compatible extension already exceeds a caller's + // write budget: rewriting the same oversized bytes would amplify I/O while deleting known + // entries cannot make that irreducible overhead fit. + if (proposed === previous) return { kind: "updated" as const, record: previous }; if (!validateRecord(proposed)) { return { kind: "invalid", message: "Codex integration record update produced a malformed v1 shape" }; } diff --git a/tests/codex-inject-write-lock.test.ts b/tests/codex-inject-write-lock.test.ts index ed49cd5e10..ea544842fc 100644 --- a/tests/codex-inject-write-lock.test.ts +++ b/tests/codex-inject-write-lock.test.ts @@ -15,7 +15,12 @@ import { resolveCodexCoordinatorDatabasePath, resolveEffectiveUserIdentity, } from "../src/codex/user-identity"; -import { boundProvenanceEntries, STABLE_ZERO_BYTE_COORDINATOR_AGE_MS } from "../src/codex/inject-coordination"; +import { + boundProvenanceEntries, + CODEX_PROVENANCE_MAX_BYTES, + CODEX_PROVENANCE_MAX_TRANSACTIONS, + STABLE_ZERO_BYTE_COORDINATOR_AGE_MS, +} from "../src/codex/inject-coordination"; import { SPAWN_BUDGET_MS } from "./helpers/test-budget"; const repoRoot = join(import.meta.dir, ".."); @@ -246,6 +251,55 @@ describe("the lock is on the production path", () => { .toBe("{ malformed"); }); + test("irreducible ledger extension overhead refuses the append without rewriting", () => { + seedNative(); + const recordPath = join(opencodexHome, "integrations", "codex.json"); + mkdirSync(join(opencodexHome, "integrations"), { recursive: true }); + writeFileSync(recordPath, JSON.stringify({ + version: 1, + provenance: { + futureLedger: "x".repeat(CODEX_PROVENANCE_MAX_BYTES + 1), + entries: [{ + artifact: { kind: "config" }, + baseline: { kind: "absent" }, + postImage: null, + txId: "tx-existing", + at: "2026-08-30T00:00:00.000Z", + }], + }, + })); + const before = readFileSync(recordPath, "utf8"); + + const result = parseChildJson<{ kind?: string; entryCount?: number }>( + runChild(["--eval", ` + const { + captureCodexPreImages, + recordCodexNativeTransactionProvenance, + } = require("./src/codex/inject-coordination"); + const result = recordCodexNativeTransactionProvenance( + captureCodexPreImages(), + "tx-must-not-append", + ); + console.log(JSON.stringify({ + kind: result.kind, + entryCount: result.kind === "updated" + ? result.record.provenance?.entries?.length + : undefined, + })); + `], { + ...process.env, + CODEX_HOME: codexHome, + OPENCODEX_HOME: opencodexHome, + }), + "irreducible ledger extension overhead", + ); + + expect(result.kind).toBe("updated"); + expect(result.entryCount).toBe(1); + // Exact bytes, not just semantics: a no-op must not pretty-print/rewrite the oversized file. + expect(readFileSync(recordPath, "utf8")).toBe(before); + }); + /** * The contention proof. A real second process holds N through the production * lock module while a real injection runs; the injection must report busy and @@ -508,4 +562,150 @@ describe("provenance ledger bound", () => { const entries = Array.from({ length: 16 }, (_, i) => transaction(`tx-${i}`)).flat(); expect(boundProvenanceEntries(entries, 16)).toBe(entries); }); + + test("an oversized baseline is omitted whole rather than amplified into the record", () => { + // The native artifacts sit outside this proxy's trust boundary, so a `config.toml` grown to + // an arbitrary size would otherwise be copied into the record as base64 and re-serialized on + // every append — the transaction window alone does not bound that. + const huge = transaction("tx-huge"); + huge[0] = { + ...huge[0]!, + baseline: { + kind: "present" as const, + sha256: "0".repeat(64), + bytesBase64: "A".repeat(CODEX_PROVENANCE_MAX_BYTES + 1), + }, + }; + // The baseline-size prefilter must reject this transaction before JSON.stringify reaches + // the tripwire. Only one sibling is oversized; all three must still be omitted together. + Object.defineProperty(huge[0]!, "toJSON", { + value: () => { + throw new Error("oversized transaction was serialized"); + }, + }); + const entries = [...transaction("tx-small"), ...huge]; + + const bounded = boundProvenanceEntries(entries, 16); + + expect(bounded.map(e => e.txId)).toEqual(["tx-small", "tx-small", "tx-small"]); + expect(Buffer.byteLength(JSON.stringify(bounded), "utf-8")) + .toBeLessThanOrEqual(CODEX_PROVENANCE_MAX_BYTES); + }); + + test("backfills the transaction window after an oversized newest transaction is skipped", () => { + const maxBytes = 64 * 1024; + const oversized = transaction("tx-16"); + oversized[0] = { + ...oversized[0]!, + baseline: { + kind: "present" as const, + sha256: "0".repeat(64), + bytesBase64: "A".repeat(maxBytes + 1), + }, + }; + const entries = [ + ...Array.from({ length: 16 }, (_, i) => transaction(`tx-${i}`)).flat(), + ...oversized, + ]; + + const bounded = boundProvenanceEntries(entries, 16, maxBytes); + const kept = [...new Set(bounded.map(entry => entry.txId))]; + + expect(kept).toEqual(Array.from({ length: 16 }, (_, i) => `tx-${i}`)); + for (const txId of kept) expect(bounded.filter(entry => entry.txId === txId)).toHaveLength(3); + }); + + test("the byte ceiling drops whole transactions, oldest first", () => { + const padded = (txId: string) => transaction(txId).map(entry => ({ + ...entry, + baseline: { + kind: "present" as const, + sha256: "0".repeat(64), + bytesBase64: "A".repeat(Math.floor(CODEX_PROVENANCE_MAX_BYTES / 4)), + }, + })); + const entries = Array.from({ length: 4 }, (_, i) => padded(`tx-${i}`)).flat(); + + const bounded = boundProvenanceEntries(entries, 16); + const kept = [...new Set(bounded.map(e => e.txId))]; + + expect(kept.length).toBeGreaterThan(0); + expect(kept.length).toBeLessThan(4); + // Newest survive; the dropped ones are the oldest, and each survivor is whole. + expect(kept.at(-1)).toBe("tx-3"); + for (const txId of kept) expect(bounded.filter(e => e.txId === txId)).toHaveLength(3); + expect(Buffer.byteLength(JSON.stringify(bounded), "utf-8")) + .toBeLessThanOrEqual(CODEX_PROVENANCE_MAX_BYTES); + }); + + test("the ceiling measures structured extensions in the pretty-printed record shape", () => { + const extended = (txId: string) => transaction(txId).map((entry, index) => ({ + ...entry, + // Integration records preserve unknown structured fields. Compact JSON can fit while the + // writer's two-space indentation does not, so the bound must use the actual write shape. + extension: index === 0 + ? { rows: Array.from({ length: 400 }, () => ({ left: "x", right: "y" })) } + : undefined, + })); + const entries = [...extended("tx-old"), ...extended("tx-new")]; + const oneTransactionBytes = Buffer.byteLength(`${JSON.stringify({ + version: 1, + provenance: { entries: extended("tx-new") }, + }, null, 2)}\n`, "utf-8"); + const bothCompactBytes = Buffer.byteLength(JSON.stringify(entries), "utf-8"); + const bothPrettyBytes = Buffer.byteLength(`${JSON.stringify({ + version: 1, + provenance: { entries }, + }, null, 2)}\n`, "utf-8"); + const maxBytes = Math.max(oneTransactionBytes, bothCompactBytes); + + expect(bothPrettyBytes).toBeGreaterThan(maxBytes); + const bounded = boundProvenanceEntries(entries, 16, maxBytes); + + expect([...new Set(bounded.map(entry => entry.txId))]).toEqual(["tx-new"]); + }); + + test("unknown ledger extensions consume the same byte budget as entries", () => { + const entries = [...transaction("tx-old"), ...transaction("tx-new")]; + const ledger = { + entries, + futureLedger: { + rows: Array.from({ length: 200 }, () => ({ left: "x", right: "y" })), + }, + }; + const writeBytes = (candidate: readonly (typeof entries)[number][]) => + Buffer.byteLength(`${JSON.stringify({ + version: 1, + provenance: { ...ledger, entries: candidate }, + }, null, 2)}\n`, "utf-8"); + const newest = transaction("tx-new"); + const maxBytes = writeBytes(newest); + + expect(writeBytes(entries)).toBeGreaterThan(maxBytes); + const bounded = boundProvenanceEntries(entries, 16, maxBytes, ledger); + + expect([...new Set(bounded.map(entry => entry.txId))]).toEqual(["tx-new"]); + expect(writeBytes(bounded)).toBeLessThanOrEqual(maxBytes); + }); + + test("a full window of ordinary transactions is still kept whole", () => { + // The ceiling exists to refuse pathological artifacts, not to shrink the window above it. + // This pins the two together: the 25 KB `config.toml` the window comment describes measures + // about 100 KiB per transaction, so a full window is roughly 1.6 MiB and must survive intact. + const ordinary = Buffer.from("x".repeat(25 * 1024)).toString("base64"); + const entries = Array.from( + { length: CODEX_PROVENANCE_MAX_TRANSACTIONS }, + (_, i) => transaction(`tx-${i}`).map(entry => ({ + ...entry, + baseline: { kind: "present" as const, sha256: "0".repeat(64), bytesBase64: ordinary }, + postImage: "0".repeat(64), + })), + ).flat(); + + const bounded = boundProvenanceEntries(entries); + + expect(bounded).toBe(entries); + expect(new Set(bounded.map(e => e.txId)).size).toBe(CODEX_PROVENANCE_MAX_TRANSACTIONS); + expect(bounded).toHaveLength(entries.length); + }); }); diff --git a/tests/codex-integration-record.test.ts b/tests/codex-integration-record.test.ts index 57ad5f9704..e0315f1522 100644 --- a/tests/codex-integration-record.test.ts +++ b/tests/codex-integration-record.test.ts @@ -6,6 +6,7 @@ import { readIntegrationRecord, updateIntegrationRecord, } from "../src/codex/integration-record"; +import { boundProvenanceEntries } from "../src/codex/inject-coordination"; import type { CodexArtifactId, CodexIntegrationRecord, @@ -145,6 +146,95 @@ describe("Codex integration record", () => { ); }); + test("does not move an omitted transaction's extensions onto its positional replacement", () => { + const maxBytes = 16 * 1024; + const entry = ( + txId: string, + at: string, + baseline: CodexProvenanceEntry["baseline"], + futureEntry?: unknown, + ): CodexProvenanceEntry => ({ + artifact: { kind: "config" }, + baseline, + postImage: null, + txId, + at, + ...(futureEntry === undefined ? {} : { futureEntry }), + }); + const kept = entry("tx-kept", "2026-08-04T00:00:00.000Z", { kind: "absent" }); + const oversized = entry( + "tx-oversized", + "2026-08-04T00:00:01.000Z", + { + kind: "present", + sha256: "0".repeat(64), + bytesBase64: "A".repeat(maxBytes + 1), + }, + { mustNotMove: "x".repeat(maxBytes * 2) }, + ); + writeRecord({ version: 1, provenance: { entries: [kept, oversized] } }); + + const appended = entry("tx-new", "2026-08-04T00:00:02.000Z", { kind: "absent" }); + const result = updateIntegrationRecord(record => ({ + ...record, + provenance: { + ...record.provenance, + entries: boundProvenanceEntries( + [...record.provenance!.entries, appended], + 16, + maxBytes, + ), + }, + })); + + expect(result.kind).toBe("updated"); + const saved = persistedRecord(); + const entries = (saved.provenance as { entries: Array> }).entries; + expect(entries.map(savedEntry => savedEntry.txId)).toEqual(["tx-kept", "tx-new"]); + expect(entries[1]).not.toHaveProperty("futureEntry"); + expect(Buffer.byteLength(JSON.stringify(saved, null, 2), "utf-8")) + .toBeLessThanOrEqual(maxBytes); + }); + + test("does not move extensions onto a different artifact in the same transaction", () => { + const at = "2026-08-04T00:00:00.000Z"; + writeRecord({ + version: 1, + provenance: { + entries: [{ + artifact: { kind: "config", futureArtifact: { owner: "config" } }, + baseline: { kind: "absent", futureBaseline: { owner: "config" } }, + postImage: null, + txId: "tx-same", + at, + futureEntry: { owner: "config" }, + } as unknown as CodexProvenanceEntry], + }, + }); + + const result = updateIntegrationRecord(record => ({ + ...record, + provenance: { + ...record.provenance, + entries: [{ + artifact: { kind: "generated-profile" }, + baseline: { kind: "absent" }, + postImage: null, + txId: "tx-same", + at, + }], + }, + })); + + expect(result.kind).toBe("updated"); + const saved = persistedRecord(); + const entries = (saved.provenance as { entries: Array> }).entries; + expect(entries).toHaveLength(1); + expect(entries[0]!.artifact).toEqual({ kind: "generated-profile" }); + expect(entries[0]!.baseline).toEqual({ kind: "absent" }); + expect(entries[0]).not.toHaveProperty("futureEntry"); + }); + test("fails closed on unparseable bytes without invoking the mutator or resetting the file", () => { mkdirSync(join(opencodexHome, "integrations"), { recursive: true }); writeFileSync(integrationRecordPath(), "{ definitely-not-json", "utf8");