From 12084c26d6625c86e02da1c961f95542bec47b42 Mon Sep 17 00:00:00 2001 From: luvs01 Date: Sun, 30 Aug 2026 09:04:26 +0900 Subject: [PATCH 1/6] fix(codex): bound the provenance ledger by bytes, not only by transaction count A `present` baseline embeds the artifact's exact bytes as base64, and the native Codex artifacts sit outside this proxy's trust boundary. The 16-transaction window bounds the ledger only while each transaction is small, so a single oversized `config.toml` was copied into integrations/codex.json in full and re-serialized on every append. Measured against current dev, three admitted transactions over a 64 MiB config produced a 576 MiB ledger write. Add a 1 MiB serialized ceiling alongside the existing window. Transactions are admitted newest-first and only whole, matching the existing rule that a partial transaction is worse than none. A transaction that cannot fit is skipped rather than ending the scan, so one pathological artifact cannot erase the smaller, still-diagnosable evidence around it. --- src/codex/inject-coordination.ts | 39 +++++++++++++++++++-- tests/codex-inject-write-lock.test.ts | 50 ++++++++++++++++++++++++++- 2 files changed, 85 insertions(+), 4 deletions(-) diff --git a/src/codex/inject-coordination.ts b/src/codex/inject-coordination.ts index c3ced10ba2..8feccf8028 100644 --- a/src/codex/inject-coordination.ts +++ b/src/codex/inject-coordination.ts @@ -196,6 +196,20 @@ export function captureCodexPreImages(): CodexPreImages { */ export const CODEX_PROVENANCE_MAX_TRANSACTIONS = 16; +/** + * How many serialized bytes of evidence the ledger keeps. + * + * The transaction window alone bounds the ledger only if each transaction is small, and a + * baseline embeds the artifact's exact bytes. Those artifacts sit outside this proxy's trust + * boundary: a native `config.toml` grown to hundreds of megabytes is copied into the record as + * base64 and re-serialized on every append, so a single oversized file turns a 16-transaction + * window into a multi-gigabyte write. + * + * 1 MiB holds a normal 16-transaction window many times over, so ordinary evidence is never + * touched and only the pathological case is refused. + */ +export const CODEX_PROVENANCE_MAX_BYTES = 1024 * 1024; + function provenanceBaseline(bytes: string | null): CodexProvenanceEntry["baseline"] { if (bytes === null) return { kind: "absent" }; return { @@ -214,21 +228,40 @@ function provenancePostImage(path: string): string | null { } /** - * Keep the newest `CODEX_PROVENANCE_MAX_TRANSACTIONS` transactions, whole. + * Keep the newest transactions that fit both the transaction window and the byte ceiling, whole. * * Trimming by ENTRY count would cut a transaction in half and leave evidence that says a * transaction touched two artifacts when it touched three — worse than dropping it outright, * because a partial record still reads as complete. Order is preserved; only whole leading * transactions are removed. + * + * The byte ceiling applies the same rule: a transaction that does not fit is omitted whole, + * including the newest one — a record silently truncated to fit would be read as a faithful + * pre-image. An oversized transaction is skipped rather than ending the scan, so one pathological + * artifact cannot erase the smaller transactions that still fit and are still diagnosable. */ export function boundProvenanceEntries( entries: readonly CodexProvenanceEntry[], maxTransactions = CODEX_PROVENANCE_MAX_TRANSACTIONS, + maxBytes = CODEX_PROVENANCE_MAX_BYTES, ): readonly CodexProvenanceEntry[] { const order: string[] = []; for (const entry of entries) if (!order.includes(entry.txId)) order.push(entry.txId); - if (order.length <= maxTransactions) return entries; - const keep = new Set(order.slice(order.length - maxTransactions)); + const windowed = order.length <= maxTransactions + ? order + : order.slice(order.length - maxTransactions); + // Newest first, so the transactions anyone diagnoses against are the ones that fit. + const keep = new Set(); + let bytes = 0; + for (let i = windowed.length - 1; i >= 0; i--) { + const txId = windowed[i]!; + const txEntries = entries.filter(entry => entry.txId === txId); + const txBytes = Buffer.byteLength(JSON.stringify(txEntries), "utf-8"); + if (bytes + txBytes > maxBytes) continue; + bytes += txBytes; + keep.add(txId); + } + if (keep.size === order.length) return entries; return entries.filter(entry => keep.has(entry.txId)); } diff --git a/tests/codex-inject-write-lock.test.ts b/tests/codex-inject-write-lock.test.ts index ed49cd5e10..fce75ddfa8 100644 --- a/tests/codex-inject-write-lock.test.ts +++ b/tests/codex-inject-write-lock.test.ts @@ -15,7 +15,11 @@ import { resolveCodexCoordinatorDatabasePath, resolveEffectiveUserIdentity, } from "../src/codex/user-identity"; -import { boundProvenanceEntries, STABLE_ZERO_BYTE_COORDINATOR_AGE_MS } from "../src/codex/inject-coordination"; +import { + boundProvenanceEntries, + CODEX_PROVENANCE_MAX_BYTES, + STABLE_ZERO_BYTE_COORDINATOR_AGE_MS, +} from "../src/codex/inject-coordination"; import { SPAWN_BUDGET_MS } from "./helpers/test-budget"; const repoRoot = join(import.meta.dir, ".."); @@ -508,4 +512,48 @@ describe("provenance ledger bound", () => { const entries = Array.from({ length: 16 }, (_, i) => transaction(`tx-${i}`)).flat(); expect(boundProvenanceEntries(entries, 16)).toBe(entries); }); + + test("an oversized baseline is omitted whole rather than amplified into the record", () => { + // The native artifacts sit outside this proxy's trust boundary, so a `config.toml` grown to + // an arbitrary size would otherwise be copied into the record as base64 and re-serialized on + // every append — the transaction window alone does not bound that. + const huge = (txId: string) => transaction(txId).map(entry => ({ + ...entry, + baseline: { + kind: "present" as const, + sha256: "0".repeat(64), + bytesBase64: "A".repeat(CODEX_PROVENANCE_MAX_BYTES), + }, + })); + const entries = [...transaction("tx-small"), ...huge("tx-huge")]; + + const bounded = boundProvenanceEntries(entries, 16); + + expect(bounded.map(e => e.txId)).toEqual(["tx-small", "tx-small", "tx-small"]); + expect(Buffer.byteLength(JSON.stringify(bounded), "utf-8")) + .toBeLessThanOrEqual(CODEX_PROVENANCE_MAX_BYTES); + }); + + test("the byte ceiling drops whole transactions, oldest first", () => { + const padded = (txId: string) => transaction(txId).map(entry => ({ + ...entry, + baseline: { + kind: "present" as const, + sha256: "0".repeat(64), + bytesBase64: "A".repeat(120 * 1024), + }, + })); + const entries = Array.from({ length: 6 }, (_, i) => padded(`tx-${i}`)).flat(); + + const bounded = boundProvenanceEntries(entries, 16); + const kept = [...new Set(bounded.map(e => e.txId))]; + + expect(kept.length).toBeGreaterThan(0); + expect(kept.length).toBeLessThan(6); + // Newest survive; the dropped ones are the oldest, and each survivor is whole. + expect(kept.at(-1)).toBe("tx-5"); + for (const txId of kept) expect(bounded.filter(e => e.txId === txId)).toHaveLength(3); + expect(Buffer.byteLength(JSON.stringify(bounded), "utf-8")) + .toBeLessThanOrEqual(CODEX_PROVENANCE_MAX_BYTES); + }); }); From 929de613dc0dbb53e6a4455fa319e6e017d02fbf Mon Sep 17 00:00:00 2001 From: luvs01 Date: Sun, 30 Aug 2026 10:16:12 +0900 Subject: [PATCH 2/6] fix(codex): keep provenance bounds intact through extension merge The initial byte ceiling was smaller than the ordinary 16-transaction window described by the same file, measured compact transaction arrays while the writer pretty-prints the record, and could serialize an oversized base64 payload merely to decide that it did not fit. Raise the ceiling to 4 MiB, preflight embedded pre-image length before serialization, and measure the exact pretty-printed provenance wrapper including its final newline. Group transactions once and preserve the original array instance when the ordinary window fits. Also fence integration-record extension preservation by transaction identity. When bounding omitted a transaction, the positional fallback could attach that transaction's unknown extensions to the new entry that shifted into its slot, falsifying provenance and regrowing the final record past the ceiling. --- src/codex/inject-coordination.ts | 63 ++++++++++++++++++---- src/codex/integration-record.ts | 12 ++++- tests/codex-inject-write-lock.test.ts | 75 ++++++++++++++++++++++---- tests/codex-integration-record.test.ts | 51 ++++++++++++++++++ 4 files changed, 181 insertions(+), 20 deletions(-) diff --git a/src/codex/inject-coordination.ts b/src/codex/inject-coordination.ts index 8feccf8028..dd6e99f7ee 100644 --- a/src/codex/inject-coordination.ts +++ b/src/codex/inject-coordination.ts @@ -205,10 +205,28 @@ export const CODEX_PROVENANCE_MAX_TRANSACTIONS = 16; * base64 and re-serialized on every append, so a single oversized file turns a 16-transaction * window into a multi-gigabyte write. * - * 1 MiB holds a normal 16-transaction window many times over, so ordinary evidence is never - * touched and only the pathological case is refused. + * The size is derived from the window above rather than picked: the 25 KB `config.toml` that + * comment describes measures 100.8 KiB per transaction, so a full 16-transaction window is + * 1.58 MiB. 4 MiB leaves that ordinary window untouched with room to spare while still refusing + * the pathological case, and a regression test pins the ordinary window so the two cannot drift + * apart silently. */ -export const CODEX_PROVENANCE_MAX_BYTES = 1024 * 1024; +export const CODEX_PROVENANCE_MAX_BYTES = 4 * 1024 * 1024; + +/** + * Measure the representation the integration-record writer actually emits. + * + * The minimal wrapper reproduces the final indentation depth of `provenance.entries`, including + * structured unknown extension fields preserved from older records. Its fixed wrapper makes this + * slightly conservative as a ledger-only ceiling, which is preferable to admitting an entry that + * expands past the limit when `JSON.stringify(record, null, 2)` writes it. + */ +function serializedProvenanceBytes(entries: readonly CodexProvenanceEntry[]): number { + return Buffer.byteLength(`${JSON.stringify({ + version: 1, + provenance: { entries }, + }, null, 2)}\n`, "utf-8"); +} function provenanceBaseline(bytes: string | null): CodexProvenanceEntry["baseline"] { if (bytes === null) return { kind: "absent" }; @@ -245,20 +263,45 @@ export function boundProvenanceEntries( maxTransactions = CODEX_PROVENANCE_MAX_TRANSACTIONS, maxBytes = CODEX_PROVENANCE_MAX_BYTES, ): readonly CodexProvenanceEntry[] { - const order: string[] = []; - for (const entry of entries) if (!order.includes(entry.txId)) order.push(entry.txId); + const transactions = new Map(); + for (const entry of entries) { + const transaction = transactions.get(entry.txId); + if (transaction) transaction.push(entry); + else transactions.set(entry.txId, [entry]); + } + const order = [...transactions.keys()]; const windowed = order.length <= maxTransactions ? order : order.slice(order.length - maxTransactions); + if (windowed.length === order.length) { + const baselineBytes = entries.reduce( + (total, entry) => total + (entry.baseline.kind === "present" ? entry.baseline.bytesBase64.length : 0), + 0, + ); + if (baselineBytes <= maxBytes && serializedProvenanceBytes(entries) <= maxBytes) return entries; + } // Newest first, so the transactions anyone diagnoses against are the ones that fit. const keep = new Set(); - let bytes = 0; + let baselineBytes = 0; for (let i = windowed.length - 1; i >= 0; i--) { const txId = windowed[i]!; - const txEntries = entries.filter(entry => entry.txId === txId); - const txBytes = Buffer.byteLength(JSON.stringify(txEntries), "utf-8"); - if (bytes + txBytes > maxBytes) continue; - bytes += txBytes; + const txEntries = transactions.get(txId)!; + // Measure the embedded pre-images first: a pathological baseline is refused without + // serializing it, so the ceiling does not itself allocate the payload it exists to reject. + const transactionBaselineBytes = txEntries.reduce( + (total, entry) => total + (entry.baseline.kind === "present" ? entry.baseline.bytesBase64.length : 0), + 0, + ); + if (baselineBytes + transactionBaselineBytes > maxBytes) continue; + const candidateKeep = new Set(keep); + candidateKeep.add(txId); + // Reuse the already-grouped transactions: this avoids another full-ledger filter on each + // iteration while preserving the original transaction and entry order. + const candidateEntries = windowed.flatMap(id => + candidateKeep.has(id) ? transactions.get(id)! : [] + ); + if (serializedProvenanceBytes(candidateEntries) > maxBytes) continue; + baselineBytes += transactionBaselineBytes; keep.add(txId); } if (keep.size === order.length) return entries; diff --git a/src/codex/integration-record.ts b/src/codex/integration-record.ts index e595d81a58..a843588e0a 100644 --- a/src/codex/integration-record.ts +++ b/src/codex/integration-record.ts @@ -209,7 +209,17 @@ function mergeLedger(previous: CodexProvenanceLedger, next: CodexProvenanceLedge && candidate.txId === entry.txId && candidate.at === entry.at && knownArtifactIdentity(candidate.artifact) === knownArtifactIdentity(entry.artifact)); - if (previousIndex < 0 && unused.has(nextIndex)) previousIndex = nextIndex; + if (previousIndex < 0 && unused.has(nextIndex)) { + const positional = previous.entries[nextIndex]!; + // Position alone is not provenance identity: a bounded ledger may drop an oversized + // transaction, shifting a newly appended transaction into its slot. Preserve extensions + // positionally only within the same transaction; otherwise an omitted transaction's + // unknown evidence would be falsely attached to the replacement and could regrow the + // record past the byte ceiling. + if (positional.txId === entry.txId && positional.at === entry.at) { + previousIndex = nextIndex; + } + } if (previousIndex < 0) return entry; unused.delete(previousIndex); return mergeEntry(previous.entries[previousIndex]!, entry); diff --git a/tests/codex-inject-write-lock.test.ts b/tests/codex-inject-write-lock.test.ts index fce75ddfa8..21e8e453f2 100644 --- a/tests/codex-inject-write-lock.test.ts +++ b/tests/codex-inject-write-lock.test.ts @@ -18,6 +18,7 @@ import { import { boundProvenanceEntries, CODEX_PROVENANCE_MAX_BYTES, + CODEX_PROVENANCE_MAX_TRANSACTIONS, STABLE_ZERO_BYTE_COORDINATOR_AGE_MS, } from "../src/codex/inject-coordination"; import { SPAWN_BUDGET_MS } from "./helpers/test-budget"; @@ -517,15 +518,23 @@ describe("provenance ledger bound", () => { // The native artifacts sit outside this proxy's trust boundary, so a `config.toml` grown to // an arbitrary size would otherwise be copied into the record as base64 and re-serialized on // every append — the transaction window alone does not bound that. - const huge = (txId: string) => transaction(txId).map(entry => ({ - ...entry, + const huge = transaction("tx-huge"); + huge[0] = { + ...huge[0]!, baseline: { kind: "present" as const, sha256: "0".repeat(64), - bytesBase64: "A".repeat(CODEX_PROVENANCE_MAX_BYTES), + bytesBase64: "A".repeat(CODEX_PROVENANCE_MAX_BYTES + 1), }, - })); - const entries = [...transaction("tx-small"), ...huge("tx-huge")]; + }; + // The baseline-size prefilter must reject this transaction before JSON.stringify reaches + // the tripwire. Only one sibling is oversized; all three must still be omitted together. + Object.defineProperty(huge[0]!, "toJSON", { + value: () => { + throw new Error("oversized transaction was serialized"); + }, + }); + const entries = [...transaction("tx-small"), ...huge]; const bounded = boundProvenanceEntries(entries, 16); @@ -540,20 +549,68 @@ describe("provenance ledger bound", () => { baseline: { kind: "present" as const, sha256: "0".repeat(64), - bytesBase64: "A".repeat(120 * 1024), + bytesBase64: "A".repeat(Math.floor(CODEX_PROVENANCE_MAX_BYTES / 4)), }, })); - const entries = Array.from({ length: 6 }, (_, i) => padded(`tx-${i}`)).flat(); + const entries = Array.from({ length: 4 }, (_, i) => padded(`tx-${i}`)).flat(); const bounded = boundProvenanceEntries(entries, 16); const kept = [...new Set(bounded.map(e => e.txId))]; expect(kept.length).toBeGreaterThan(0); - expect(kept.length).toBeLessThan(6); + expect(kept.length).toBeLessThan(4); // Newest survive; the dropped ones are the oldest, and each survivor is whole. - expect(kept.at(-1)).toBe("tx-5"); + expect(kept.at(-1)).toBe("tx-3"); for (const txId of kept) expect(bounded.filter(e => e.txId === txId)).toHaveLength(3); expect(Buffer.byteLength(JSON.stringify(bounded), "utf-8")) .toBeLessThanOrEqual(CODEX_PROVENANCE_MAX_BYTES); }); + + test("the ceiling measures structured extensions in the pretty-printed record shape", () => { + const extended = (txId: string) => transaction(txId).map((entry, index) => ({ + ...entry, + // Integration records preserve unknown structured fields. Compact JSON can fit while the + // writer's two-space indentation does not, so the bound must use the actual write shape. + extension: index === 0 + ? { rows: Array.from({ length: 400 }, () => ({ left: "x", right: "y" })) } + : undefined, + })); + const entries = [...extended("tx-old"), ...extended("tx-new")]; + const oneTransactionBytes = Buffer.byteLength(`${JSON.stringify({ + version: 1, + provenance: { entries: extended("tx-new") }, + }, null, 2)}\n`, "utf-8"); + const bothCompactBytes = Buffer.byteLength(JSON.stringify(entries), "utf-8"); + const bothPrettyBytes = Buffer.byteLength(`${JSON.stringify({ + version: 1, + provenance: { entries }, + }, null, 2)}\n`, "utf-8"); + const maxBytes = Math.max(oneTransactionBytes, bothCompactBytes); + + expect(bothPrettyBytes).toBeGreaterThan(maxBytes); + const bounded = boundProvenanceEntries(entries, 16, maxBytes); + + expect([...new Set(bounded.map(entry => entry.txId))]).toEqual(["tx-new"]); + }); + + test("a full window of ordinary transactions is still kept whole", () => { + // The ceiling exists to refuse pathological artifacts, not to shrink the window above it. + // This pins the two together: the 25 KB `config.toml` the window comment describes measures + // about 100 KiB per transaction, so a full window is roughly 1.6 MiB and must survive intact. + const ordinary = Buffer.from("x".repeat(25 * 1024)).toString("base64"); + const entries = Array.from( + { length: CODEX_PROVENANCE_MAX_TRANSACTIONS }, + (_, i) => transaction(`tx-${i}`).map(entry => ({ + ...entry, + baseline: { kind: "present" as const, sha256: "0".repeat(64), bytesBase64: ordinary }, + postImage: "0".repeat(64), + })), + ).flat(); + + const bounded = boundProvenanceEntries(entries); + + expect(bounded).toBe(entries); + expect(new Set(bounded.map(e => e.txId)).size).toBe(CODEX_PROVENANCE_MAX_TRANSACTIONS); + expect(bounded).toHaveLength(entries.length); + }); }); diff --git a/tests/codex-integration-record.test.ts b/tests/codex-integration-record.test.ts index 57ad5f9704..9dae3b2f24 100644 --- a/tests/codex-integration-record.test.ts +++ b/tests/codex-integration-record.test.ts @@ -6,6 +6,7 @@ import { readIntegrationRecord, updateIntegrationRecord, } from "../src/codex/integration-record"; +import { boundProvenanceEntries } from "../src/codex/inject-coordination"; import type { CodexArtifactId, CodexIntegrationRecord, @@ -145,6 +146,56 @@ describe("Codex integration record", () => { ); }); + test("does not move an omitted transaction's extensions onto its positional replacement", () => { + const maxBytes = 16 * 1024; + const entry = ( + txId: string, + at: string, + baseline: CodexProvenanceEntry["baseline"], + futureEntry?: unknown, + ): CodexProvenanceEntry => ({ + artifact: { kind: "config" }, + baseline, + postImage: null, + txId, + at, + ...(futureEntry === undefined ? {} : { futureEntry }), + }); + const kept = entry("tx-kept", "2026-08-04T00:00:00.000Z", { kind: "absent" }); + const oversized = entry( + "tx-oversized", + "2026-08-04T00:00:01.000Z", + { + kind: "present", + sha256: "0".repeat(64), + bytesBase64: "A".repeat(maxBytes + 1), + }, + { mustNotMove: "x".repeat(maxBytes * 2) }, + ); + writeRecord({ version: 1, provenance: { entries: [kept, oversized] } }); + + const appended = entry("tx-new", "2026-08-04T00:00:02.000Z", { kind: "absent" }); + const result = updateIntegrationRecord(record => ({ + ...record, + provenance: { + ...record.provenance, + entries: boundProvenanceEntries( + [...record.provenance!.entries, appended], + 16, + maxBytes, + ), + }, + })); + + expect(result.kind).toBe("updated"); + const saved = persistedRecord(); + const entries = (saved.provenance as { entries: Array> }).entries; + expect(entries.map(savedEntry => savedEntry.txId)).toEqual(["tx-kept", "tx-new"]); + expect(entries[1]).not.toHaveProperty("futureEntry"); + expect(Buffer.byteLength(JSON.stringify(saved, null, 2), "utf-8")) + .toBeLessThanOrEqual(maxBytes); + }); + test("fails closed on unparseable bytes without invoking the mutator or resetting the file", () => { mkdirSync(join(opencodexHome, "integrations"), { recursive: true }); writeFileSync(integrationRecordPath(), "{ definitely-not-json", "utf8"); From 72ccc2e20cc067e1e03a5fe39c74210bfb6cf0cf Mon Sep 17 00:00:00 2001 From: luvs01 Date: Sun, 30 Aug 2026 10:29:21 +0900 Subject: [PATCH 3/6] fix(codex): charge ledger extensions to the provenance byte budget The writer preserves forward-compatible fields directly under `provenance`, but the byte bound measured only a synthetic ledger containing `entries`. A large unknown ledger extension therefore consumed no budget and could keep every subsequent append above the ceiling. Let the bound receive the existing ledger template and replace only its entries while measuring. The production append passes that template explicitly, so ledger-, entry-, artifact-, and baseline-level extensions are all charged in the same pretty-printed shape that will be written. --- src/codex/inject-coordination.ts | 38 +++++++++++++++++++-------- tests/codex-inject-write-lock.test.ts | 23 ++++++++++++++++ 2 files changed, 50 insertions(+), 11 deletions(-) diff --git a/src/codex/inject-coordination.ts b/src/codex/inject-coordination.ts index dd6e99f7ee..2d5549bb07 100644 --- a/src/codex/inject-coordination.ts +++ b/src/codex/inject-coordination.ts @@ -16,6 +16,7 @@ import { CODEX_CONFIG_PATH, CODEX_PROFILE_PATH } from "./paths"; import type { CodexArtifactId, CodexProvenanceEntry, + CodexProvenanceLedger, } from "./convergence-types"; import { codexWriteCoordination, @@ -221,10 +222,13 @@ export const CODEX_PROVENANCE_MAX_BYTES = 4 * 1024 * 1024; * slightly conservative as a ledger-only ceiling, which is preferable to admitting an entry that * expands past the limit when `JSON.stringify(record, null, 2)` writes it. */ -function serializedProvenanceBytes(entries: readonly CodexProvenanceEntry[]): number { +function serializedProvenanceBytes( + entries: readonly CodexProvenanceEntry[], + ledger: CodexProvenanceLedger | undefined, +): number { return Buffer.byteLength(`${JSON.stringify({ version: 1, - provenance: { entries }, + provenance: { ...ledger, entries }, }, null, 2)}\n`, "utf-8"); } @@ -262,6 +266,7 @@ export function boundProvenanceEntries( entries: readonly CodexProvenanceEntry[], maxTransactions = CODEX_PROVENANCE_MAX_TRANSACTIONS, maxBytes = CODEX_PROVENANCE_MAX_BYTES, + ledger?: CodexProvenanceLedger, ): readonly CodexProvenanceEntry[] { const transactions = new Map(); for (const entry of entries) { @@ -278,7 +283,10 @@ export function boundProvenanceEntries( (total, entry) => total + (entry.baseline.kind === "present" ? entry.baseline.bytesBase64.length : 0), 0, ); - if (baselineBytes <= maxBytes && serializedProvenanceBytes(entries) <= maxBytes) return entries; + if ( + baselineBytes <= maxBytes + && serializedProvenanceBytes(entries, ledger) <= maxBytes + ) return entries; } // Newest first, so the transactions anyone diagnoses against are the ones that fit. const keep = new Set(); @@ -300,7 +308,7 @@ export function boundProvenanceEntries( const candidateEntries = windowed.flatMap(id => candidateKeep.has(id) ? transactions.get(id)! : [] ); - if (serializedProvenanceBytes(candidateEntries) > maxBytes) continue; + if (serializedProvenanceBytes(candidateEntries, ledger) > maxBytes) continue; baselineBytes += transactionBaselineBytes; keep.add(txId); } @@ -326,13 +334,21 @@ export function recordCodexNativeTransactionProvenance( txId, at, })); - return updateIntegrationRecord(record => ({ - ...record, - provenance: { - ...record.provenance, - entries: boundProvenanceEntries([...(record.provenance?.entries ?? []), ...entries]), - }, - })); + return updateIntegrationRecord(record => { + const previousLedger = record.provenance; + return { + ...record, + provenance: { + ...previousLedger, + entries: boundProvenanceEntries( + [...(previousLedger?.entries ?? []), ...entries], + CODEX_PROVENANCE_MAX_TRANSACTIONS, + CODEX_PROVENANCE_MAX_BYTES, + previousLedger, + ), + }, + }; + }); } /** diff --git a/tests/codex-inject-write-lock.test.ts b/tests/codex-inject-write-lock.test.ts index 21e8e453f2..3d27785c5a 100644 --- a/tests/codex-inject-write-lock.test.ts +++ b/tests/codex-inject-write-lock.test.ts @@ -593,6 +593,29 @@ describe("provenance ledger bound", () => { expect([...new Set(bounded.map(entry => entry.txId))]).toEqual(["tx-new"]); }); + test("unknown ledger extensions consume the same byte budget as entries", () => { + const entries = [...transaction("tx-old"), ...transaction("tx-new")]; + const ledger = { + entries, + futureLedger: { + rows: Array.from({ length: 200 }, () => ({ left: "x", right: "y" })), + }, + }; + const writeBytes = (candidate: readonly (typeof entries)[number][]) => + Buffer.byteLength(`${JSON.stringify({ + version: 1, + provenance: { ...ledger, entries: candidate }, + }, null, 2)}\n`, "utf-8"); + const newest = transaction("tx-new"); + const maxBytes = writeBytes(newest); + + expect(writeBytes(entries)).toBeGreaterThan(maxBytes); + const bounded = boundProvenanceEntries(entries, 16, maxBytes, ledger); + + expect([...new Set(bounded.map(entry => entry.txId))]).toEqual(["tx-new"]); + expect(writeBytes(bounded)).toBeLessThanOrEqual(maxBytes); + }); + test("a full window of ordinary transactions is still kept whole", () => { // The ceiling exists to refuse pathological artifacts, not to shrink the window above it. // This pins the two together: the 25 KB `config.toml` the window comment describes measures From ebc02011201c84d654e2aeb629e94d303339be07 Mon Sep 17 00:00:00 2001 From: luvs01 Date: Sun, 30 Aug 2026 10:41:59 +0900 Subject: [PATCH 4/6] fix(codex): refuse irreducibly oversized provenance writes If forward-compatible ledger fields alone exceed the byte ceiling, no entry selection can make the final write compliant. Returning an empty entry set would delete every known receipt while extension preservation still rewrote the same oversized file. Detect that fixed overhead before trimming and return the exact existing record. `updateIntegrationRecord` now treats same-object return as an explicit no-op, skipping validation, extension merge, directory creation, and atomic rewrite. The production regression proves the append returns safely while the record's entry count and exact bytes remain unchanged. --- src/codex/inject-coordination.ts | 8 +++++ src/codex/integration-record.ts | 5 +++ tests/codex-inject-write-lock.test.ts | 49 +++++++++++++++++++++++++++ 3 files changed, 62 insertions(+) diff --git a/src/codex/inject-coordination.ts b/src/codex/inject-coordination.ts index 2d5549bb07..803f062012 100644 --- a/src/codex/inject-coordination.ts +++ b/src/codex/inject-coordination.ts @@ -336,6 +336,14 @@ export function recordCodexNativeTransactionProvenance( })); return updateIntegrationRecord(record => { const previousLedger = record.provenance; + // Unknown ledger extensions are forward-compatible and must be preserved. If those fixed + // fields alone exceed the ceiling, no entry selection can make the write compliant. Keep the + // existing record byte-for-byte instead of deleting all known evidence and rewriting the + // same oversized extension on every native transaction. + if ( + previousLedger + && serializedProvenanceBytes([], previousLedger) > CODEX_PROVENANCE_MAX_BYTES + ) return record; return { ...record, provenance: { diff --git a/src/codex/integration-record.ts b/src/codex/integration-record.ts index a843588e0a..2eeab6dfe3 100644 --- a/src/codex/integration-record.ts +++ b/src/codex/integration-record.ts @@ -255,6 +255,11 @@ export const updateIntegrationRecord = ( if (read.kind === "invalid") return read; const previous: CodexIntegrationRecord = read.kind === "ready" ? read.record : { version: 1 }; const proposed = mutate(previous); + // The mutator can explicitly decline an update by returning the exact record it received. + // This matters when a preserved forward-compatible extension already exceeds a caller's + // write budget: rewriting the same oversized bytes would amplify I/O while deleting known + // entries cannot make that irreducible overhead fit. + if (proposed === previous) return { kind: "updated" as const, record: previous }; if (!validateRecord(proposed)) { return { kind: "invalid", message: "Codex integration record update produced a malformed v1 shape" }; } diff --git a/tests/codex-inject-write-lock.test.ts b/tests/codex-inject-write-lock.test.ts index 3d27785c5a..01c7c67e26 100644 --- a/tests/codex-inject-write-lock.test.ts +++ b/tests/codex-inject-write-lock.test.ts @@ -251,6 +251,55 @@ describe("the lock is on the production path", () => { .toBe("{ malformed"); }); + test("irreducible ledger extension overhead refuses the append without rewriting", () => { + seedNative(); + const recordPath = join(opencodexHome, "integrations", "codex.json"); + mkdirSync(join(opencodexHome, "integrations"), { recursive: true }); + writeFileSync(recordPath, JSON.stringify({ + version: 1, + provenance: { + futureLedger: "x".repeat(CODEX_PROVENANCE_MAX_BYTES + 1), + entries: [{ + artifact: { kind: "config" }, + baseline: { kind: "absent" }, + postImage: null, + txId: "tx-existing", + at: "2026-08-30T00:00:00.000Z", + }], + }, + })); + const before = readFileSync(recordPath, "utf8"); + + const result = parseChildJson<{ kind?: string; entryCount?: number }>( + runChild(["--eval", ` + const { + captureCodexPreImages, + recordCodexNativeTransactionProvenance, + } = require("./src/codex/inject-coordination"); + const result = recordCodexNativeTransactionProvenance( + captureCodexPreImages(), + "tx-must-not-append", + ); + console.log(JSON.stringify({ + kind: result.kind, + entryCount: result.kind === "updated" + ? result.record.provenance?.entries?.length + : undefined, + })); + `], { + ...process.env, + CODEX_HOME: codexHome, + OPENCODEX_HOME: opencodexHome, + }), + "irreducible ledger extension overhead", + ); + + expect(result.kind).toBe("updated"); + expect(result.entryCount).toBe(1); + // Exact bytes, not just semantics: a no-op must not pretty-print/rewrite the oversized file. + expect(readFileSync(recordPath, "utf8")).toBe(before); + }); + /** * The contention proof. A real second process holds N through the production * lock module while a real injection runs; the injection must report busy and From ebc5268d92612f223dda7c19b3d333f4c7d01bf8 Mon Sep 17 00:00:00 2001 From: luvs01 Date: Sun, 30 Aug 2026 11:15:29 +0900 Subject: [PATCH 5/6] fix(codex): backfill the bounded provenance window --- src/codex/inject-coordination.ts | 15 ++++++--------- tests/codex-inject-write-lock.test.ts | 23 +++++++++++++++++++++++ 2 files changed, 29 insertions(+), 9 deletions(-) diff --git a/src/codex/inject-coordination.ts b/src/codex/inject-coordination.ts index 803f062012..e91e9f339b 100644 --- a/src/codex/inject-coordination.ts +++ b/src/codex/inject-coordination.ts @@ -254,8 +254,8 @@ function provenancePostImage(path: string): string | null { * * Trimming by ENTRY count would cut a transaction in half and leave evidence that says a * transaction touched two artifacts when it touched three — worse than dropping it outright, - * because a partial record still reads as complete. Order is preserved; only whole leading - * transactions are removed. + * because a partial record still reads as complete. Order is preserved; only whole transactions + * are removed. * * The byte ceiling applies the same rule: a transaction that does not fit is omitted whole, * including the newest one — a record silently truncated to fit would be read as a faithful @@ -275,10 +275,7 @@ export function boundProvenanceEntries( else transactions.set(entry.txId, [entry]); } const order = [...transactions.keys()]; - const windowed = order.length <= maxTransactions - ? order - : order.slice(order.length - maxTransactions); - if (windowed.length === order.length) { + if (order.length <= maxTransactions) { const baselineBytes = entries.reduce( (total, entry) => total + (entry.baseline.kind === "present" ? entry.baseline.bytesBase64.length : 0), 0, @@ -291,8 +288,8 @@ export function boundProvenanceEntries( // Newest first, so the transactions anyone diagnoses against are the ones that fit. const keep = new Set(); let baselineBytes = 0; - for (let i = windowed.length - 1; i >= 0; i--) { - const txId = windowed[i]!; + for (let i = order.length - 1; i >= 0 && keep.size < maxTransactions; i--) { + const txId = order[i]!; const txEntries = transactions.get(txId)!; // Measure the embedded pre-images first: a pathological baseline is refused without // serializing it, so the ceiling does not itself allocate the payload it exists to reject. @@ -305,7 +302,7 @@ export function boundProvenanceEntries( candidateKeep.add(txId); // Reuse the already-grouped transactions: this avoids another full-ledger filter on each // iteration while preserving the original transaction and entry order. - const candidateEntries = windowed.flatMap(id => + const candidateEntries = order.flatMap(id => candidateKeep.has(id) ? transactions.get(id)! : [] ); if (serializedProvenanceBytes(candidateEntries, ledger) > maxBytes) continue; diff --git a/tests/codex-inject-write-lock.test.ts b/tests/codex-inject-write-lock.test.ts index 01c7c67e26..ea544842fc 100644 --- a/tests/codex-inject-write-lock.test.ts +++ b/tests/codex-inject-write-lock.test.ts @@ -592,6 +592,29 @@ describe("provenance ledger bound", () => { .toBeLessThanOrEqual(CODEX_PROVENANCE_MAX_BYTES); }); + test("backfills the transaction window after an oversized newest transaction is skipped", () => { + const maxBytes = 64 * 1024; + const oversized = transaction("tx-16"); + oversized[0] = { + ...oversized[0]!, + baseline: { + kind: "present" as const, + sha256: "0".repeat(64), + bytesBase64: "A".repeat(maxBytes + 1), + }, + }; + const entries = [ + ...Array.from({ length: 16 }, (_, i) => transaction(`tx-${i}`)).flat(), + ...oversized, + ]; + + const bounded = boundProvenanceEntries(entries, 16, maxBytes); + const kept = [...new Set(bounded.map(entry => entry.txId))]; + + expect(kept).toEqual(Array.from({ length: 16 }, (_, i) => `tx-${i}`)); + for (const txId of kept) expect(bounded.filter(entry => entry.txId === txId)).toHaveLength(3); + }); + test("the byte ceiling drops whole transactions, oldest first", () => { const padded = (txId: string) => transaction(txId).map(entry => ({ ...entry, From 32db37e69443e0d96747efe165340fd76a91223e Mon Sep 17 00:00:00 2001 From: luvs01 Date: Sun, 30 Aug 2026 13:47:50 +0900 Subject: [PATCH 6/6] fix(codex): bind provenance extension preservation to artifact identity --- src/codex/integration-record.ts | 19 +++++-------- tests/codex-integration-record.test.ts | 39 ++++++++++++++++++++++++++ 2 files changed, 46 insertions(+), 12 deletions(-) diff --git a/src/codex/integration-record.ts b/src/codex/integration-record.ts index 2eeab6dfe3..177d4dd1a4 100644 --- a/src/codex/integration-record.ts +++ b/src/codex/integration-record.ts @@ -204,22 +204,17 @@ function mergeEntry(previous: CodexProvenanceEntry, next: CodexProvenanceEntry): function mergeLedger(previous: CodexProvenanceLedger, next: CodexProvenanceLedger): CodexProvenanceLedger { const unused = new Set(previous.entries.map((_, index) => index)); const entries = next.entries.map((entry, nextIndex) => { - let previousIndex = previous.entries.findIndex((candidate, index) => + // Extensions follow provenance identity, never position. A bounded ledger may drop an + // oversized transaction, shifting an unrelated entry into that slot, and even within one + // transaction and timestamp the artifact may differ. Since this search already requires + // txId, timestamp, and artifact identity, any remaining positional match is by definition + // a different artifact, so preserving its unknown entry/artifact/baseline fields would + // attach false evidence and could regrow the record past the measured byte ceiling. + const previousIndex = previous.entries.findIndex((candidate, index) => unused.has(index) && candidate.txId === entry.txId && candidate.at === entry.at && knownArtifactIdentity(candidate.artifact) === knownArtifactIdentity(entry.artifact)); - if (previousIndex < 0 && unused.has(nextIndex)) { - const positional = previous.entries[nextIndex]!; - // Position alone is not provenance identity: a bounded ledger may drop an oversized - // transaction, shifting a newly appended transaction into its slot. Preserve extensions - // positionally only within the same transaction; otherwise an omitted transaction's - // unknown evidence would be falsely attached to the replacement and could regrow the - // record past the byte ceiling. - if (positional.txId === entry.txId && positional.at === entry.at) { - previousIndex = nextIndex; - } - } if (previousIndex < 0) return entry; unused.delete(previousIndex); return mergeEntry(previous.entries[previousIndex]!, entry); diff --git a/tests/codex-integration-record.test.ts b/tests/codex-integration-record.test.ts index 9dae3b2f24..e0315f1522 100644 --- a/tests/codex-integration-record.test.ts +++ b/tests/codex-integration-record.test.ts @@ -196,6 +196,45 @@ describe("Codex integration record", () => { .toBeLessThanOrEqual(maxBytes); }); + test("does not move extensions onto a different artifact in the same transaction", () => { + const at = "2026-08-04T00:00:00.000Z"; + writeRecord({ + version: 1, + provenance: { + entries: [{ + artifact: { kind: "config", futureArtifact: { owner: "config" } }, + baseline: { kind: "absent", futureBaseline: { owner: "config" } }, + postImage: null, + txId: "tx-same", + at, + futureEntry: { owner: "config" }, + } as unknown as CodexProvenanceEntry], + }, + }); + + const result = updateIntegrationRecord(record => ({ + ...record, + provenance: { + ...record.provenance, + entries: [{ + artifact: { kind: "generated-profile" }, + baseline: { kind: "absent" }, + postImage: null, + txId: "tx-same", + at, + }], + }, + })); + + expect(result.kind).toBe("updated"); + const saved = persistedRecord(); + const entries = (saved.provenance as { entries: Array> }).entries; + expect(entries).toHaveLength(1); + expect(entries[0]!.artifact).toEqual({ kind: "generated-profile" }); + expect(entries[0]!.baseline).toEqual({ kind: "absent" }); + expect(entries[0]).not.toHaveProperty("futureEntry"); + }); + test("fails closed on unparseable bytes without invoking the mutator or resetting the file", () => { mkdirSync(join(opencodexHome, "integrations"), { recursive: true }); writeFileSync(integrationRecordPath(), "{ definitely-not-json", "utf8");