fix(provenance): preserve mixed files and repeated durable bindings

Author: Vikhyath Mondreti

apps/sim/content/blog/secret-provenance/index.mdx

@@ -21,7 +21,7 @@ faq:
   - q: "Does Sim use regular expressions to detect secrets in logs?"
     a: "No. Sim resolved the secret itself, so it can compare content against the exact values activated in that run. Pattern matching is useful when a scanner does not know which secrets exist; the execution runtime has stronger context."
   - q: "What happens if Sim cannot determine whether content contains a secret?"
-    a: "The boundary fails closed. The payload is withheld, while non-sensitive structure such as timing, block identity, and status remains available for debugging."
+    a: "Incomplete provenance at a run's own egress boundaries withholds the payload while preserving safe diagnostic structure. Durable reads follow the surface's enforcement policy: legacy content remains readable, and unknown stored provenance can be observed before enforcement is enabled."
   - q: "Are secrets tracked across files and tables, or only within one execution?"
     a: "Across both. Durable content such as workspace files, table cells, knowledge documents, and agent memory carries encrypted provenance so a later run does not mistake secret-bearing data for clean data."
   - q: "Is the model treated as an internal component or an egress boundary?"

apps/sim/lib/execution/durable-secret-provenance.test.ts

@@ -19,7 +19,13 @@ import {
   filterDurableSecretProvenanceBySourceValues,
   hashDurableSecretProvenanceValue,
   importDurableSecretProvenance,
+  mergeDurableSecretProvenance,
+  normalizeDurableSecretProvenanceEntries,
 } from '@/lib/execution/durable-secret-provenance'
+import {
+  PROVENANCE_MAX_ENTRIES,
+  PROVENANCE_MAX_SERIALIZED_BYTES,
+} from '@/lib/execution/provenance-limits'
 import { ResolvedSecretTraceRegistry } from '@/executor/utils/resolved-secret-trace-registry'
 
 function privateBundle(scope?: { userId: string; workspaceId?: string }) {
@@ -41,6 +47,17 @@ function privateBundle(scope?: { userId: string; workspaceId?: string }) {
 }
 
 describe('durable secret provenance hashing', () => {
+  it('hashes many small values within the byte budget without a separate node cutoff', () => {
+    const messages = Array.from({ length: 17_000 }, (_, index) => ({
+      role: 'user',
+      content: `message ${index}`,
+    }))
+    const hash = hashDurableSecretProvenanceValue(messages)
+    expect(hash).toMatch(/^[a-f0-9]{64}$/)
+    expect(hashDurableSecretProvenanceValue(structuredClone(messages))).toBe(hash)
+    expect(hashDurableSecretProvenanceValue('x'.repeat(16 * 1024 * 1024))).toBeUndefined()
+  })
+
   it('hashes equivalent plain JSON deterministically without key-order sensitivity', () => {
     expect(hashDurableSecretProvenanceValue({ b: [true, null], a: 'value' })).toBe(
       hashDurableSecretProvenanceValue({ a: 'value', b: [true, null] })
@@ -83,6 +100,101 @@ describe('durable secret provenance hashing', () => {
   })
 })
 
+describe('durable provenance binding capacity', () => {
+  it('folds duplicates while preserving more than 10,000 bindings of one secret', () => {
+    const entries = Array.from({ length: PROVENANCE_MAX_ENTRIES + 1 }, (_, index) => ({
+      encryptedValue: 'ciphertext',
+      sourceValueHash: `hash-${index}`,
+    }))
+    const normalized = normalizeDurableSecretProvenanceEntries(entries)
+    expect(normalized).toHaveLength(entries.length)
+    expect(
+      mergeDurableSecretProvenance({ status: 'exact', entries }, { status: 'exact', entries })
+    ).toEqual({ status: 'exact', entries: normalized })
+    expect(normalizeDurableSecretProvenanceEntries(Array(entries.length).fill(entries[0]))).toEqual(
+      [entries[0]]
+    )
+  })
+
+  it('still refuses more than 10,000 distinct secrets', () => {
+    const entries = Array.from({ length: PROVENANCE_MAX_ENTRIES + 1 }, (_, index) => ({
+      encryptedValue: `ciphertext-${index}`,
+    }))
+    expect(normalizeDurableSecretProvenanceEntries(entries.slice(0, -1))).toHaveLength(
+      PROVENANCE_MAX_ENTRIES
+    )
+    expect(normalizeDurableSecretProvenanceEntries(entries)).toBeUndefined()
+  })
+
+  it('measures escaped UTF-8 JSON bytes including array separators', () => {
+    const entry = { encryptedValue: 'ciphertext', name: 'é\n' }
+    const overhead = Buffer.byteLength(JSON.stringify([entry]), 'utf8')
+    const exact = {
+      ...entry,
+      encryptedValue: entry.encryptedValue + 'x'.repeat(PROVENANCE_MAX_SERIALIZED_BYTES - overhead),
+    }
+    expect(normalizeDurableSecretProvenanceEntries([exact])).toEqual([exact])
+    expect(
+      normalizeDurableSecretProvenanceEntries([
+        { ...exact, encryptedValue: `${exact.encryptedValue}x` },
+      ])
+    ).toBeUndefined()
+    expect(normalizeDurableSecretProvenanceEntries([exact, entry])).toBeUndefined()
+  })
+
+  it('preserves distinct bindings whose fields contain delimiter characters', () => {
+    const entries = [
+      { encryptedValue: 'ciphertext', name: 'name', sourceValueHash: 'hash\u0000part' },
+      { encryptedValue: 'ciphertext', name: 'part\u0000name', sourceValueHash: 'hash' },
+    ]
+    expect(normalizeDurableSecretProvenanceEntries(entries)).toHaveLength(2)
+  })
+
+  it('folds message hashes only after selection while retaining source scope and names on import', async () => {
+    const registry = new ResolvedSecretTraceRegistry()
+    const imported = vi.spyOn(registry, 'importProvenance').mockResolvedValue(true)
+    const entries = Array.from({ length: PROVENANCE_MAX_ENTRIES + 1 }, (_, index) => ({
+      encryptedValue: 'ciphertext',
+      name: 'TOKEN',
+      sourceUserId: 'source-user',
+      sourceWorkspaceId: 'source-workspace',
+      sourceValueHash: `hash-${index}`,
+    }))
+    await expect(
+      importDurableSecretProvenance(registry, {
+        status: 'exact',
+        entries: [
+          ...entries,
+          { ...entries[0], name: 'ALIAS' },
+          { ...entries[0], sourceUserId: 'other-user' },
+        ],
+      })
+    ).resolves.toBe(true)
+    expect(imported).toHaveBeenCalledTimes(2)
+    expect(imported).toHaveBeenCalledWith(
+      {
+        version: 1,
+        complete: true,
+        scope: { userId: 'source-user', workspaceId: 'source-workspace' },
+        entries: [
+          { encryptedValue: 'ciphertext', name: 'ALIAS' },
+          { encryptedValue: 'ciphertext', name: 'TOKEN' },
+        ],
+      },
+      { trusted: true, origin: 'durableProvenance.envelope' }
+    )
+    expect(imported).toHaveBeenCalledWith(
+      {
+        version: 1,
+        complete: true,
+        scope: { userId: 'other-user', workspaceId: 'source-workspace' },
+        entries: [{ encryptedValue: 'ciphertext', name: 'TOKEN' }],
+      },
+      { trusted: true, origin: 'durableProvenance.envelope' }
+    )
+  })
+})
+
 describe('private durable provenance scope admission', () => {
   it('accepts a different source user in the authorized destination workspace', () => {
     expect(

apps/sim/lib/execution/durable-secret-provenance.ts

@@ -9,17 +9,14 @@ import {
   isPrivateSecretProvenanceBundleV1,
   type PrivateSecretProvenanceBundleV1,
 } from '@/lib/execution/model-input-provenance'
-import {
-  PROVENANCE_MAX_ENTRIES,
-  PROVENANCE_MAX_SERIALIZED_BYTES,
-} from '@/lib/execution/provenance-limits'
+import { SecretProvenanceBudget } from '@/lib/execution/provenance-budget'
+import { PROVENANCE_MAX_SERIALIZED_BYTES } from '@/lib/execution/provenance-limits'
 import {
   type ResolvedSecretTraceProvenanceV1,
   ResolvedSecretTraceRegistry,
   type ResolvedSecretTraceScopeV1,
 } from '@/executor/utils/resolved-secret-trace-registry'
 
-const MAX_DURABLE_HASH_NODES = 50_000
 const MAX_DURABLE_HASH_DEPTH = 100
 const MAX_DURABLE_HASH_BYTES = 16 * 1024 * 1024
 
@@ -40,13 +37,15 @@ function compareStrings(left: string, right: string): number {
 export function normalizeDurableSecretProvenanceEntries(
   value: unknown
 ): DurableSecretProvenanceEntry[] | undefined {
-  if (!Array.isArray(value) || value.length > PROVENANCE_MAX_ENTRIES) {
-    return undefined
-  }
+  return Array.isArray(value) ? normalizeDurableSecretProvenanceBindings(value) : undefined
+}
 
+function normalizeDurableSecretProvenanceBindings(
+  candidates: Iterable<unknown>
+): DurableSecretProvenanceEntry[] | undefined {
   const entries = new Map<string, DurableSecretProvenanceEntry>()
-  let bytes = 0
-  for (const candidate of value) {
+  const budget = new SecretProvenanceBudget()
+  for (const candidate of candidates) {
     if (!candidate || typeof candidate !== 'object' || Array.isArray(candidate)) return undefined
     const record = candidate as Record<string, unknown>
     if (
@@ -73,25 +72,25 @@ export function normalizeDurableSecretProvenanceEntries(
         ? { sourceValueHash: record.sourceValueHash }
         : {}),
     }
-    const key = `${entry.sourceUserId ?? ''}\u0000${entry.sourceWorkspaceId ?? ''}\u0000${entry.sourceValueHash ?? ''}\u0000${entry.name ?? ''}\u0000${entry.encryptedValue}`
+    let minimumBytes = 0
+    for (const field of Object.values(entry)) {
+      minimumBytes += Buffer.byteLength(field, 'utf8')
+      if (minimumBytes > PROVENANCE_MAX_SERIALIZED_BYTES) return undefined
+    }
+    const key = JSON.stringify(entry)
     if (entries.has(key)) continue
-    bytes += Buffer.byteLength(key, 'utf8')
-    if (bytes > PROVENANCE_MAX_SERIALIZED_BYTES) return undefined
+    if (!budget.add(entry.encryptedValue, Buffer.byteLength(key, 'utf8'))) return undefined
     entries.set(key, entry)
   }
 
-  const normalized = [...entries.values()].sort(
+  return [...entries.values()].sort(
     (left, right) =>
       compareStrings(left.sourceUserId ?? '', right.sourceUserId ?? '') ||
       compareStrings(left.sourceWorkspaceId ?? '', right.sourceWorkspaceId ?? '') ||
       compareStrings(left.sourceValueHash ?? '', right.sourceValueHash ?? '') ||
       compareStrings(left.name ?? '', right.name ?? '') ||
       compareStrings(left.encryptedValue, right.encryptedValue)
   )
-  if (Buffer.byteLength(JSON.stringify(normalized), 'utf8') > PROVENANCE_MAX_SERIALIZED_BYTES) {
-    return undefined
-  }
-  return normalized
 }
 
 /** Converts a complete transport envelope into its scope-preserving durable representation. */
@@ -165,8 +164,12 @@ export function mergeDurableSecretProvenance(
   ...values: readonly DurableSecretProvenance[]
 ): DurableSecretProvenance {
   if (values.some((value) => value.status === 'unknown')) return { status: 'unknown' }
-  const normalized = normalizeDurableSecretProvenanceEntries(
-    values.flatMap((value) => (value.status === 'exact' ? value.entries : []))
+  const normalized = normalizeDurableSecretProvenanceBindings(
+    (function* () {
+      for (const value of values) {
+        if (value.status === 'exact') yield* value.entries
+      }
+    })()
   )
   return normalized ? { status: 'exact', entries: normalized } : { status: 'unknown' }
 }
@@ -244,32 +247,40 @@ export async function importDurableSecretProvenance(
     return false
   }
 
-  const grouped = new Map<string, DurableSecretProvenanceEntry[]>()
+  const grouped = new Map<
+    string,
+    {
+      scope?: ResolvedSecretTraceScopeV1
+      entries: Map<string, ResolvedSecretTraceProvenanceV1['entries'][number]>
+    }
+  >()
   for (const entry of entries) {
-    const key = `${entry.sourceUserId ?? ''}\u0000${entry.sourceWorkspaceId ?? ''}`
-    const group = grouped.get(key) ?? []
-    group.push(entry)
-    grouped.set(key, group)
+    const scope = entry.sourceUserId
+      ? {
+          userId: entry.sourceUserId,
+          ...(entry.sourceWorkspaceId ? { workspaceId: entry.sourceWorkspaceId } : {}),
+        }
+      : undefined
+    const key = JSON.stringify(scope ?? null)
+    let group = grouped.get(key)
+    if (!group) {
+      group = { scope, entries: new Map() }
+      grouped.set(key, group)
+    }
+    const transportEntry = {
+      encryptedValue: entry.encryptedValue,
+      ...(entry.name ? { name: entry.name } : {}),
+    }
+    group.entries.set(JSON.stringify(transportEntry), transportEntry)
   }
 
   let complete = true
   for (const group of grouped.values()) {
-    const first = group[0]
     const envelope: ResolvedSecretTraceProvenanceV1 = {
       version: 1,
       complete: true,
-      entries: group.map((entry) => ({
-        encryptedValue: entry.encryptedValue,
-        ...(entry.name ? { name: entry.name } : {}),
-      })),
-      ...(first.sourceUserId
-        ? {
-            scope: {
-              userId: first.sourceUserId,
-              ...(first.sourceWorkspaceId ? { workspaceId: first.sourceWorkspaceId } : {}),
-            },
-          }
-        : {}),
+      entries: [...group.entries.values()],
+      ...(group.scope ? { scope: group.scope } : {}),
     }
     const imported =
       value === undefined
@@ -309,7 +320,6 @@ export async function createDurableSecretProvenanceRegistry(
 export function hashDurableSecretProvenanceValue(value: unknown): string | undefined {
   const hash = createHash('sha256')
   const ancestors = new WeakSet<object>()
-  let nodes = 0
   let bytes = 0
 
   const append = (chunk: string): boolean => {
@@ -320,8 +330,7 @@ export function hashDurableSecretProvenanceValue(value: unknown): string | undef
   }
 
   const visit = (candidate: unknown, depth: number): boolean => {
-    nodes++
-    if (nodes > MAX_DURABLE_HASH_NODES || depth > MAX_DURABLE_HASH_DEPTH) return false
+    if (depth > MAX_DURABLE_HASH_DEPTH) return false
     if (candidate === null) return append('null')
     if (typeof candidate === 'string') return append(JSON.stringify(candidate))
     if (typeof candidate === 'boolean') return append(candidate ? 'true' : 'false')

apps/sim/lib/execution/provenance-budget.test.ts

@@ -0,0 +1,35 @@
+/**
+ * @vitest-environment node
+ */
+import { describe, expect, it } from 'vitest'
+import { SecretProvenanceBudget } from '@/lib/execution/provenance-budget'
+import {
+  PROVENANCE_MAX_ENTRIES,
+  PROVENANCE_MAX_SERIALIZED_BYTES,
+} from '@/lib/execution/provenance-limits'
+
+describe('incremental provenance budget', () => {
+  it('charges repeated bindings against bytes but only distinct ciphertexts against secrets', () => {
+    const budget = new SecretProvenanceBudget()
+    for (let index = 0; index < PROVENANCE_MAX_ENTRIES + 1; index++) {
+      expect(budget.add('same-ciphertext', 100)).toBe(true)
+    }
+  })
+
+  it('refuses an additional distinct secret without refusing another binding of a known secret', () => {
+    const budget = new SecretProvenanceBudget()
+    for (let index = 0; index < PROVENANCE_MAX_ENTRIES; index++) {
+      expect(budget.add(`ciphertext-${index}`, 100)).toBe(true)
+    }
+    expect(budget.add('additional-secret', 100)).toBe(false)
+    expect(budget.add('ciphertext-0', 100)).toBe(true)
+  })
+
+  it('includes brackets and commas and does not consume capacity on a rejected binding', () => {
+    const budget = new SecretProvenanceBudget()
+    expect(budget.add('first', PROVENANCE_MAX_SERIALIZED_BYTES - 5)).toBe(true)
+    expect(budget.add('too-large', 3)).toBe(false)
+    expect(budget.add('fits', 2)).toBe(true)
+    expect(budget.add('fits', 2)).toBe(false)
+  })
+})

apps/sim/lib/execution/provenance-budget.ts

@@ -0,0 +1,27 @@
+import {
+  PROVENANCE_MAX_ENTRIES,
+  PROVENANCE_MAX_SERIALIZED_BYTES,
+} from '@/lib/execution/provenance-limits'
+
+/** Tracks distinct secrets and serialized bytes while callers fold durable bindings incrementally. */
+export class SecretProvenanceBudget {
+  private readonly encryptedValues = new Set<string>()
+  private serializedBytes = 2
+  private bindingCount = 0
+
+  /** Admits one deduplicated binding, including its JSON array separator in the byte budget. */
+  add(encryptedValue: string, serializedEntryBytes: number): boolean {
+    const nextBytes = this.serializedBytes + serializedEntryBytes + (this.bindingCount > 0 ? 1 : 0)
+    if (
+      nextBytes > PROVENANCE_MAX_SERIALIZED_BYTES ||
+      (!this.encryptedValues.has(encryptedValue) &&
+        this.encryptedValues.size >= PROVENANCE_MAX_ENTRIES)
+    ) {
+      return false
+    }
+    this.encryptedValues.add(encryptedValue)
+    this.serializedBytes = nextBytes
+    this.bindingCount++
+    return true
+  }
+}