From c1592423cfc381c124962c191da8ab0e3ac6a648 Mon Sep 17 00:00:00 2001 From: Gorkem Date: Sat, 8 Aug 2026 15:42:28 +0300 Subject: [PATCH 1/7] feat: route reflection mapped rows through the uniform dedup/merge pipeline --- dist/index.js | 109 ++- dist/src/smart-extractor.js | 584 ++++++++++++- index.ts | 116 ++- package.json | 2 +- scripts/ci-test-manifest.mjs | 1 + src/smart-extractor.ts | 722 +++++++++++++++- .../reflection-mapped-rows-admission.test.mjs | 78 +- ...eflection-mapped-uniform-pipeline.test.mjs | 794 ++++++++++++++++++ test/smart-extractor-batch-admission.test.mjs | 1 + test/smart-extractor-batch-embed.test.mjs | 1 + .../smart-extractor-merge-accounting.test.mjs | 1 + 11 files changed, 2264 insertions(+), 145 deletions(-) create mode 100644 test/reflection-mapped-uniform-pipeline.test.mjs diff --git a/dist/index.js b/dist/index.js index a81e9c19f..cb54039b8 100644 --- a/dist/index.js +++ b/dist/index.js @@ -36,7 +36,7 @@ import { storeReflectionToLanceDB, loadAgentReflectionSlicesFromEntries, DEFAULT import { parseReflectionMetadata } from "./src/reflection-metadata.js"; import { extractReflectionLearningGovernanceCandidates, extractInjectableReflectionMappedMemoryItems, isRecallUsed, } from "./src/reflection-slices.js"; import { createReflectionEventId } from "./src/reflection-event-store.js"; -import { buildReflectionMappedMetadata, getReflectionMappedStorageCategory } from "./src/reflection-mapped-metadata.js"; +import { buildReflectionMappedMetadata, getReflectionMappedMemoryCategory, getReflectionMappedStorageCategory } from "./src/reflection-mapped-metadata.js"; import { buildFallbackCandidate, gateRegexFallbackCapture } from "./src/autocapture-fallback-admission.js"; import { gateMappedReflectionEntries, resolveMappedRowAdmissionController } from "./src/reflection-mapped-admission.js"; import { createMemoryCLI } from "./cli.js"; @@ -4704,8 +4704,9 @@ const memoryLanceDBProPlugin = { const MAX_MAPPED_ENTRIES = 100; const mappedReflectionMemories = extractInjectableReflectionMappedMemoryItems(reflectionText); const mappedEntries = []; - // Per-row embed + near-duplicate pre-check first, collecting the - // gate-eligible rows so the whole burst can share one admission call. + const mappedGatedItems = []; + // Per-row embed first, collecting the gate-eligible rows so the + // whole burst can share one admission call. const gateEligible = []; for (const mapped of mappedReflectionMemories) { if (gateEligible.length >= MAX_MAPPED_ENTRIES) { @@ -4720,28 +4721,25 @@ const memoryLanceDBProPlugin = { api.logger.warn(`memory-reflection: mapped row embedding failed after retry, skipping row: ${String(embedErr)}`); continue; } - let existing = []; - let searchFailed = false; - try { - existing = await store.vectorSearch(vector, 1, 0.1, [targetScope]); - } - catch (err) { - api.logger.warn(`memory-reflection: mapped memory duplicate pre-check failed, skip store: ${String(err)}`); - searchFailed = true; - } - if (searchFailed) { - continue; - } - // Near-duplicate pre-check ahead of admission gating. This is the only dedup mapped - // rows get: a single vector-similarity threshold, direct skip, no LLM-mediated - // merge/contextualize/contradict decision. Extraction candidates own deduplicate() - // (src/smart-extractor.ts) is a genuinely different, richer pipeline (a 0.7 - // pre-filter feeding an LLM decision, not a single hard cutoff) - deliberately not - // reused here yet. AdmissionController's "pass_to_dedup" decision for a mapped row - // is therefore always treated as "admit, subject to this cheaper pre-check" below, - // not "route through the same merge pipeline extraction candidates get". - if (existing.length > 0 && existing[0].score > 0.95) { - continue; + // Extractor-backed runs take the SAME dedup/merge pipeline + // extraction candidates get (persistGatedCandidates below), so no + // bespoke similarity cutoff runs here. The no-extractor fallback + // keeps the historical near-duplicate pre-check, downgraded from + // fail-closed to fail-open: a search blip stores the row (worst + // case the near-duplicate lands as a separate row — this path + // only pre-checks, it has no merge step) instead of silently + // dropping it. + if (!smartExtractor) { + let existing = []; + try { + existing = await store.vectorSearch(vector, 1, 0.1, [targetScope]); + } + catch (err) { + api.logger.warn(`memory-reflection: mapped memory duplicate pre-check failed, storing without pre-check: ${String(err)}`); + } + if (existing.length > 0 && existing[0].score > 0.95) { + continue; + } } gateEligible.push({ mapped, vector }); } @@ -4794,14 +4792,61 @@ const memoryLanceDBProPlugin = { baseMetadata.admission_audit = mappedGate.auditJson; } const metadata = JSON.stringify(baseMetadata); - mappedEntries.push({ - text: mapped.text, - vector, - importance, - category: getReflectionMappedStorageCategory(mapped.mappedKind), - scope: targetScope, - metadata, + if (smartExtractor) { + // Uniform pipeline: judge (done above) -> dedup -> merge-writer, + // identical to extraction candidates. The entry builder keeps + // the reflection metadata on CREATE-shaped verdicts. + mappedGatedItems.push({ + candidate: { + category: getReflectionMappedMemoryCategory(mapped.mappedKind), + abstract: mapped.text, + overview: `## ${mapped.heading}`, + content: mapped.text, + }, + vector, + buildEntry: (v) => ({ + text: mapped.text, + vector: v, + importance, + category: getReflectionMappedStorageCategory(mapped.mappedKind), + scope: targetScope, + metadata, + }), + }); + } + else { + mappedEntries.push({ + text: mapped.text, + vector, + importance, + category: getReflectionMappedStorageCategory(mapped.mappedKind), + scope: targetScope, + metadata, + }); + } + } + if (smartExtractor && mappedGatedItems.length > 0) { + const gatedResult = await smartExtractor.persistGatedCandidates(mappedGatedItems, { + sessionKey, + targetScope, + scopeFilter: [targetScope], + agentId: ownerAgentId, + conversationText: conversation, }); + api.logger.info(`memory-reflection: mapped rows through uniform pipeline: ${gatedResult.createdEntries.length} created, ${gatedResult.stats.merged} merged, ${gatedResult.stats.skipped} skipped`); + if (mdMirror) { + for (const stored of gatedResult.createdEntries) { + let heading = "unknown"; + try { + const storedMeta = stored.metadata ? JSON.parse(stored.metadata) : {}; + heading = storedMeta._reflectionHeading ?? "unknown"; + } + catch { + api.logger.warn(`memory-reflection: failed to parse stored metadata for entry ${stored.id}, using "unknown"`); + } + await mdMirror({ text: stored.text, category: stored.category, scope: stored.scope, timestamp: stored.timestamp }, { source: `reflection:${heading}`, agentId: sourceAgentId }); + } + } } if (mappedEntries.length > 0) { const storedEntries = await store.bulkStore(mappedEntries, ({ index, reason }) => { diff --git a/dist/src/smart-extractor.js b/dist/src/smart-extractor.js index 56eb6dfe3..5f0d63e15 100644 --- a/dist/src/smart-extractor.js +++ b/dist/src/smart-extractor.js @@ -15,6 +15,37 @@ import { classifyTemporal, inferExpiry } from "./temporal-classifier.js"; import { inferAtomicBrandItemPreferenceSlot } from "./preference-slots.js"; import { batchDedup } from "./batch-dedup.js"; import { buildBoundedTranscriptWithStats, } from "./auto-capture-cleanup.js"; +/** + * The caller's own admission audit, as carried inside an externally-built + * entry's metadata. Used by the gated-candidate lane so downstream verdict + * handling persists the real gate record, never a synthetic marker. + */ +function parseEntryAdmissionAudit(entry) { + const raw = entry.metadata; + if (typeof raw !== "string" || raw.length === 0) { + return undefined; + } + try { + const meta = JSON.parse(raw); + // Production external builders (the reflection mapped lane) persist the + // gate record as a nested JSON string under admission_audit. + const audit = meta.admission_audit; + const parsed = typeof audit === "string" && audit.length > 0 ? JSON.parse(audit) : audit; + // Fail-open gate markers ({provenance, failedOpen, reason, error}) are + // evidence of a skipped evaluation, not an audit: adopting one would let + // MERGE/SUPPORT overwrite a target's complete audit with it. + if (parsed && + typeof parsed === "object" && + parsed.version === "amac-v1" && + typeof parsed.decision === "string") { + return parsed; + } + return undefined; + } + catch { + return undefined; + } +} // ============================================================================ // Envelope Metadata Stripping // ============================================================================ @@ -289,6 +320,26 @@ function normalizeRegisterToken(value) { // Constants // ============================================================================ const SIMILARITY_THRESHOLD = 0.7; +const NO_SIMILAR_MEMORIES_REASON = "No similar memories found"; +// Burst-lane identity of a row's serialized metadata: the mapped kind wins, +// then the reflection heading; anything else is the shared empty lane. +function laneFromMetadata(rawMeta) { + if (typeof rawMeta === "string" && rawMeta.length > 0) { + try { + const meta = JSON.parse(rawMeta); + if (typeof meta.mappedKind === "string" && meta.mappedKind.length > 0) { + return meta.mappedKind; + } + if (typeof meta._reflectionHeading === "string" && meta._reflectionHeading.length > 0) { + return meta._reflectionHeading; + } + } + catch { + // unparseable metadata falls through to the empty lane + } + } + return ""; +} const MAX_SIMILAR_FOR_PROMPT = 3; const MAX_MEMORIES_PER_EXTRACTION = 5; /** Max candidates decided in one batched dedup LLM call; larger batches are chunked. */ @@ -622,6 +673,320 @@ export class SmartExtractor { 0; return stats; } + /** + * Uniform-pipeline entry for candidates whose extraction AND admission + * already happened in another lane (the reflection writer's mapped rows: + * distilled by the reflection model, gated by gateMappedReflectionEntries). + * From here on they take exactly the extraction candidates' path -- + * batched dedup decider, verdict handling, batched merge writer, bulk + * create -- so a duplicate mapped row MERGES into its target instead of + * landing beside it. + * + * Each item supplies its own store-entry builder: a CREATE-shaped verdict + * persists the caller's entry (reflection metadata intact), while + * merge/supersede/support/contextualize/contradict operate on existing + * rows through the shared machinery. Callers own persistence + * notifications for created rows (the returned entries), keeping their + * lane-specific journal labels. + */ + async persistGatedCandidates(items, options) { + const stats = { created: 0, merged: 0, skipped: 0, boundarySkipped: 0 }; + const sessionKey = options.sessionKey ?? "reflection"; + const targetScope = options.targetScope; + const scopeFilter = options.scopeFilter ?? [targetScope]; + const conversationText = options.conversationText ?? ""; + for (const item of items) { + const prebuilt = item.buildEntry(item.vector); + this.externalEntryBuilders.set(item.candidate, { + build: item.buildEntry, + prebuilt, + audit: parseEntryAdmissionAudit(prebuilt), + }); + } + // Admission already ran in the caller's gate; the evaluation handed to + // processCandidate only tells it not to score again. Its audit is the + // CALLER'S OWN record (parsed from the built entry) — never a synthetic + // stub — so anything persisted downstream carries the real gate audit. + const preGatedFor = (candidate) => ({ + decision: "pass_to_dedup", + audit: this.externalEntryBuilders.get(candidate)?.audit, + }); + // Same-burst twin guard: collapse EXACT normalized duplicates within one + // caller lane. The lane identity comes from the prebuilt entry's mapped + // kind (its reflection heading as fallback): lessons and decisions share + // one candidate category while carrying different kinds, headings, + // importance, and decay, so a category+text key would deterministically + // drop the later lane's row. Anything short of textual identity within a + // lane proceeds to the dedup judge. + const burstLaneOf = (candidate) => laneFromMetadata(this.externalEntryBuilders.get(candidate)?.prebuilt?.metadata); + const seenBurstKeys = new Set(); + const surviving = []; + for (const item of items) { + const key = JSON.stringify([ + burstLaneOf(item.candidate), + item.candidate.category, + item.candidate.abstract.toLowerCase().replace(/\s+/g, " ").trim(), + ]); + if (seenBurstKeys.has(key)) { + stats.skipped++; + this.log(`memory-pro: smart-extractor: gated-candidate burst twin dropped [${item.candidate.category}]`); + continue; + } + seenBurstKeys.add(key); + surviving.push(item); + } + // Same-lane siblings earlier in one burst act as virtual dedup + // neighbors: with no similar row in the store yet, two related mapped + // rows arriving together would otherwise BOTH short-circuit to CREATE + // and the semantic judge would never see the pair. A verdict against a + // sibling resolves after bulkStore assigns the sibling's real id, then + // reuses the normal merge/support machinery. + const BURST_SIBLING_PREFIX = "burst-sibling:"; + const laneKeyOf = (candidate) => JSON.stringify([burstLaneOf(candidate), candidate.category]); + const cosineOf = (a, b) => { + if (a.length === 0 || a.length !== b.length) { + return 0; + } + let dot = 0; + let normA = 0; + let normB = 0; + for (let d = 0; d < a.length; d++) { + dot += a[d] * b[d]; + normA += a[d] * a[d]; + normB += b[d] * b[d]; + } + if (normA === 0 || normB === 0) { + return 0; + } + return dot / (Math.sqrt(normA) * Math.sqrt(normB)); + }; + const burstSiblingsFor = (index) => { + const { candidate, vector } = surviving[index]; + if (!vector || vector.length === 0) { + return []; + } + const lane = laneKeyOf(candidate); + const out = []; + for (let j = 0; j < index; j++) { + const sibling = surviving[j]; + if (laneKeyOf(sibling.candidate) !== lane) { + continue; + } + const score = cosineOf(vector, sibling.vector || []); + if (score < SIMILARITY_THRESHOLD) { + continue; + } + const prebuilt = this.externalEntryBuilders.get(sibling.candidate)?.prebuilt; + const entryCategory = typeof prebuilt?.category === "string" + ? prebuilt.category + : this.mapToStoreCategory(sibling.candidate.category); + out.push({ + entry: { + id: `${BURST_SIBLING_PREFIX}${j}`, + text: sibling.candidate.abstract, + vector: [], + category: entryCategory, + scope: typeof prebuilt?.scope === "string" ? prebuilt.scope : targetScope, + importance: typeof prebuilt?.importance === "number" ? prebuilt.importance : 0.8, + timestamp: Date.now(), + metadata: typeof prebuilt?.metadata === "string" ? prebuilt.metadata : "{}", + }, + score, + }); + } + return out; + }; + const precomputedDedups = new Map(); + const dedupLlmItems = []; + for (let i = 0; i < surviving.length; i++) { + const { candidate, vector } = surviving[i]; + const siblings = burstSiblingsFor(i); + try { + const prefilter = await this.dedupPrefilter(candidate, vector, scopeFilter); + const emptyStoreShortCircuit = prefilter.shortCircuit?.reason === NO_SIMILAR_MEMORIES_REASON; + if (prefilter.shortCircuit && !(siblings.length > 0 && emptyStoreShortCircuit)) { + // Domain short-circuits (e.g. the preference-slot guard) stay + // authoritative even when burst siblings exist; only the plain + // "nothing similar stored yet" bypass yields to sibling context. + precomputedDedups.set(i, prefilter.shortCircuit); + } + else { + const topSimilar = [...prefilter.topSimilar, ...siblings] + .sort((a, b) => b.score - a.score) + .slice(0, 5); + dedupLlmItems.push({ index: i, candidate, topSimilar }); + } + } + catch (err) { + this.log(`memory-pro: smart-extractor: gated-candidate dedup pre-filter failed, deferring to inline dedup: ${String(err)}`); + } + } + if (dedupLlmItems.length > 0) { + const verdicts = await this.llmDedupDecisionBatch(dedupLlmItems); + dedupLlmItems.forEach((item, i) => { + precomputedDedups.set(item.index, verdicts[i]); + }); + } + const createEntries = []; + const pendingSupersedeInvalidations = []; + const pendingMerges = []; + const pendingSiblingVerdicts = []; + const createSlotBySurviving = new Map(); + for (let i = 0; i < surviving.length; i++) { + const { candidate, vector } = surviving[i]; + const pre = precomputedDedups.get(i); + if (pre?.matchId && pre.matchId.startsWith(BURST_SIBLING_PREFIX)) { + const siblingIndex = Number(pre.matchId.slice(BURST_SIBLING_PREFIX.length)); + const resolvable = Number.isInteger(siblingIndex) && siblingIndex >= 0 && siblingIndex < i; + if (pre.decision === "skip" && resolvable) { + stats.skipped++; + this.log(`memory-pro: smart-extractor: gated candidate judged same-burst duplicate of an earlier sibling [${candidate.category}]`); + continue; + } + if ((pre.decision === "merge" || pre.decision === "support") && resolvable) { + pendingSiblingVerdicts.push({ + candidate, + vector, + siblingIndex, + decision: pre.decision, + reason: pre.reason, + contextLabel: pre.contextLabel, + }); + continue; + } + // Any other verdict against a not-yet-persisted sibling row keeps + // the caller's entry: fail open to a plain create. + precomputedDedups.set(i, { + decision: "create", + reason: "sibling verdict fallback (unsupported decision for a pending row)", + }); + } + const createCountBefore = createEntries.length; + try { + await this.processCandidate(candidate, conversationText, sessionKey, stats, targetScope, scopeFilter, vector, createEntries, pendingSupersedeInvalidations, options.agentId, preGatedFor(candidate), precomputedDedups.get(i), pendingMerges); + if (createEntries.length === createCountBefore + 1) { + createSlotBySurviving.set(i, createCountBefore); + } + } + catch (err) { + this.log(`memory-pro: smart-extractor: failed to process gated candidate [${candidate.category}]: ${String(err)}`); + // Fail open: this candidate already passed the caller's admission + // gate, so a processing failure (dedup search, verdict handling) + // must not silently drop it — store the caller-built row as-is. + const ext = this.externalEntryBuilders.get(candidate); + if (ext) { + createEntries.push(ext.prebuilt ?? ext.build(vector)); + createSlotBySurviving.set(i, createEntries.length - 1); + stats.created++; + this.log(`memory-pro: smart-extractor: fail-open create for gated candidate after processing failure [${candidate.category}]`); + } + } + } + await this.flushPendingMerges(pendingMerges, stats, createEntries); + let createdEntries = []; + if (createEntries.length > 0) { + const stored = await this.bulkStoreAndValidate(createEntries); + if (stored) { + createdEntries = stored; + await this.applyPendingSupersedeInvalidations(stored, pendingSupersedeInvalidations); + } + else if (pendingSupersedeInvalidations.length > 0) { + this.log("memory-pro: smart-extractor: gated-candidate supersede invalidation skipped because bulkStore() did not return created entries"); + } + } + // Deferred same-burst verdicts: the sibling's row now has a real id, so + // merge/support resolve through the normal machinery. Anything that + // cannot be resolved keeps the caller's row (fail open to create). + if (pendingSiblingVerdicts.length > 0) { + const claimedIds = new Set(); + const storedIdForSurviving = (survivingIndex) => { + const slot = createSlotBySurviving.get(survivingIndex); + if (slot === undefined) { + return undefined; + } + if (createdEntries.length === createEntries.length) { + const id = createdEntries[slot]?.id; + if (id) { + claimedIds.add(id); + } + return id; + } + // bulkStore may filter entries, shifting positions: fall back to the + // first unclaimed row with the same text AND the same lane/category + // identity — identical text is legal across lanes, so a text-only + // match could bind the verdict to another lane's row. + const want = createEntries[slot]; + const hit = want + ? createdEntries.find((e) => e.text === want.text && + e.category === want.category && + laneFromMetadata(e.metadata) === laneFromMetadata(want.metadata) && + !claimedIds.has(e.id)) + : undefined; + if (hit) { + claimedIds.add(hit.id); + } + return hit?.id; + }; + const followupMerges = []; + const followupCreates = []; + for (const pending of pendingSiblingVerdicts) { + const audit = this.externalEntryBuilders.get(pending.candidate)?.audit; + const failOpenCreate = async (why) => { + followupCreates.push(await this.externalOrBuiltFallbackEntry(pending.candidate, targetScope, sessionKey, pending.vector, audit)); + stats.created++; + this.log(`memory-pro: smart-extractor: ${why}, storing gated candidate as new [${pending.candidate.category}]`); + }; + try { + const targetId = storedIdForSurviving(pending.siblingIndex); + if (!targetId) { + await failOpenCreate("same-burst sibling row not persisted"); + continue; + } + if (pending.decision === "support") { + const outcome = await this.handleSupport(targetId, { session: sessionKey, timestamp: Date.now() }, pending.reason, pending.contextLabel, scopeFilter, audit); + if (outcome === "supported") { + stats.supported = (stats.supported ?? 0) + 1; + } + else { + await failOpenCreate("same-burst support target vanished"); + } + continue; + } + const queued = await this.queueMergeJob(followupMerges, pending.candidate, targetId, targetScope, scopeFilter, pending.contextLabel, audit, followupCreates, options.agentId); + if (queued === "created") { + stats.created++; + } + } + catch (err) { + // A deferred verdict degrades ALONE: the candidate already passed + // the caller's admission gate, so a throwing store read/write on + // one resolution must fall open to that candidate's own create — + // never reject the whole persistence call, discard follow-up work + // queued by earlier verdicts, or skip the verdicts still pending. + // Push paths above enqueue only after their awaits resolve, so a + // caught verdict has enqueued nothing yet and the fallback row + // lands exactly once. + this.log(`memory-pro: smart-extractor: deferred sibling ${pending.decision} failed: ${String(err)}`); + try { + await failOpenCreate(`deferred sibling ${pending.decision} unresolved`); + } + catch (fallbackErr) { + this.log(`memory-pro: smart-extractor: fail-open create failed for a deferred sibling verdict [${pending.candidate.category}]: ${String(fallbackErr)}`); + } + } + } + if (followupMerges.length > 0) { + await this.flushPendingMerges(followupMerges, stats, followupCreates); + } + if (followupCreates.length > 0) { + const extra = await this.bulkStoreAndValidate(followupCreates); + if (extra) { + createdEntries = createdEntries.concat(extra); + } + } + } + return { stats, createdEntries }; + } // -------------------------------------------------------------------------- // Embedding Noise Pre-Filter // -------------------------------------------------------------------------- @@ -1339,8 +1704,16 @@ export class SmartExtractor { break; case "support": if (dedupResult.matchId) { - await this.handleSupport(dedupResult.matchId, { session: sessionKey, timestamp: Date.now() }, dedupResult.reason, dedupResult.contextLabel, scopeFilter, admission?.audit); - stats.supported = (stats.supported ?? 0) + 1; + const supportOutcome = await this.handleSupport(dedupResult.matchId, { session: sessionKey, timestamp: Date.now() }, dedupResult.reason, dedupResult.contextLabel, scopeFilter, admission?.audit); + if (supportOutcome === "supported") { + stats.supported = (stats.supported ?? 0) + 1; + } + else { + // Target vanished mid-flight: same semantics as a support verdict + // with no target — the candidate lands as a new row. + createEntries?.push(await this.externalOrBuiltFallbackEntry(candidate, targetScope, sessionKey, vector, admission?.audit)); + stats.created++; + } } else { createEntries?.push(this.buildStoreEntry(candidate, vector, sessionKey, targetScope, admission?.audit)); @@ -1405,7 +1778,7 @@ export class SmartExtractor { const activeSimilar = await this.store.vectorSearch(candidateVector, 5, SIMILARITY_THRESHOLD, scopeFilter, { excludeInactive: true }); if (activeSimilar.length === 0) { return { - shortCircuit: { decision: "create", reason: "No similar memories found" }, + shortCircuit: { decision: "create", reason: NO_SIMILAR_MEMORIES_REASON }, topSimilar: [], }; } @@ -1641,7 +2014,11 @@ export class SmartExtractor { this.log("memory-pro: smart-extractor: merge LLM failed, skipping merge"); return "llm-failed"; } - await this.applyMergedContent(matchId, candidate.category, merged, targetScope, scopeFilter, [contextLabel], admissionAudit, agentId); + const applied = await this.applyMergedContent(matchId, candidate.category, merged, targetScope, scopeFilter, [contextLabel], admissionAudit, agentId); + if (applied === "target-missing") { + createEntries?.push(await this.externalOrBuiltFallbackEntry(candidate, targetScope, "merge-fallback", undefined, admissionAudit)); + return "created"; + } return "merged"; } /** @@ -1650,25 +2027,43 @@ export class SmartExtractor { * exactly the fallback the inline merge path always used) and null is * returned so the caller can account for it as "created". */ + /** + * The CREATE row a candidate falls back to when its verdict target cannot + * be mutated: the caller's own prebuilt entry for externally gated + * candidates (shape, provenance, and audit stay the caller's), the + * standard auto-capture entry otherwise. + */ + async externalOrBuiltFallbackEntry(candidate, targetScope, sessionLabel, vector, admissionAudit) { + const ext = this.externalEntryBuilders.get(candidate); + if (ext?.prebuilt) { + return ext.prebuilt; + } + const v = vector && vector.length > 0 + ? vector + : (await this.embedder.embed(`${candidate.abstract} ${candidate.content}`)) || []; + return this.buildStoreEntry(candidate, v, sessionLabel, targetScope, admissionAudit); + } async readMergeTarget(candidate, matchId, targetScope, scopeFilter, createEntries) { try { const existing = await this.store.getById(matchId, scopeFilter); - let abstract = ""; - let overview = ""; - let content = ""; - if (existing) { - const meta = parseSmartMetadata(existing.metadata, existing); - abstract = meta.l0_abstract || existing.text; - overview = meta.l1_overview || ""; - content = meta.l2_content || existing.text; + if (!existing) { + // Target vanished between dedup and read: merging into a missing row + // would silently drop the candidate, so store it as new instead. + this.log(`memory-pro: smart-extractor: merge target ${matchId.slice(0, 8)} no longer exists, storing as new`); + createEntries?.push(await this.externalOrBuiltFallbackEntry(candidate, targetScope, "merge-fallback")); + return null; } - return { abstract, overview, content }; + const meta = parseSmartMetadata(existing.metadata, existing); + return { + abstract: meta.l0_abstract || existing.text, + overview: meta.l1_overview || "", + content: meta.l2_content || existing.text, + }; } catch { // Fallback: store as new this.log(`memory-pro: smart-extractor: could not read existing memory ${matchId}, storing as new`); - const vector = await this.embedder.embed(`${candidate.abstract} ${candidate.content}`); - createEntries?.push(this.buildStoreEntry(candidate, vector || [], "merge-fallback", targetScope)); + createEntries?.push(await this.externalOrBuiltFallbackEntry(candidate, targetScope, "merge-fallback")); return null; } } @@ -1711,24 +2106,69 @@ export class SmartExtractor { * itself fails degrades every job in that chunk the same way. Never * throws, never fans back out into per-job LLM calls. */ - async flushPendingMerges(pendingMerges, stats) { + async flushPendingMerges(pendingMerges, stats, createEntries) { if (pendingMerges.length === 0) { return; } + // Extraction-lane additions degrade like the single-call merge failure + // path (nothing persisted, target untouched). Externally-gated additions + // must NOT disappear on a degraded merge: they already passed the + // caller's admission gate and were previously direct-stored, so they + // fall back to a create built from the caller's own entry. + const failOpenAdditions = (job, why) => { + if (!createEntries) { + return; + } + for (const addition of job.additions) { + const ext = this.externalEntryBuilders.get(addition.candidate); + if (ext?.prebuilt) { + createEntries.push(ext.prebuilt); + stats.created++; + this.log(`memory-pro: smart-extractor: merge ${why} — falling back to create for gated candidate [${addition.candidate.category}]`); + } + } + }; + // A job carrying an externally gated addition mirrors under the caller's + // reflection provenance instead of the generic extraction label. + const mirrorSourceFor = (job) => { + for (const addition of job.additions) { + const ext = this.externalEntryBuilders.get(addition.candidate); + if (ext?.prebuilt) { + try { + const rawMeta = ext.prebuilt.metadata; + const meta = typeof rawMeta === "string" && rawMeta.length > 0 + ? JSON.parse(rawMeta) + : {}; + const heading = meta._reflectionHeading; + return `reflection:${typeof heading === "string" && heading.length > 0 ? heading : "unknown"}`; + } + catch { + return "reflection:unknown"; + } + } + } + return undefined; + }; const contents = await this.llmMergeContentBatch(pendingMerges); for (let i = 0; i < pendingMerges.length; i++) { const job = pendingMerges[i]; const merged = contents[i]; if (!merged) { this.log("memory-pro: smart-extractor: merge LLM failed, skipping merge"); + failOpenAdditions(job, "generation failed"); continue; } try { - await this.applyMergedContent(job.matchId, job.category, merged, job.targetScope, job.scopeFilter, job.additions.map((a) => a.contextLabel), job.additions[0]?.admissionAudit, job.agentId); + const applied = await this.applyMergedContent(job.matchId, job.category, merged, job.targetScope, job.scopeFilter, job.additions.map((a) => a.contextLabel), job.additions[0]?.admissionAudit, job.agentId, mirrorSourceFor(job)); + if (applied === "target-missing") { + failOpenAdditions(job, "target vanished"); + continue; + } stats.merged += job.additions.length; } catch (err) { this.log(`memory-pro: smart-extractor: failed to apply merged content for ${job.matchId.slice(0, 8)}: ${String(err)}`); + failOpenAdditions(job, "apply failed"); } } } @@ -1786,31 +2226,50 @@ export class SmartExtractor { * stats update once per merged-in candidate. Shared by the inline * single-call merge path and the batched merge writer. */ - async applyMergedContent(matchId, category, merged, targetScope, scopeFilter, contextLabels, admissionAudit, agentId) { + async applyMergedContent(matchId, category, merged, targetScope, scopeFilter, contextLabels, admissionAudit, agentId, mirrorSource = "smart-extraction") { // Re-embed the merged content const mergedText = `${merged.abstract} ${merged.content}`; const newVector = await this.embedder.embed(mergedText); - // Update existing memory via store.update() + // Update existing memory via store.update(). A target that vanished + // between dedup and this write must surface as "target-missing" so the + // caller can fall back — reporting success here would silently drop the + // candidate and emit a persistence notification for a write that never + // durably landed. const existing = await this.store.getById(matchId, scopeFilter); - const metadata = stringifySmartMetadata(this.withAdmissionAudit(buildSmartMetadata(existing ?? { text: merged.abstract }, { + if (!existing) { + this.log(`memory-pro: smart-extractor: merge target ${matchId.slice(0, 8)} vanished before update`); + return "target-missing"; + } + // A merge enriches the target's content; it never reclassifies the row. + // Stamping the incoming candidate's category would desync the metadata + // from the legacy category column, and list()'s two stages (column + // SQL-prefilter, then metadata validation) would drop the row from BOTH + // category views on a cross-category merge. + const existingMeta = parseSmartMetadata(existing.metadata, existing); + const targetCategory = existingMeta.memory_category || category; + const metadata = stringifySmartMetadata(this.withAdmissionAudit(buildSmartMetadata(existing, { l0_abstract: merged.abstract, l1_overview: merged.overview, l2_content: merged.content, - memory_category: category, + memory_category: targetCategory, tier: "working", confidence: 0.8, }), admissionAudit)); - await this.store.update(matchId, { + const updated = await this.store.update(matchId, { text: merged.abstract, vector: newVector, metadata, }, scopeFilter); + if (!updated) { + this.log(`memory-pro: smart-extractor: merge target ${matchId.slice(0, 8)} vanished during update`); + return "target-missing"; + } await this.notifyPersisted({ text: merged.abstract, - category: this.mapToStoreCategory(category), + category: this.mapToStoreCategory(targetCategory), scope: targetScope, timestamp: Date.now(), - }, "smart-extraction", agentId); + }, mirrorSource, agentId); for (const contextLabel of contextLabels) { // Update support stats on the merged memory try { @@ -1826,8 +2285,9 @@ export class SmartExtractor { catch { // Non-critical: merge succeeded, support stats update is best-effort } - this.log(`memory-pro: smart-extractor: merged [${category}]${contextLabel ? ` [${contextLabel}]` : ""} into ${matchId.slice(0, 8)}`); + this.log(`memory-pro: smart-extractor: merged [${targetCategory}]${contextLabel ? ` [${contextLabel}]` : ""} into ${matchId.slice(0, 8)}`); } + return "updated"; } /** * Handle SUPERSEDE: preserve the old record as historical but mark it as no @@ -1836,7 +2296,7 @@ export class SmartExtractor { async handleSupersede(candidate, vector, matchId, sessionKey, targetScope, scopeFilter, admissionAudit, createEntries, pendingSupersedeInvalidations, agentId) { const existing = await this.store.getById(matchId, scopeFilter); if (!existing) { - createEntries?.push(this.buildStoreEntry(candidate, vector || [], sessionKey, targetScope)); + createEntries?.push(await this.externalOrBuiltFallbackEntry(candidate, targetScope, sessionKey, vector, admissionAudit)); return; } const now = Date.now(); @@ -1844,7 +2304,15 @@ export class SmartExtractor { const factKey = existingMeta.fact_key ?? deriveFactKey(candidate.category, candidate.abstract); const storeCategory = this.mapToStoreCategory(candidate.category); const supersedeClassifyText = candidate.content || candidate.abstract; - const entry = { + const entry = this.externalVerdictEntry(candidate, { + state: "confirmed", + valid_from: now, + fact_key: factKey, + supersedes: matchId, + relations: appendRelation([], { type: "supersedes", targetId: matchId }), + memory_temporal_type: classifyTemporal(supersedeClassifyText), + valid_until: inferExpiry(supersedeClassifyText), + }) ?? { text: candidate.abstract, vector, category: storeCategory, @@ -1928,14 +2396,21 @@ export class SmartExtractor { */ async handleSupport(matchId, source, reason, contextLabel, scopeFilter, admissionAudit) { const existing = await this.store.getById(matchId, scopeFilter); - if (!existing) - return; + if (!existing) { + this.log(`memory-pro: smart-extractor: support target ${matchId.slice(0, 8)} no longer exists`); + return "target-missing"; + } const meta = parseSmartMetadata(existing.metadata, existing); const supportInfo = parseSupportInfo(meta.support_info); const updated = updateSupportStats(supportInfo, contextLabel, "support"); meta.support_info = updated; - await this.store.update(matchId, { metadata: stringifySmartMetadata(this.withAdmissionAudit(meta, admissionAudit)) }, scopeFilter); + const written = await this.store.update(matchId, { metadata: stringifySmartMetadata(this.withAdmissionAudit(meta, admissionAudit)) }, scopeFilter); + if (!written) { + this.log(`memory-pro: smart-extractor: support target ${matchId.slice(0, 8)} vanished during update`); + return "target-missing"; + } this.log(`memory-pro: smart-extractor: support [${contextLabel || "general"}] on ${matchId.slice(0, 8)} — ${reason}`); + return "supported"; } /** * Handle CONTEXTUALIZE: create a new entry that adds situational nuance, @@ -1962,7 +2437,11 @@ export class SmartExtractor { contexts: contextLabel ? [contextLabel] : [], relations: [{ type: "contextualizes", targetId: matchId }], }, admissionAudit)); - const entry_c = { + const entry_c = this.externalVerdictEntry(candidate, { + state: "confirmed", + contexts: contextLabel ? [contextLabel] : [], + relations: [{ type: "contextualizes", targetId: matchId }], + }) ?? { text: candidate.abstract, vector, category: storeCategory, @@ -2014,7 +2493,11 @@ export class SmartExtractor { contexts: contextLabel ? [contextLabel] : [], relations: [{ type: "contradicts", targetId: matchId }], }, admissionAudit)); - const entry_d = { + const entry_d = this.externalVerdictEntry(candidate, { + state: "confirmed", + contexts: contextLabel ? [contextLabel] : [], + relations: [{ type: "contradicts", targetId: matchId }], + }) ?? { text: candidate.abstract, vector, category: storeCategory, @@ -2034,11 +2517,48 @@ export class SmartExtractor { // -------------------------------------------------------------------------- // Store Helper // -------------------------------------------------------------------------- + /** + * Entry-shape overrides for candidates persisted on behalf of another + * lane (persistGatedCandidates): the reflection writer supplies its own + * store entry (reflection metadata, decay model, importance) while the + * dedup/merge pipeline stays byte-identical to extraction's. Keyed by + * candidate object identity, so extraction's own candidates can never + * collide with an external lane's builders. + */ + externalEntryBuilders = new WeakMap(); + /** + * Verdict rows for externally-gated candidates must originate from the + * caller's own entry — reflection provenance, heading, mapped kind, decay + * model, importance, and admission audit all live there — with only the + * verdict-specific fields layered on top. Returns null for ordinary + * extraction candidates, which keep the auto-capture shape. + */ + externalVerdictEntry(candidate, overlay) { + const ext = this.externalEntryBuilders.get(candidate); + if (!ext?.prebuilt) { + return null; + } + const base = ext.prebuilt; + let meta = {}; + if (typeof base.metadata === "string" && base.metadata.length > 0) { + try { + meta = JSON.parse(base.metadata); + } + catch { + meta = {}; + } + } + return { ...base, metadata: JSON.stringify({ ...meta, ...overlay }) }; + } /** * Build a memory entry from candidate data (without writing). * Used by batch creation to reduce lock acquisitions. */ buildStoreEntry(candidate, vector, sessionKey, targetScope, admissionAudit) { + const external = this.externalEntryBuilders.get(candidate); + if (external) { + return external.prebuilt ?? external.build(vector); + } const storeCategory = this.mapToStoreCategory(candidate.category); const classifyText = candidate.content || candidate.abstract; const metadata = stringifySmartMetadata(buildSmartMetadata({ diff --git a/index.ts b/index.ts index 0396503ed..a437e346c 100644 --- a/index.ts +++ b/index.ts @@ -5949,8 +5949,13 @@ const memoryLanceDBProPlugin = { const MAX_MAPPED_ENTRIES = 100; const mappedReflectionMemories = extractInjectableReflectionMappedMemoryItems(reflectionText); const mappedEntries: Array<{ text: string; vector: number[]; importance: number; category: string; scope: string; metadata: string }> = []; - // Per-row embed + near-duplicate pre-check first, collecting the - // gate-eligible rows so the whole burst can share one admission call. + const mappedGatedItems: Array<{ + candidate: import("./src/memory-categories.js").CandidateMemory; + vector: number[]; + buildEntry: (v: number[]) => Omit; + }> = []; + // Per-row embed first, collecting the gate-eligible rows so the + // whole burst can share one admission call. const gateEligible: Array<{ mapped: (typeof mappedReflectionMemories)[number]; vector: number[] }> = []; for (const mapped of mappedReflectionMemories) { if (gateEligible.length >= MAX_MAPPED_ENTRIES) { @@ -5966,29 +5971,26 @@ const memoryLanceDBProPlugin = { ); continue; } - let existing: Awaited> = []; - let searchFailed = false; - try { - existing = await store.vectorSearch(vector, 1, 0.1, [targetScope]); - } catch (err) { - api.logger.warn( - `memory-reflection: mapped memory duplicate pre-check failed, skip store: ${String(err)}`, - ); - searchFailed = true; - } - if (searchFailed) { - continue; - } - // Near-duplicate pre-check ahead of admission gating. This is the only dedup mapped - // rows get: a single vector-similarity threshold, direct skip, no LLM-mediated - // merge/contextualize/contradict decision. Extraction candidates own deduplicate() - // (src/smart-extractor.ts) is a genuinely different, richer pipeline (a 0.7 - // pre-filter feeding an LLM decision, not a single hard cutoff) - deliberately not - // reused here yet. AdmissionController's "pass_to_dedup" decision for a mapped row - // is therefore always treated as "admit, subject to this cheaper pre-check" below, - // not "route through the same merge pipeline extraction candidates get". - if (existing.length > 0 && existing[0].score > 0.95) { - continue; + // Extractor-backed runs take the SAME dedup/merge pipeline + // extraction candidates get (persistGatedCandidates below), so no + // bespoke similarity cutoff runs here. The no-extractor fallback + // keeps the historical near-duplicate pre-check, downgraded from + // fail-closed to fail-open: a search blip stores the row (worst + // case the near-duplicate lands as a separate row — this path + // only pre-checks, it has no merge step) instead of silently + // dropping it. + if (!smartExtractor) { + let existing: Awaited> = []; + try { + existing = await store.vectorSearch(vector, 1, 0.1, [targetScope]); + } catch (err) { + api.logger.warn( + `memory-reflection: mapped memory duplicate pre-check failed, storing without pre-check: ${String(err)}`, + ); + } + if (existing.length > 0 && existing[0].score > 0.95) { + continue; + } } gateEligible.push({ mapped, vector }); } @@ -6050,14 +6052,64 @@ const memoryLanceDBProPlugin = { } const metadata = JSON.stringify(baseMetadata); - mappedEntries.push({ - text: mapped.text, - vector, - importance, - category: getReflectionMappedStorageCategory(mapped.mappedKind), - scope: targetScope, - metadata, + if (smartExtractor) { + // Uniform pipeline: judge (done above) -> dedup -> merge-writer, + // identical to extraction candidates. The entry builder keeps + // the reflection metadata on CREATE-shaped verdicts. + mappedGatedItems.push({ + candidate: { + category: getReflectionMappedMemoryCategory(mapped.mappedKind), + abstract: mapped.text, + overview: `## ${mapped.heading}`, + content: mapped.text, + }, + vector, + buildEntry: (v: number[]) => ({ + text: mapped.text, + vector: v, + importance, + category: getReflectionMappedStorageCategory(mapped.mappedKind), + scope: targetScope, + metadata, + }), + }); + } else { + mappedEntries.push({ + text: mapped.text, + vector, + importance, + category: getReflectionMappedStorageCategory(mapped.mappedKind), + scope: targetScope, + metadata, + }); + } + } + if (smartExtractor && mappedGatedItems.length > 0) { + const gatedResult = await smartExtractor.persistGatedCandidates(mappedGatedItems, { + sessionKey, + targetScope, + scopeFilter: [targetScope], + agentId: ownerAgentId, + conversationText: conversation, }); + api.logger.info( + `memory-reflection: mapped rows through uniform pipeline: ${gatedResult.createdEntries.length} created, ${gatedResult.stats.merged} merged, ${gatedResult.stats.skipped} skipped`, + ); + if (mdMirror) { + for (const stored of gatedResult.createdEntries) { + let heading = "unknown"; + try { + const storedMeta = stored.metadata ? JSON.parse(stored.metadata) : {}; + heading = storedMeta._reflectionHeading ?? "unknown"; + } catch { + api.logger.warn(`memory-reflection: failed to parse stored metadata for entry ${stored.id}, using "unknown"`); + } + await mdMirror( + { text: stored.text, category: stored.category, scope: stored.scope, timestamp: stored.timestamp }, + { source: `reflection:${heading}`, agentId: sourceAgentId }, + ); + } + } } if (mappedEntries.length > 0) { const storedEntries = await store.bulkStore(mappedEntries, ({ index, reason }) => { diff --git a/package.json b/package.json index 4a6dd3848..95d4a6520 100644 --- a/package.json +++ b/package.json @@ -33,7 +33,7 @@ "skills/**/*.md" ], "scripts": { - "test": "node test/embedder-error-hints.test.mjs && node --test test/embedder-max-input-chars.test.mjs && node test/cjk-recursion-regression.test.mjs && node test/extraction-prompt-structural-noise.test.mjs && node test/i18n-memory-triggers.test.mjs && node test/migrate-legacy-schema.test.mjs && node --test test/config-session-strategy-migration.test.mjs && node --test test/scope-access-undefined.test.mjs && node --test test/reflection-bypass-hook.test.mjs && node --test test/reflection-unattributed-session-read.test.mjs && node --test test/smart-extractor-scope-filter.test.mjs && node --test test/store-empty-scope-filter.test.mjs && node --test test/recall-text-cleanup.test.mjs && node test/update-consistency-lancedb.test.mjs && node --test test/strip-envelope-metadata.test.mjs && node test/cli-smoke.mjs && node test/functional-e2e.mjs && node --test test/per-agent-auto-recall.test.mjs && node test/retriever-rerank-regression.mjs && node test/smart-memory-lifecycle.mjs && node test/smart-extractor-branches.mjs && node --test test/smart-extractor-noise-gating.test.mjs && node test/memory-capability-runtime.test.mjs && node --test test/startup-health-diagnostics.test.mjs && node test/corpus-indexer.test.mjs && node --test test/regex-fallback-bulk-store.test.mjs && node test/plugin-manifest-regression.mjs && node --test test/dreaming-engine.test.mjs && node --test test/session-summary-before-reset.test.mjs && node --test test/sync-plugin-version.test.mjs && node test/smart-metadata-v2.mjs && node test/vector-search-cosine.test.mjs && node test/context-support-e2e.mjs && node test/temporal-facts.test.mjs && node test/memory-update-supersede.test.mjs && node test/memory-update-metadata-refresh.test.mjs && node test/memory-upgrader-diagnostics.test.mjs && node --test test/llm-api-key-client.test.mjs && node --test test/llm-oauth-client.test.mjs && node --test test/cli-oauth-login.test.mjs && node --test test/workflow-fork-guards.test.mjs && node --test test/clawteam-scope.test.mjs && node --test test/cross-process-lock.test.mjs && node --test test/preference-slots.test.mjs && node test/is-latest-auto-supersede.test.mjs && node --test test/temporal-awareness.test.mjs && node --test test/command-reflection-guard.test.mjs && node --test test/tier1-counters.test.mjs && node --test test/startup-check-timeout.test.mjs && node --test test/memory-subsession-prompt-hooks.test.mjs && node --test test/read-consistency-interval.test.mjs && node --test test/reflection-distiller-hook-skip.test.mjs && node --test test/register-scope-dedup.test.mjs && node --test test/raw-run-distiller-hooks.test.mjs && node --test test/autocapture-watermark-reset.test.mjs && node --test test/autocapture-internal-session-guard.test.mjs && node --test test/memory-categories-storage-map.test.mjs && node --test test/delete-invalidate-reflection-caches.test.mjs && node --test test/reflection-mapped-rows-admission.test.mjs && node --test test/smart-metadata-source-classification.test.mjs && node --test test/reflection-embed-transient-retry.test.mjs && node --test test/scope-owner-leak-hardening.test.mjs && node --test test/isOwnedByAgent.test.mjs && node --test test/typed-array-vector-fetch.test.mjs && node --test test/extraction-grounding-register.test.mjs && node test/grounding-rejudge.test.mjs && node --test test/reverse-map-legacy-category.test.mjs && node --test test/reflection-mapped-category-stamping.test.mjs && node --test test/memory-upgrader-category-normalization.test.mjs && node --test test/autocapture-fallback-gating.test.mjs && node --test test/prompt-architecture.test.mjs && node test/extraction-category-rubric.test.mjs && node --test test/admission-control-batch-utility.test.mjs && node --test test/smart-extractor-batch-admission.test.mjs && node --test test/admission-control-prompt-shape.test.mjs && node --test test/smart-extractor-merge-accounting.test.mjs && node --test test/admission-utility-veto.test.mjs && node --test test/cli-subcommand-attachment.test.mjs && node --test test/admission-lane-model-affinity.test.mjs && node --test test/admission-model-resolution.test.mjs && node --test test/admission-controller-standalone.test.mjs && node --test test/smart-extractor-admission-controller-injection.test.mjs && node --test test/admission-without-smart-extraction.test.mjs && node --test test/llm-thinklevel.test.mjs && node --test test/memory-id-prefix-resolution.test.mjs && node --test test/manual-store-supersede.test.mjs && node --test test/extraction-transcript-speaker-tags.test.mjs && node --test test/session-compressor.test.mjs && node --test test/reflection-derived-cache-invalidation.test.mjs && node --test test/reflection-tagged-input.test.mjs", + "test": "node test/embedder-error-hints.test.mjs && node --test test/embedder-max-input-chars.test.mjs && node test/cjk-recursion-regression.test.mjs && node test/extraction-prompt-structural-noise.test.mjs && node test/i18n-memory-triggers.test.mjs && node test/migrate-legacy-schema.test.mjs && node --test test/config-session-strategy-migration.test.mjs && node --test test/scope-access-undefined.test.mjs && node --test test/reflection-bypass-hook.test.mjs && node --test test/reflection-unattributed-session-read.test.mjs && node --test test/smart-extractor-scope-filter.test.mjs && node --test test/store-empty-scope-filter.test.mjs && node --test test/recall-text-cleanup.test.mjs && node test/update-consistency-lancedb.test.mjs && node --test test/strip-envelope-metadata.test.mjs && node test/cli-smoke.mjs && node test/functional-e2e.mjs && node --test test/per-agent-auto-recall.test.mjs && node test/retriever-rerank-regression.mjs && node test/smart-memory-lifecycle.mjs && node test/smart-extractor-branches.mjs && node --test test/smart-extractor-noise-gating.test.mjs && node test/memory-capability-runtime.test.mjs && node --test test/startup-health-diagnostics.test.mjs && node test/corpus-indexer.test.mjs && node --test test/regex-fallback-bulk-store.test.mjs && node test/plugin-manifest-regression.mjs && node --test test/dreaming-engine.test.mjs && node --test test/session-summary-before-reset.test.mjs && node --test test/sync-plugin-version.test.mjs && node test/smart-metadata-v2.mjs && node test/vector-search-cosine.test.mjs && node test/context-support-e2e.mjs && node test/temporal-facts.test.mjs && node test/memory-update-supersede.test.mjs && node test/memory-update-metadata-refresh.test.mjs && node test/memory-upgrader-diagnostics.test.mjs && node --test test/llm-api-key-client.test.mjs && node --test test/llm-oauth-client.test.mjs && node --test test/cli-oauth-login.test.mjs && node --test test/workflow-fork-guards.test.mjs && node --test test/clawteam-scope.test.mjs && node --test test/cross-process-lock.test.mjs && node --test test/preference-slots.test.mjs && node test/is-latest-auto-supersede.test.mjs && node --test test/temporal-awareness.test.mjs && node --test test/command-reflection-guard.test.mjs && node --test test/tier1-counters.test.mjs && node --test test/startup-check-timeout.test.mjs && node --test test/memory-subsession-prompt-hooks.test.mjs && node --test test/read-consistency-interval.test.mjs && node --test test/reflection-distiller-hook-skip.test.mjs && node --test test/register-scope-dedup.test.mjs && node --test test/raw-run-distiller-hooks.test.mjs && node --test test/autocapture-watermark-reset.test.mjs && node --test test/autocapture-internal-session-guard.test.mjs && node --test test/memory-categories-storage-map.test.mjs && node --test test/delete-invalidate-reflection-caches.test.mjs && node --test test/reflection-mapped-rows-admission.test.mjs && node --test test/smart-metadata-source-classification.test.mjs && node --test test/reflection-embed-transient-retry.test.mjs && node --test test/scope-owner-leak-hardening.test.mjs && node --test test/isOwnedByAgent.test.mjs && node --test test/typed-array-vector-fetch.test.mjs && node --test test/extraction-grounding-register.test.mjs && node test/grounding-rejudge.test.mjs && node --test test/reverse-map-legacy-category.test.mjs && node --test test/reflection-mapped-category-stamping.test.mjs && node --test test/memory-upgrader-category-normalization.test.mjs && node --test test/autocapture-fallback-gating.test.mjs && node --test test/prompt-architecture.test.mjs && node test/extraction-category-rubric.test.mjs && node --test test/admission-control-batch-utility.test.mjs && node --test test/smart-extractor-batch-admission.test.mjs && node --test test/admission-control-prompt-shape.test.mjs && node --test test/smart-extractor-merge-accounting.test.mjs && node --test test/admission-utility-veto.test.mjs && node --test test/cli-subcommand-attachment.test.mjs && node --test test/admission-lane-model-affinity.test.mjs && node --test test/admission-model-resolution.test.mjs && node --test test/admission-controller-standalone.test.mjs && node --test test/smart-extractor-admission-controller-injection.test.mjs && node --test test/admission-without-smart-extraction.test.mjs && node --test test/llm-thinklevel.test.mjs && node --test test/memory-id-prefix-resolution.test.mjs && node --test test/manual-store-supersede.test.mjs && node --test test/extraction-transcript-speaker-tags.test.mjs && node --test test/session-compressor.test.mjs && node --test test/reflection-derived-cache-invalidation.test.mjs && node --test test/reflection-tagged-input.test.mjs && node --test test/reflection-mapped-uniform-pipeline.test.mjs", "test:cli-smoke": "node scripts/run-ci-tests.mjs --group cli-smoke", "test:core-regression": "node scripts/run-ci-tests.mjs --group core-regression", "test:storage-and-schema": "node scripts/run-ci-tests.mjs --group storage-and-schema", diff --git a/scripts/ci-test-manifest.mjs b/scripts/ci-test-manifest.mjs index 0d296b243..9b73716d7 100644 --- a/scripts/ci-test-manifest.mjs +++ b/scripts/ci-test-manifest.mjs @@ -117,6 +117,7 @@ export const CI_TEST_MANIFEST = [ // Delete/delete-bulk must synchronously invalidate in-process reflection read caches { group: "core-regression", runner: "node", file: "test/delete-invalidate-reflection-caches.test.mjs", args: ["--test"] }, { group: "core-regression", runner: "node", file: "test/reflection-mapped-rows-admission.test.mjs", args: ["--test"] }, + { group: "core-regression", runner: "node", file: "test/reflection-mapped-uniform-pipeline.test.mjs", args: ["--test"] }, { group: "core-regression", runner: "node", file: "test/reflection-embed-transient-retry.test.mjs", args: ["--test"] }, { group: "storage-and-schema", runner: "node", file: "test/smart-metadata-source-classification.test.mjs", args: ["--test"] }, { group: "core-regression", runner: "node", file: "test/scope-owner-leak-hardening.test.mjs", args: ["--test"] }, diff --git a/src/smart-extractor.ts b/src/smart-extractor.ts index f51092fbe..fda5ab9f9 100644 --- a/src/smart-extractor.ts +++ b/src/smart-extractor.ts @@ -73,6 +73,41 @@ type PendingMergeAddition = { contextLabel?: string; admissionAudit?: AdmissionAuditRecord; }; + +/** + * The caller's own admission audit, as carried inside an externally-built + * entry's metadata. Used by the gated-candidate lane so downstream verdict + * handling persists the real gate record, never a synthetic marker. + */ +function parseEntryAdmissionAudit( + entry: Omit, +): AdmissionAuditRecord | undefined { + const raw = entry.metadata; + if (typeof raw !== "string" || raw.length === 0) { + return undefined; + } + try { + const meta = JSON.parse(raw); + // Production external builders (the reflection mapped lane) persist the + // gate record as a nested JSON string under admission_audit. + const audit = meta.admission_audit; + const parsed = typeof audit === "string" && audit.length > 0 ? JSON.parse(audit) : audit; + // Fail-open gate markers ({provenance, failedOpen, reason, error}) are + // evidence of a skipped evaluation, not an audit: adopting one would let + // MERGE/SUPPORT overwrite a target's complete audit with it. + if ( + parsed && + typeof parsed === "object" && + (parsed as Record).version === "amac-v1" && + typeof (parsed as Record).decision === "string" + ) { + return parsed as AdmissionAuditRecord; + } + return undefined; + } catch { + return undefined; + } +} /** * One deferred merge write, queued while candidates are processed and * flushed through the single batched merge-writer call afterwards. Multiple @@ -409,6 +444,26 @@ function normalizeRegisterToken(value: unknown): string { // ============================================================================ const SIMILARITY_THRESHOLD = 0.7; +const NO_SIMILAR_MEMORIES_REASON = "No similar memories found"; + +// Burst-lane identity of a row's serialized metadata: the mapped kind wins, +// then the reflection heading; anything else is the shared empty lane. +function laneFromMetadata(rawMeta: unknown): string { + if (typeof rawMeta === "string" && rawMeta.length > 0) { + try { + const meta = JSON.parse(rawMeta) as Record; + if (typeof meta.mappedKind === "string" && meta.mappedKind.length > 0) { + return meta.mappedKind; + } + if (typeof meta._reflectionHeading === "string" && meta._reflectionHeading.length > 0) { + return meta._reflectionHeading; + } + } catch { + // unparseable metadata falls through to the empty lane + } + } + return ""; +} const MAX_SIMILAR_FOR_PROMPT = 3; const MAX_MEMORIES_PER_EXTRACTION = 5; /** Max candidates decided in one batched dedup LLM call; larger batches are chunked. */ @@ -912,6 +967,407 @@ export class SmartExtractor { return stats; } + /** + * Uniform-pipeline entry for candidates whose extraction AND admission + * already happened in another lane (the reflection writer's mapped rows: + * distilled by the reflection model, gated by gateMappedReflectionEntries). + * From here on they take exactly the extraction candidates' path -- + * batched dedup decider, verdict handling, batched merge writer, bulk + * create -- so a duplicate mapped row MERGES into its target instead of + * landing beside it. + * + * Each item supplies its own store-entry builder: a CREATE-shaped verdict + * persists the caller's entry (reflection metadata intact), while + * merge/supersede/support/contextualize/contradict operate on existing + * rows through the shared machinery. Callers own persistence + * notifications for created rows (the returned entries), keeping their + * lane-specific journal labels. + */ + async persistGatedCandidates( + items: Array<{ + candidate: CandidateMemory; + vector: number[]; + buildEntry: (vector: number[]) => StoreEntry; + }>, + options: { + sessionKey?: string; + targetScope: string; + scopeFilter?: string[]; + agentId?: string; + conversationText?: string; + }, + ): Promise<{ stats: ExtractionStats; createdEntries: MemoryEntry[] }> { + const stats: ExtractionStats = { created: 0, merged: 0, skipped: 0, boundarySkipped: 0 }; + const sessionKey = options.sessionKey ?? "reflection"; + const targetScope = options.targetScope; + const scopeFilter = options.scopeFilter ?? [targetScope]; + const conversationText = options.conversationText ?? ""; + + for (const item of items) { + const prebuilt = item.buildEntry(item.vector); + this.externalEntryBuilders.set(item.candidate, { + build: item.buildEntry, + prebuilt, + audit: parseEntryAdmissionAudit(prebuilt), + }); + } + + // Admission already ran in the caller's gate; the evaluation handed to + // processCandidate only tells it not to score again. Its audit is the + // CALLER'S OWN record (parsed from the built entry) — never a synthetic + // stub — so anything persisted downstream carries the real gate audit. + const preGatedFor = (candidate: CandidateMemory): AdmissionEvaluation => + ({ + decision: "pass_to_dedup", + audit: this.externalEntryBuilders.get(candidate)?.audit, + }) as unknown as AdmissionEvaluation; + + // Same-burst twin guard: collapse EXACT normalized duplicates within one + // caller lane. The lane identity comes from the prebuilt entry's mapped + // kind (its reflection heading as fallback): lessons and decisions share + // one candidate category while carrying different kinds, headings, + // importance, and decay, so a category+text key would deterministically + // drop the later lane's row. Anything short of textual identity within a + // lane proceeds to the dedup judge. + const burstLaneOf = (candidate: CandidateMemory): string => + laneFromMetadata(this.externalEntryBuilders.get(candidate)?.prebuilt?.metadata); + const seenBurstKeys = new Set(); + const surviving: typeof items = []; + for (const item of items) { + const key = JSON.stringify([ + burstLaneOf(item.candidate), + item.candidate.category, + item.candidate.abstract.toLowerCase().replace(/\s+/g, " ").trim(), + ]); + if (seenBurstKeys.has(key)) { + stats.skipped++; + this.log( + `memory-pro: smart-extractor: gated-candidate burst twin dropped [${item.candidate.category}]`, + ); + continue; + } + seenBurstKeys.add(key); + surviving.push(item); + } + + // Same-lane siblings earlier in one burst act as virtual dedup + // neighbors: with no similar row in the store yet, two related mapped + // rows arriving together would otherwise BOTH short-circuit to CREATE + // and the semantic judge would never see the pair. A verdict against a + // sibling resolves after bulkStore assigns the sibling's real id, then + // reuses the normal merge/support machinery. + const BURST_SIBLING_PREFIX = "burst-sibling:"; + const laneKeyOf = (candidate: CandidateMemory): string => + JSON.stringify([burstLaneOf(candidate), candidate.category]); + const cosineOf = (a: number[], b: number[]): number => { + if (a.length === 0 || a.length !== b.length) { + return 0; + } + let dot = 0; + let normA = 0; + let normB = 0; + for (let d = 0; d < a.length; d++) { + dot += a[d] * b[d]; + normA += a[d] * a[d]; + normB += b[d] * b[d]; + } + if (normA === 0 || normB === 0) { + return 0; + } + return dot / (Math.sqrt(normA) * Math.sqrt(normB)); + }; + const burstSiblingsFor = (index: number): MemorySearchResult[] => { + const { candidate, vector } = surviving[index]; + if (!vector || vector.length === 0) { + return []; + } + const lane = laneKeyOf(candidate); + const out: MemorySearchResult[] = []; + for (let j = 0; j < index; j++) { + const sibling = surviving[j]; + if (laneKeyOf(sibling.candidate) !== lane) { + continue; + } + const score = cosineOf(vector, sibling.vector || []); + if (score < SIMILARITY_THRESHOLD) { + continue; + } + const prebuilt = this.externalEntryBuilders.get(sibling.candidate)?.prebuilt; + const entryCategory = + typeof prebuilt?.category === "string" + ? (prebuilt.category as import("./store.js").MemoryEntry["category"]) + : this.mapToStoreCategory(sibling.candidate.category); + out.push({ + entry: { + id: `${BURST_SIBLING_PREFIX}${j}`, + text: sibling.candidate.abstract, + vector: [], + category: entryCategory, + scope: typeof prebuilt?.scope === "string" ? prebuilt.scope : targetScope, + importance: typeof prebuilt?.importance === "number" ? prebuilt.importance : 0.8, + timestamp: Date.now(), + metadata: typeof prebuilt?.metadata === "string" ? prebuilt.metadata : "{}", + }, + score, + }); + } + return out; + }; + + const precomputedDedups = new Map(); + const dedupLlmItems: Array<{ + index: number; + candidate: CandidateMemory; + topSimilar: MemorySearchResult[]; + }> = []; + for (let i = 0; i < surviving.length; i++) { + const { candidate, vector } = surviving[i]; + const siblings = burstSiblingsFor(i); + try { + const prefilter = await this.dedupPrefilter(candidate, vector, scopeFilter); + const emptyStoreShortCircuit = prefilter.shortCircuit?.reason === NO_SIMILAR_MEMORIES_REASON; + if (prefilter.shortCircuit && !(siblings.length > 0 && emptyStoreShortCircuit)) { + // Domain short-circuits (e.g. the preference-slot guard) stay + // authoritative even when burst siblings exist; only the plain + // "nothing similar stored yet" bypass yields to sibling context. + precomputedDedups.set(i, prefilter.shortCircuit); + } else { + const topSimilar = [...prefilter.topSimilar, ...siblings] + .sort((a, b) => b.score - a.score) + .slice(0, 5); + dedupLlmItems.push({ index: i, candidate, topSimilar }); + } + } catch (err) { + this.log( + `memory-pro: smart-extractor: gated-candidate dedup pre-filter failed, deferring to inline dedup: ${String(err)}`, + ); + } + } + if (dedupLlmItems.length > 0) { + const verdicts = await this.llmDedupDecisionBatch(dedupLlmItems); + dedupLlmItems.forEach((item, i) => { + precomputedDedups.set(item.index, verdicts[i]); + }); + } + + const createEntries: StoreEntry[] = []; + const pendingSupersedeInvalidations: PendingSupersedeInvalidation[] = []; + const pendingMerges: PendingMergeJob[] = []; + const pendingSiblingVerdicts: Array<{ + candidate: CandidateMemory; + vector: number[]; + siblingIndex: number; + decision: "merge" | "support"; + reason: string; + contextLabel?: string; + }> = []; + const createSlotBySurviving = new Map(); + + for (let i = 0; i < surviving.length; i++) { + const { candidate, vector } = surviving[i]; + const pre = precomputedDedups.get(i); + if (pre?.matchId && pre.matchId.startsWith(BURST_SIBLING_PREFIX)) { + const siblingIndex = Number(pre.matchId.slice(BURST_SIBLING_PREFIX.length)); + const resolvable = Number.isInteger(siblingIndex) && siblingIndex >= 0 && siblingIndex < i; + if (pre.decision === "skip" && resolvable) { + stats.skipped++; + this.log( + `memory-pro: smart-extractor: gated candidate judged same-burst duplicate of an earlier sibling [${candidate.category}]`, + ); + continue; + } + if ((pre.decision === "merge" || pre.decision === "support") && resolvable) { + pendingSiblingVerdicts.push({ + candidate, + vector, + siblingIndex, + decision: pre.decision, + reason: pre.reason, + contextLabel: pre.contextLabel, + }); + continue; + } + // Any other verdict against a not-yet-persisted sibling row keeps + // the caller's entry: fail open to a plain create. + precomputedDedups.set(i, { + decision: "create", + reason: "sibling verdict fallback (unsupported decision for a pending row)", + }); + } + const createCountBefore = createEntries.length; + try { + await this.processCandidate( + candidate, + conversationText, + sessionKey, + stats, + targetScope, + scopeFilter, + vector, + createEntries, + pendingSupersedeInvalidations, + options.agentId, + preGatedFor(candidate), + precomputedDedups.get(i), + pendingMerges, + ); + if (createEntries.length === createCountBefore + 1) { + createSlotBySurviving.set(i, createCountBefore); + } + } catch (err) { + this.log( + `memory-pro: smart-extractor: failed to process gated candidate [${candidate.category}]: ${String(err)}`, + ); + // Fail open: this candidate already passed the caller's admission + // gate, so a processing failure (dedup search, verdict handling) + // must not silently drop it — store the caller-built row as-is. + const ext = this.externalEntryBuilders.get(candidate); + if (ext) { + createEntries.push(ext.prebuilt ?? ext.build(vector)); + createSlotBySurviving.set(i, createEntries.length - 1); + stats.created++; + this.log( + `memory-pro: smart-extractor: fail-open create for gated candidate after processing failure [${candidate.category}]`, + ); + } + } + } + + await this.flushPendingMerges(pendingMerges, stats, createEntries); + + let createdEntries: MemoryEntry[] = []; + if (createEntries.length > 0) { + const stored = await this.bulkStoreAndValidate(createEntries); + if (stored) { + createdEntries = stored; + await this.applyPendingSupersedeInvalidations(stored, pendingSupersedeInvalidations); + } else if (pendingSupersedeInvalidations.length > 0) { + this.log( + "memory-pro: smart-extractor: gated-candidate supersede invalidation skipped because bulkStore() did not return created entries", + ); + } + } + + // Deferred same-burst verdicts: the sibling's row now has a real id, so + // merge/support resolve through the normal machinery. Anything that + // cannot be resolved keeps the caller's row (fail open to create). + if (pendingSiblingVerdicts.length > 0) { + const claimedIds = new Set(); + const storedIdForSurviving = (survivingIndex: number): string | undefined => { + const slot = createSlotBySurviving.get(survivingIndex); + if (slot === undefined) { + return undefined; + } + if (createdEntries.length === createEntries.length) { + const id = createdEntries[slot]?.id; + if (id) { + claimedIds.add(id); + } + return id; + } + // bulkStore may filter entries, shifting positions: fall back to the + // first unclaimed row with the same text AND the same lane/category + // identity — identical text is legal across lanes, so a text-only + // match could bind the verdict to another lane's row. + const want = createEntries[slot]; + const hit = want + ? createdEntries.find( + (e) => + e.text === want.text && + e.category === want.category && + laneFromMetadata(e.metadata) === laneFromMetadata(want.metadata) && + !claimedIds.has(e.id), + ) + : undefined; + if (hit) { + claimedIds.add(hit.id); + } + return hit?.id; + }; + const followupMerges: PendingMergeJob[] = []; + const followupCreates: StoreEntry[] = []; + for (const pending of pendingSiblingVerdicts) { + const audit = this.externalEntryBuilders.get(pending.candidate)?.audit; + const failOpenCreate = async (why: string) => { + followupCreates.push( + await this.externalOrBuiltFallbackEntry(pending.candidate, targetScope, sessionKey, pending.vector, audit), + ); + stats.created++; + this.log( + `memory-pro: smart-extractor: ${why}, storing gated candidate as new [${pending.candidate.category}]`, + ); + }; + try { + const targetId = storedIdForSurviving(pending.siblingIndex); + if (!targetId) { + await failOpenCreate("same-burst sibling row not persisted"); + continue; + } + if (pending.decision === "support") { + const outcome = await this.handleSupport( + targetId, + { session: sessionKey, timestamp: Date.now() }, + pending.reason, + pending.contextLabel, + scopeFilter, + audit, + ); + if (outcome === "supported") { + stats.supported = (stats.supported ?? 0) + 1; + } else { + await failOpenCreate("same-burst support target vanished"); + } + continue; + } + const queued = await this.queueMergeJob( + followupMerges, + pending.candidate, + targetId, + targetScope, + scopeFilter, + pending.contextLabel, + audit, + followupCreates, + options.agentId, + ); + if (queued === "created") { + stats.created++; + } + } catch (err) { + // A deferred verdict degrades ALONE: the candidate already passed + // the caller's admission gate, so a throwing store read/write on + // one resolution must fall open to that candidate's own create — + // never reject the whole persistence call, discard follow-up work + // queued by earlier verdicts, or skip the verdicts still pending. + // Push paths above enqueue only after their awaits resolve, so a + // caught verdict has enqueued nothing yet and the fallback row + // lands exactly once. + this.log( + `memory-pro: smart-extractor: deferred sibling ${pending.decision} failed: ${String(err)}`, + ); + try { + await failOpenCreate(`deferred sibling ${pending.decision} unresolved`); + } catch (fallbackErr) { + this.log( + `memory-pro: smart-extractor: fail-open create failed for a deferred sibling verdict [${pending.candidate.category}]: ${String(fallbackErr)}`, + ); + } + } + } + if (followupMerges.length > 0) { + await this.flushPendingMerges(followupMerges, stats, followupCreates); + } + if (followupCreates.length > 0) { + const extra = await this.bulkStoreAndValidate(followupCreates); + if (extra) { + createdEntries = createdEntries.concat(extra); + } + } + } + + return { stats, createdEntries }; + } + // -------------------------------------------------------------------------- // Embedding Noise Pre-Filter // -------------------------------------------------------------------------- @@ -1851,8 +2307,15 @@ export class SmartExtractor { case "support": if (dedupResult.matchId) { - await this.handleSupport(dedupResult.matchId, { session: sessionKey, timestamp: Date.now() }, dedupResult.reason, dedupResult.contextLabel, scopeFilter, admission?.audit); - stats.supported = (stats.supported ?? 0) + 1; + const supportOutcome = await this.handleSupport(dedupResult.matchId, { session: sessionKey, timestamp: Date.now() }, dedupResult.reason, dedupResult.contextLabel, scopeFilter, admission?.audit); + if (supportOutcome === "supported") { + stats.supported = (stats.supported ?? 0) + 1; + } else { + // Target vanished mid-flight: same semantics as a support verdict + // with no target — the candidate lands as a new row. + createEntries?.push(await this.externalOrBuiltFallbackEntry(candidate, targetScope, sessionKey, vector, admission?.audit)); + stats.created++; + } } else { createEntries?.push(this.buildStoreEntry(candidate, vector, sessionKey, targetScope, admission?.audit)); stats.created++; @@ -1946,7 +2409,7 @@ export class SmartExtractor { if (activeSimilar.length === 0) { return { - shortCircuit: { decision: "create", reason: "No similar memories found" }, + shortCircuit: { decision: "create", reason: NO_SIMILAR_MEMORIES_REASON }, topSimilar: [], }; } @@ -2302,7 +2765,7 @@ export class SmartExtractor { return "llm-failed"; } - await this.applyMergedContent( + const applied = await this.applyMergedContent( matchId, candidate.category, merged, @@ -2312,6 +2775,12 @@ export class SmartExtractor { admissionAudit, agentId, ); + if (applied === "target-missing") { + createEntries?.push( + await this.externalOrBuiltFallbackEntry(candidate, targetScope, "merge-fallback", undefined, admissionAudit), + ); + return "created"; + } return "merged"; } @@ -2321,6 +2790,30 @@ export class SmartExtractor { * exactly the fallback the inline merge path always used) and null is * returned so the caller can account for it as "created". */ + /** + * The CREATE row a candidate falls back to when its verdict target cannot + * be mutated: the caller's own prebuilt entry for externally gated + * candidates (shape, provenance, and audit stay the caller's), the + * standard auto-capture entry otherwise. + */ + private async externalOrBuiltFallbackEntry( + candidate: CandidateMemory, + targetScope: string, + sessionLabel: string, + vector?: number[], + admissionAudit?: AdmissionAuditRecord, + ): Promise { + const ext = this.externalEntryBuilders.get(candidate); + if (ext?.prebuilt) { + return ext.prebuilt; + } + const v = + vector && vector.length > 0 + ? vector + : (await this.embedder.embed(`${candidate.abstract} ${candidate.content}`)) || []; + return this.buildStoreEntry(candidate, v, sessionLabel, targetScope, admissionAudit); + } + private async readMergeTarget( candidate: CandidateMemory, matchId: string, @@ -2330,30 +2823,31 @@ export class SmartExtractor { ): Promise<{ abstract: string; overview: string; content: string } | null> { try { const existing = await this.store.getById(matchId, scopeFilter); - let abstract = ""; - let overview = ""; - let content = ""; - if (existing) { - const meta = parseSmartMetadata(existing.metadata, existing); - abstract = meta.l0_abstract || existing.text; - overview = meta.l1_overview || ""; - content = meta.l2_content || existing.text; - } - return { abstract, overview, content }; + if (!existing) { + // Target vanished between dedup and read: merging into a missing row + // would silently drop the candidate, so store it as new instead. + this.log( + `memory-pro: smart-extractor: merge target ${matchId.slice(0, 8)} no longer exists, storing as new`, + ); + createEntries?.push( + await this.externalOrBuiltFallbackEntry(candidate, targetScope, "merge-fallback"), + ); + return null; + } + const meta = parseSmartMetadata(existing.metadata, existing); + return { + abstract: meta.l0_abstract || existing.text, + overview: meta.l1_overview || "", + content: meta.l2_content || existing.text, + }; } catch { // Fallback: store as new this.log( `memory-pro: smart-extractor: could not read existing memory ${matchId}, storing as new`, ); - const vector = await this.embedder.embed( - `${candidate.abstract} ${candidate.content}`, + createEntries?.push( + await this.externalOrBuiltFallbackEntry(candidate, targetScope, "merge-fallback"), ); - createEntries?.push(this.buildStoreEntry( - candidate, - vector || [], - "merge-fallback", - targetScope, - )); return null; } } @@ -2411,20 +2905,63 @@ export class SmartExtractor { private async flushPendingMerges( pendingMerges: PendingMergeJob[], stats: ExtractionStats, + createEntries?: StoreEntry[], ): Promise { if (pendingMerges.length === 0) { return; } + // Extraction-lane additions degrade like the single-call merge failure + // path (nothing persisted, target untouched). Externally-gated additions + // must NOT disappear on a degraded merge: they already passed the + // caller's admission gate and were previously direct-stored, so they + // fall back to a create built from the caller's own entry. + const failOpenAdditions = (job: PendingMergeJob, why: string) => { + if (!createEntries) { + return; + } + for (const addition of job.additions) { + const ext = this.externalEntryBuilders.get(addition.candidate); + if (ext?.prebuilt) { + createEntries.push(ext.prebuilt); + stats.created++; + this.log( + `memory-pro: smart-extractor: merge ${why} — falling back to create for gated candidate [${addition.candidate.category}]`, + ); + } + } + }; + // A job carrying an externally gated addition mirrors under the caller's + // reflection provenance instead of the generic extraction label. + const mirrorSourceFor = (job: PendingMergeJob): string | undefined => { + for (const addition of job.additions) { + const ext = this.externalEntryBuilders.get(addition.candidate); + if (ext?.prebuilt) { + try { + const rawMeta = ext.prebuilt.metadata; + const meta = + typeof rawMeta === "string" && rawMeta.length > 0 + ? (JSON.parse(rawMeta) as Record) + : {}; + const heading = (meta as Record)._reflectionHeading; + return `reflection:${typeof heading === "string" && heading.length > 0 ? heading : "unknown"}`; + } catch { + return "reflection:unknown"; + } + } + } + return undefined; + }; const contents = await this.llmMergeContentBatch(pendingMerges); for (let i = 0; i < pendingMerges.length; i++) { const job = pendingMerges[i]; const merged = contents[i]; if (!merged) { this.log("memory-pro: smart-extractor: merge LLM failed, skipping merge"); + failOpenAdditions(job, "generation failed"); continue; } try { - await this.applyMergedContent( + const applied = await this.applyMergedContent( job.matchId, job.category, merged, @@ -2433,12 +2970,18 @@ export class SmartExtractor { job.additions.map((a) => a.contextLabel), job.additions[0]?.admissionAudit, job.agentId, + mirrorSourceFor(job), ); + if (applied === "target-missing") { + failOpenAdditions(job, "target vanished"); + continue; + } stats.merged += job.additions.length; } catch (err) { this.log( `memory-pro: smart-extractor: failed to apply merged content for ${job.matchId.slice(0, 8)}: ${String(err)}`, ); + failOpenAdditions(job, "apply failed"); } } } @@ -2517,20 +3060,38 @@ export class SmartExtractor { contextLabels: Array, admissionAudit: AdmissionAuditRecord | undefined, agentId: string | undefined, - ): Promise { + mirrorSource: string = "smart-extraction", + ): Promise<"updated" | "target-missing"> { // Re-embed the merged content const mergedText = `${merged.abstract} ${merged.content}`; const newVector = await this.embedder.embed(mergedText); - // Update existing memory via store.update() + // Update existing memory via store.update(). A target that vanished + // between dedup and this write must surface as "target-missing" so the + // caller can fall back — reporting success here would silently drop the + // candidate and emit a persistence notification for a write that never + // durably landed. const existing = await this.store.getById(matchId, scopeFilter); + if (!existing) { + this.log( + `memory-pro: smart-extractor: merge target ${matchId.slice(0, 8)} vanished before update`, + ); + return "target-missing"; + } + // A merge enriches the target's content; it never reclassifies the row. + // Stamping the incoming candidate's category would desync the metadata + // from the legacy category column, and list()'s two stages (column + // SQL-prefilter, then metadata validation) would drop the row from BOTH + // category views on a cross-category merge. + const existingMeta = parseSmartMetadata(existing.metadata, existing); + const targetCategory = (existingMeta.memory_category as MemoryCategory) || category; const metadata = stringifySmartMetadata( this.withAdmissionAudit( - buildSmartMetadata(existing ?? { text: merged.abstract }, { + buildSmartMetadata(existing, { l0_abstract: merged.abstract, l1_overview: merged.overview, l2_content: merged.content, - memory_category: category, + memory_category: targetCategory, tier: "working", confidence: 0.8, }), @@ -2538,7 +3099,7 @@ export class SmartExtractor { ), ); - await this.store.update( + const updated = await this.store.update( matchId, { text: merged.abstract, @@ -2547,15 +3108,21 @@ export class SmartExtractor { }, scopeFilter, ); + if (!updated) { + this.log( + `memory-pro: smart-extractor: merge target ${matchId.slice(0, 8)} vanished during update`, + ); + return "target-missing"; + } await this.notifyPersisted( { text: merged.abstract, - category: this.mapToStoreCategory(category), + category: this.mapToStoreCategory(targetCategory), scope: targetScope, timestamp: Date.now(), }, - "smart-extraction", + mirrorSource, agentId, ); @@ -2575,9 +3142,10 @@ export class SmartExtractor { } this.log( - `memory-pro: smart-extractor: merged [${category}]${contextLabel ? ` [${contextLabel}]` : ""} into ${matchId.slice(0, 8)}`, + `memory-pro: smart-extractor: merged [${targetCategory}]${contextLabel ? ` [${contextLabel}]` : ""} into ${matchId.slice(0, 8)}`, ); } + return "updated"; } /** @@ -2598,7 +3166,9 @@ export class SmartExtractor { ): Promise { const existing = await this.store.getById(matchId, scopeFilter); if (!existing) { - createEntries?.push(this.buildStoreEntry(candidate, vector || [], sessionKey, targetScope)); + createEntries?.push( + await this.externalOrBuiltFallbackEntry(candidate, targetScope, sessionKey, vector, admissionAudit), + ); return; } @@ -2608,7 +3178,15 @@ export class SmartExtractor { existingMeta.fact_key ?? deriveFactKey(candidate.category, candidate.abstract); const storeCategory = this.mapToStoreCategory(candidate.category); const supersedeClassifyText = candidate.content || candidate.abstract; - const entry: StoreEntry = { + const entry: StoreEntry = this.externalVerdictEntry(candidate, { + state: "confirmed", + valid_from: now, + fact_key: factKey, + supersedes: matchId, + relations: appendRelation([], { type: "supersedes", targetId: matchId }), + memory_temporal_type: classifyTemporal(supersedeClassifyText), + valid_until: inferExpiry(supersedeClassifyText), + }) ?? { text: candidate.abstract, vector, category: storeCategory, @@ -2745,24 +3323,36 @@ export class SmartExtractor { contextLabel?: string, scopeFilter?: string[], admissionAudit?: AdmissionAuditRecord, - ): Promise { + ): Promise<"supported" | "target-missing"> { const existing = await this.store.getById(matchId, scopeFilter); - if (!existing) return; + if (!existing) { + this.log( + `memory-pro: smart-extractor: support target ${matchId.slice(0, 8)} no longer exists`, + ); + return "target-missing"; + } const meta = parseSmartMetadata(existing.metadata, existing); const supportInfo = parseSupportInfo(meta.support_info); const updated = updateSupportStats(supportInfo, contextLabel, "support"); meta.support_info = updated; - await this.store.update( + const written = await this.store.update( matchId, { metadata: stringifySmartMetadata(this.withAdmissionAudit(meta, admissionAudit)) }, scopeFilter, ); + if (!written) { + this.log( + `memory-pro: smart-extractor: support target ${matchId.slice(0, 8)} vanished during update`, + ); + return "target-missing"; + } this.log( `memory-pro: smart-extractor: support [${contextLabel || "general"}] on ${matchId.slice(0, 8)} — ${reason}`, ); + return "supported"; } /** @@ -2802,7 +3392,11 @@ export class SmartExtractor { relations: [{ type: "contextualizes", targetId: matchId }], }, admissionAudit)); - const entry_c: StoreEntry = { + const entry_c: StoreEntry = this.externalVerdictEntry(candidate, { + state: "confirmed", + contexts: contextLabel ? [contextLabel] : [], + relations: [{ type: "contextualizes", targetId: matchId }], + }) ?? { text: candidate.abstract, vector, category: storeCategory, @@ -2878,7 +3472,11 @@ export class SmartExtractor { relations: [{ type: "contradicts", targetId: matchId }], }, admissionAudit)); - const entry_d: StoreEntry = { + const entry_d: StoreEntry = this.externalVerdictEntry(candidate, { + state: "confirmed", + contexts: contextLabel ? [contextLabel] : [], + relations: [{ type: "contradicts", targetId: matchId }], + }) ?? { text: candidate.abstract, vector, category: storeCategory, @@ -2906,6 +3504,50 @@ export class SmartExtractor { // Store Helper // -------------------------------------------------------------------------- + /** + * Entry-shape overrides for candidates persisted on behalf of another + * lane (persistGatedCandidates): the reflection writer supplies its own + * store entry (reflection metadata, decay model, importance) while the + * dedup/merge pipeline stays byte-identical to extraction's. Keyed by + * candidate object identity, so extraction's own candidates can never + * collide with an external lane's builders. + */ + private readonly externalEntryBuilders = new WeakMap< + CandidateMemory, + { + build: (vector: number[]) => StoreEntry; + prebuilt?: StoreEntry; + audit?: AdmissionAuditRecord; + } + >(); + + /** + * Verdict rows for externally-gated candidates must originate from the + * caller's own entry — reflection provenance, heading, mapped kind, decay + * model, importance, and admission audit all live there — with only the + * verdict-specific fields layered on top. Returns null for ordinary + * extraction candidates, which keep the auto-capture shape. + */ + private externalVerdictEntry( + candidate: CandidateMemory, + overlay: Record, + ): StoreEntry | null { + const ext = this.externalEntryBuilders.get(candidate); + if (!ext?.prebuilt) { + return null; + } + const base = ext.prebuilt; + let meta: Record = {}; + if (typeof base.metadata === "string" && base.metadata.length > 0) { + try { + meta = JSON.parse(base.metadata); + } catch { + meta = {}; + } + } + return { ...base, metadata: JSON.stringify({ ...meta, ...overlay }) }; + } + /** * Build a memory entry from candidate data (without writing). * Used by batch creation to reduce lock acquisitions. @@ -2917,6 +3559,10 @@ export class SmartExtractor { targetScope: string, admissionAudit?: AdmissionAuditRecord, ): Omit { + const external = this.externalEntryBuilders.get(candidate); + if (external) { + return external.prebuilt ?? external.build(vector); + } const storeCategory = this.mapToStoreCategory(candidate.category); const classifyText = candidate.content || candidate.abstract; const metadata = stringifySmartMetadata( diff --git a/test/reflection-mapped-rows-admission.test.mjs b/test/reflection-mapped-rows-admission.test.mjs index 76f0d0e04..394819e63 100644 --- a/test/reflection-mapped-rows-admission.test.mjs +++ b/test/reflection-mapped-rows-admission.test.mjs @@ -414,14 +414,15 @@ describe("gateMappedReflectionEntries (batched burst)", () => { }); }); -describe("production pipeline: parse distillate -> gate -> bulkStore (end to end)", () => { +describe("production pipeline: parse distillate -> gate -> persist (end to end)", () => { // Mirrors index.ts's runMemoryReflection loop shape exactly: parse mapped items from - // the distillate, gate each one, skip on reject, push admitted rows to bulkStore. - // A change to that orchestration (e.g. the item-2 bug this PR itself fixed, where - // pass_to_dedup was silently treated as unconditional admit with no way to reject) - // would be caught here without needing to drive the full agent_end hook and mock an - // embedded reflection LLM run just to reach this loop. - async function runMappedRowPipeline({ reflectionText, admissionController, conversationText }) { + // the distillate, gate each one, skip on reject, then persist admitted rows through + // SmartExtractor.persistGatedCandidates when smart extraction is on, or through the + // exclusive bulkStore fallback when it is off. A change to that orchestration (e.g. + // the item-2 bug an earlier revision fixed, where pass_to_dedup was silently treated + // as unconditional admit with no way to reject) would be caught here without needing + // to drive the full agent_end hook and mock an embedded reflection LLM run. + async function runMappedRowPipeline({ reflectionText, admissionController, conversationText, smartExtractor = null }) { const { extractInjectableReflectionMappedMemoryItems } = jiti("../src/reflection-slices.ts"); const bulkStoreCalls = []; const store = { @@ -450,6 +451,7 @@ describe("production pipeline: parse distillate -> gate -> bulkStore (end to end }); const mappedEntries = []; + const mappedGatedItems = []; const rejections = []; for (let i = 0; i < mappedReflectionMemories.length; i++) { const mapped = mappedReflectionMemories[i]; @@ -458,7 +460,24 @@ describe("production pipeline: parse distillate -> gate -> bulkStore (end to end rejections.push({ text: mapped.text, reason: gate.reason }); continue; } - mappedEntries.push({ text: mapped.text, category: getReflectionMappedStorageCategory(mapped.mappedKind), metadata: JSON.stringify({ admission_audit: gate.auditJson }) }); + const metadata = JSON.stringify({ admission_audit: gate.auditJson }); + if (smartExtractor) { + mappedGatedItems.push({ + candidate: { + category: getReflectionMappedMemoryCategory(mapped.mappedKind), + abstract: mapped.text, + overview: `## ${mapped.heading}`, + content: mapped.text, + }, + vector: [1, 0, 0], + buildEntry: (v) => ({ text: mapped.text, vector: v, category: getReflectionMappedStorageCategory(mapped.mappedKind), metadata }), + }); + } else { + mappedEntries.push({ text: mapped.text, category: getReflectionMappedStorageCategory(mapped.mappedKind), metadata }); + } + } + if (smartExtractor && mappedGatedItems.length > 0) { + await smartExtractor.persistGatedCandidates(mappedGatedItems, { targetScope: "global", scopeFilter: ["global"] }); } if (mappedEntries.length > 0) { await store.bulkStore(mappedEntries); @@ -466,7 +485,7 @@ describe("production pipeline: parse distillate -> gate -> bulkStore (end to end return { bulkStoreCalls, rejections }; } - it("a rejected mapped row is never passed to store.bulkStore, an admitted sibling still is", async () => { + it("no-extractor fallback: a rejected mapped row is never passed to store.bulkStore, an admitted sibling still is", async () => { const realConversation = "User: I mostly work on backend Python services.\nAssistant: noted."; const distillate = [ "## User model deltas (about the human)", @@ -505,7 +524,7 @@ describe("production pipeline: parse distillate -> gate -> bulkStore (end to end ); }); - it("when every mapped row is rejected, bulkStore is never called at all", async () => { + it("no-extractor fallback: when every mapped row is rejected, bulkStore is never called at all", async () => { const distillate = [ "## User model deltas (about the human)", "- User lives on Mars.", @@ -525,6 +544,45 @@ describe("production pipeline: parse distillate -> gate -> bulkStore (end to end assert.equal(rejections.length, 1); assert.equal(bulkStoreCalls.length, 0, "bulkStore must not be called when nothing was admitted"); }); + + it("routes admitted rows through the extractor's uniform pipeline and never calls bulkStore directly when smart extraction is on", async () => { + const distillate = [ + "## User model deltas (about the human)", + "- Operator prefers streaming test reporters for long suites.", + "## Decisions (durable)", + "- Decision: keep the deploy branch cut from a fresh master.", + ].join("\n"); + const controller = { + async evaluate() { + return { decision: "pass_to_dedup", audit: { decision: "pass_to_dedup", reason: "grounded" } }; + }, + }; + const persistCalls = []; + const smartExtractor = { + async persistGatedCandidates(items, options) { + persistCalls.push({ items, options }); + return { stats: { created: items.length, merged: 0, skipped: 0, boundarySkipped: 0 }, createdEntries: [] }; + }, + }; + + const { bulkStoreCalls, rejections } = await runMappedRowPipeline({ + reflectionText: distillate, + admissionController: controller, + conversationText: "User: I prefer streaming test reporters, and let's keep cutting deploy branches from a fresh master.", + smartExtractor, + }); + + assert.equal(rejections.length, 0); + assert.equal(persistCalls.length, 1, "one uniform-pipeline call per admitted burst"); + assert.equal(persistCalls[0].items.length, 2, "every admitted row rides the same burst"); + const first = persistCalls[0].items[0]; + assert.equal(first.candidate.category, getReflectionMappedMemoryCategory("user-model")); + assert.equal(typeof first.buildEntry, "function"); + const built = first.buildEntry([1, 0, 0]); + assert.equal(built.category, getReflectionMappedStorageCategory("user-model")); + assert.ok(JSON.parse(built.metadata).admission_audit, "the gate audit must survive into the built entry"); + assert.equal(bulkStoreCalls.length, 0, "the uniform route must never call store.bulkStore directly"); + }); }); describe("buildReflectionPrompt grounding discipline", () => { diff --git a/test/reflection-mapped-uniform-pipeline.test.mjs b/test/reflection-mapped-uniform-pipeline.test.mjs new file mode 100644 index 000000000..1bbfa50e6 --- /dev/null +++ b/test/reflection-mapped-uniform-pipeline.test.mjs @@ -0,0 +1,794 @@ +// Uniform pipeline for reflection mapped rows: after the reflection-lane +// admission gate, mapped rows take exactly the extraction candidates' path -- +// batched dedup decider, verdict handling, batched merge writer, bulk create -- +// via SmartExtractor.persistGatedCandidates, so a duplicate mapped row MERGES +// into its existing target instead of landing beside it, a judge outage +// creates instead of dropping, and a whole burst costs one batched dedup call. +// +// Fixtures are entirely synthetic; no real conversation data. +import { describe, it } from "node:test"; +import assert from "node:assert/strict"; +import { createHash } from "node:crypto"; +import jitiFactory from "jiti"; + +const jiti = jitiFactory(import.meta.url, { interopDefault: true }); +const { SmartExtractor } = jiti("../src/smart-extractor.ts"); +const { matchesMemoryCategoryFilter, resolveCategoryFilterCandidates } = jiti("../src/memory-categories.ts"); + +function vectorFor(text) { + const vec = []; + for (let d = 0; d < 16; d++) { + const digest = createHash("sha256").update(`${text}:${d}`).digest(); + vec.push(((digest.readUInt32BE(0) % 2000) - 1000) / 1000); + } + return vec; +} + +function makeEmbedder() { + return { + embed: async (text) => vectorFor(text), + embedBatch: async (texts) => texts.map((t) => vectorFor(t)), + }; +} + +function makeStore({ neighbors = [] } = {}) { + const rows = new Map(); + for (const n of neighbors) rows.set(n.id, n); + const updates = []; + const bulkStored = []; + return { + rows, + updates, + bulkStored, + async vectorSearch() { + return [...rows.values()].map((entry) => ({ entry, score: 0.85 })); + }, + async getById(id) { + return rows.get(id) ?? null; + }, + async update(id, patch) { + updates.push({ id, patch }); + return rows.get(id) ?? null; + }, + async store() {}, + async bulkStore(entries) { + bulkStored.push(...entries); + const stored = entries.map((e, i) => ({ ...e, id: `new-${rows.size + i + 1}`, timestamp: 1_700_000_500_000 })); + for (const s of stored) rows.set(s.id, s); + return stored; + }, + }; +} + +function neighborRow(id, text) { + return { + id, + text, + category: "patterns", + scope: "agent:probe", + importance: 0.8, + timestamp: 1_700_000_000_000, + metadata: JSON.stringify({ + memory_category: "patterns", + l0_abstract: text, + l1_overview: `## Existing\n${text}`, + l2_content: text, + }), + }; +} + +function makeLlm({ onDedupBatch, onMergeBatch } = {}) { + const calls = []; + return { + calls, + async completeJson(prompt, label) { + calls.push(label); + if (label === "dedup-decision-batch") { + if (!onDedupBatch) throw new Error("unexpected dedup-decision-batch call"); + return onDedupBatch(prompt); + } + if (label === "merge-memory-batch") { + if (!onMergeBatch) throw new Error("unexpected merge-memory-batch call"); + return onMergeBatch(prompt); + } + throw new Error(`unexpected llm call: ${label}`); + }, + }; +} + +function makeExtractor(store, llm, extraConfig = {}) { + return new SmartExtractor(store, makeEmbedder(), llm, { + user: "User", + extractMinMessages: 1, + extractMaxChars: 8000, + defaultScope: "agent:probe", + log() {}, + debugLog() {}, + ...extraConfig, + }); +} + +function reflectionItem(text, { category = "patterns", heading = "Agent model deltas (about the assistant/system)", mappedKind } = {}) { + const metadata = JSON.stringify({ + type: "memory-reflection-mapped", + memory_category: category, + _reflectionHeading: heading, + ...(mappedKind ? { mappedKind } : {}), + marker: "reflection-metadata-preserved", + }); + return { + candidate: { category, abstract: text, overview: `## ${heading}`, content: text }, + vector: vectorFor(text), + buildEntry: (v) => ({ + text, + vector: v, + importance: 0.8, + category, + scope: "agent:probe", + metadata, + }), + }; +} + +describe("reflection mapped rows: uniform dedup -> merge pipeline", () => { + it("merges a duplicate mapped row into its existing target instead of storing it beside it", async () => { + const store = makeStore({ neighbors: [neighborRow("row-1", "Prefer bulleted answers when the user asks for outlines.")] }); + const llm = makeLlm({ + onDedupBatch: () => ({ + results: [{ index: 1, decision: "merge", match_index: 1, reason: "adds detail" }], + }), + onMergeBatch: () => ({ + results: [{ index: 1, abstract: "merged abstract", overview: "o", content: "merged content" }], + }), + }); + const extractor = makeExtractor(store, llm); + + const { stats, createdEntries } = await extractor.persistGatedCandidates( + [reflectionItem("Prefer short answers whenever the user explicitly requests brevity in chat.")], + { targetScope: "agent:probe", scopeFilter: ["agent:probe"], sessionKey: "refl-test" }, + ); + + assert.equal(stats.merged, 1, "the duplicate mapped row must merge"); + assert.equal(createdEntries.length, 0, "nothing new lands beside the target"); + assert.equal(store.bulkStored.length, 0); + const contentUpdate = store.updates.find((u) => u.patch && u.patch.text); + assert.ok(contentUpdate, "the merge target must be updated"); + assert.equal(contentUpdate.id, "row-1"); + assert.deepEqual( + llm.calls.filter((c) => c === "dedup-decision-batch"), + ["dedup-decision-batch"], + "exactly one batched dedup call", + ); + assert.deepEqual( + llm.calls.filter((c) => c === "merge-memory-batch"), + ["merge-memory-batch"], + "exactly one batched merge-writer call", + ); + }); + + it("stores a novel mapped row through the caller's entry builder, reflection metadata intact", async () => { + const store = makeStore({ neighbors: [] }); + const llm = makeLlm({}); + const extractor = makeExtractor(store, llm); + + const { stats, createdEntries } = await extractor.persistGatedCandidates( + [reflectionItem("Do not restate a setting once its owner has withdrawn it.")], + { targetScope: "agent:probe", scopeFilter: ["agent:probe"], sessionKey: "refl-test" }, + ); + + assert.equal(stats.created, 1); + assert.equal(createdEntries.length, 1); + assert.equal(store.bulkStored.length, 1); + const meta = JSON.parse(store.bulkStored[0].metadata); + assert.equal(meta.marker, "reflection-metadata-preserved", "CREATE writes must keep the reflection metadata"); + assert.equal(meta.type, "memory-reflection-mapped"); + assert.equal(store.bulkStored[0].category, "patterns"); + assert.equal(llm.calls.length, 0, "no similar rows -> no dedup or merge LLM calls"); + }); + + it("decides a whole burst with exactly one batched dedup call and drops skip verdicts", async () => { + const store = makeStore({ + neighbors: [ + neighborRow("row-1", "Prefer bulleted answers when the user asks for outlines."), + neighborRow("row-2", "Always honor a session-scoped no-tools constraint."), + ], + }); + const llm = makeLlm({ + onDedupBatch: () => ({ + results: [ + { index: 1, decision: "skip", match_index: 1, reason: "duplicate" }, + { index: 2, decision: "skip", match_index: 2, reason: "duplicate" }, + { index: 3, decision: "create", reason: "new" }, + ], + }), + }); + const extractor = makeExtractor(store, llm); + + const { stats, createdEntries } = await extractor.persistGatedCandidates( + [ + reflectionItem("Keep replies compact once a requester opts into terse output."), + reflectionItem("Apply the per-thread capability limits on every turn."), + reflectionItem("Confirm the target branch before opening a pull request."), + ], + { targetScope: "agent:probe", scopeFilter: ["agent:probe"], sessionKey: "refl-test" }, + ); + + assert.equal(llm.calls.filter((c) => c === "dedup-decision-batch").length, 1, "one dedup call for the burst"); + assert.equal(stats.skipped, 2); + assert.equal(stats.created, 1); + assert.equal(createdEntries.length, 1); + }); + + it("persists the row when the dedup judge fails, instead of dropping it (fail-open)", async () => { + const store = makeStore({ neighbors: [neighborRow("row-1", "Prefer bulleted answers when the user asks for outlines.")] }); + const llm = makeLlm({ + onDedupBatch: () => { + throw new Error("judge outage"); + }, + }); + const extractor = makeExtractor(store, llm); + + const { stats, createdEntries } = await extractor.persistGatedCandidates( + [reflectionItem("Prefer concise answers when the user explicitly asks for brevity.")], + { targetScope: "agent:probe", scopeFilter: ["agent:probe"], sessionKey: "refl-test" }, + ); + + assert.equal(stats.created, 1, "a judge outage must not lose the reflection row"); + assert.equal(createdEntries.length, 1); + assert.equal(store.bulkStored.length, 1); + }); + + it("never re-scores pre-gated rows through admission control", async () => { + const store = makeStore({ neighbors: [] }); + const llm = makeLlm({}); + const extractor = makeExtractor(store, llm, { admissionControl: { enabled: true } }); + + const { stats } = await extractor.persistGatedCandidates( + [reflectionItem("Track the deploy window in the release checklist.")], + { targetScope: "agent:probe", scopeFilter: ["agent:probe"], sessionKey: "refl-test" }, + ); + + assert.equal(stats.created, 1, "the pre-gated row must persist without a second admission pass"); + assert.deepEqual(llm.calls, [], "no admission (or any other) LLM call may fire for pre-gated rows"); + }); +}); + +// Production shape: index.ts's mapped lane serializes the gate's admission +// record (plus provenance) and stores it as a nested JSON STRING under +// metadata.admission_audit — mirror that exactly, never an invented field. +const PRODUCTION_MAPPED_AUDIT = { + version: "amac-v1", + decision: "pass_to_dedup", + score: 0.62, + reason: "caller-gate-probe", + thresholds: { reject: 0.25, admit: 0.55 }, + weights: { similarity: 0.4, utility: 0.3, novelty: 0.3 }, + feature_scores: { similarity: 0.5, utility: 0.7, novelty: 0.6 }, + matched_existing_memory_ids: [], + compared_existing_memory_ids: [], + max_similarity: 0.5, + evaluated_at: 1_700_000_400_000, + provenance: "memory-reflection-mapped", +}; + +function auditedReflectionItem(text, opts = {}) { + const item = reflectionItem(text, opts); + const build = item.buildEntry; + item.buildEntry = (v) => { + const entry = build(v); + const meta = JSON.parse(entry.metadata); + meta.admission_audit = JSON.stringify(PRODUCTION_MAPPED_AUDIT); + return { ...entry, metadata: JSON.stringify(meta) }; + }; + return item; +} + +describe("reflection mapped rows: review-round hardening (audit fidelity, provenance, fail-open, burst dedup)", () => { + it("persists the caller's own admission audit on a merge target, never the pre-gated marker", async () => { + const store = makeStore({ neighbors: [neighborRow("row-1", "Track deploy windows in the release checklist file.")] }); + const llm = makeLlm({ + onDedupBatch: () => ({ results: [{ index: 1, decision: "merge", match_index: 1, reason: "adds detail" }] }), + onMergeBatch: () => ({ results: [{ index: 1, abstract: "merged abstract", overview: "o", content: "merged content" }] }), + }); + const extractor = makeExtractor(store, llm, { admissionControl: { enabled: true } }); + + await extractor.persistGatedCandidates( + [auditedReflectionItem("Record every deploy window inside the shared release checklist.")], + { targetScope: "agent:probe", scopeFilter: ["agent:probe"], sessionKey: "refl-test" }, + ); + + const auditUpdates = store.updates + .map((u) => { + try { return JSON.parse(u.patch.metadata).admission_control; } catch { return undefined; } + }) + .filter(Boolean); + assert.ok(auditUpdates.length >= 1, "the merged target must carry an admission audit"); + for (const audit of auditUpdates) { + assert.equal(audit.reason, "caller-gate-probe", "the caller's own gate record must persist"); + assert.equal(audit.provenance, "memory-reflection-mapped", "the mapped-lane provenance must survive the nested-JSON parse"); + assert.equal(audit.version, "amac-v1", "the full production record flows through, not a synthetic marker"); + } + }); + + it("builds supersede rows from the caller's entry, layering the verdict fields on top", async () => { + const store = makeStore({ neighbors: [neighborRow("row-1", "The staging smoke test runs before every deploy.")] }); + const llm = makeLlm({ + onDedupBatch: () => ({ results: [{ index: 1, decision: "supersede", match_index: 1, reason: "newer fact" }] }), + }); + const extractor = makeExtractor(store, llm); + + const { stats } = await extractor.persistGatedCandidates( + [auditedReflectionItem("The staging smoke test now runs after every deploy instead of before it.", { category: "preferences" })], + { targetScope: "agent:probe", scopeFilter: ["agent:probe"], sessionKey: "refl-test" }, + ); + + assert.equal(store.bulkStored.length, 1, "the superseding row must be created"); + const meta = JSON.parse(store.bulkStored[0].metadata); + assert.equal(meta.marker, "reflection-metadata-preserved", "reflection provenance must survive the supersede path"); + assert.equal(meta.type, "memory-reflection-mapped", "the mapped kind must survive the supersede path"); + assert.equal(JSON.parse(meta.admission_audit).reason, "caller-gate-probe", "the caller's audit must survive the supersede path"); + assert.equal(meta.supersedes, "row-1", "the verdict linkage must be layered on"); + assert.ok(meta.fact_key, "the verdict fact_key must be layered on"); + assert.equal(store.bulkStored[0].importance, 0.8, "the caller's importance must survive"); + assert.ok(stats.created >= 1 || stats.merged >= 1, "the outcome is accounted"); + }); + + it("fails open to the caller-built row when the dedup search fails twice", async () => { + const store = makeStore({ neighbors: [] }); + store.vectorSearch = async () => { + throw new Error("simulated search outage"); + }; + const llm = makeLlm({}); + const extractor = makeExtractor(store, llm); + + const { stats } = await extractor.persistGatedCandidates( + [auditedReflectionItem("Keep one canonical runbook per service in the operations space.")], + { targetScope: "agent:probe", scopeFilter: ["agent:probe"], sessionKey: "refl-test" }, + ); + + assert.equal(stats.created, 1, "an admitted row must never be dropped by a failing dedup search"); + assert.equal(store.bulkStored.length, 1); + const meta = JSON.parse(store.bulkStored[0].metadata); + assert.equal(meta.marker, "reflection-metadata-preserved", "the fail-open row is the caller's own entry"); + }); + + it("falls back to create when the batched merge writer degrades, instead of dropping the addition", async () => { + const store = makeStore({ neighbors: [neighborRow("row-1", "Rotate the API token on the first Monday of the month.")] }); + const llm = makeLlm({ + onDedupBatch: () => ({ results: [{ index: 1, decision: "merge", match_index: 1, reason: "adds detail" }] }), + onMergeBatch: () => ({ results: [] }), + }); + const extractor = makeExtractor(store, llm); + + const { stats } = await extractor.persistGatedCandidates( + [auditedReflectionItem("Rotate the API token on the first Monday, and log the rotation in the audit sheet.")], + { targetScope: "agent:probe", scopeFilter: ["agent:probe"], sessionKey: "refl-test" }, + ); + + assert.equal(stats.merged, 0, "a degraded merge must not count as merged"); + assert.equal(stats.created, 1, "the admitted addition must fall back to create"); + assert.equal(store.bulkStored.length, 1, "the caller-built row lands instead of disappearing"); + const contentUpdate = store.updates.find((u) => u.patch && u.patch.text); + assert.equal(contentUpdate, undefined, "the merge target stays untouched"); + const meta = JSON.parse(store.bulkStored[0].metadata); + assert.equal(meta.marker, "reflection-metadata-preserved"); + }); + + it("falls back to the caller-built row when the merge target vanishes before the read", async () => { + const store = makeStore({ neighbors: [neighborRow("row-1", "Publish the weekly changelog digest every Friday afternoon.")] }); + const realGet = store.getById.bind(store); + store.getById = async (id, scopeFilter) => (id === "row-1" ? null : realGet(id, scopeFilter)); + const llm = makeLlm({ + onDedupBatch: () => ({ results: [{ index: 1, decision: "merge", match_index: 1, reason: "adds detail" }] }), + onMergeBatch: () => ({ results: [{ index: 1, abstract: "merged abstract", overview: "o", content: "merged content" }] }), + }); + const extractor = makeExtractor(store, llm); + + const { stats } = await extractor.persistGatedCandidates( + [auditedReflectionItem("Publish the changelog digest each Friday and pin it in the team space.")], + { targetScope: "agent:probe", scopeFilter: ["agent:probe"], sessionKey: "refl-test" }, + ); + + assert.equal(store.bulkStored.length, 1, "the admitted row must land as a create, never vanish"); + const meta = JSON.parse(store.bulkStored[0].metadata); + assert.equal(meta.marker, "reflection-metadata-preserved", "the fallback row is the caller's own entry"); + assert.equal(stats.merged ?? 0, 0, "a vanished target must not count as merged"); + assert.equal(stats.created, 1, "the fallback is accounted as a create"); + assert.ok(!llm.calls.includes("merge-memory-batch"), "no merge may be generated against a vanished target"); + assert.equal(store.updates.length, 0, "nothing is written over the missing row"); + }); + + it("falls back to the caller-built row when the merge target vanishes during the update", async () => { + const store = makeStore({ neighbors: [neighborRow("row-1", "Send the incident retro invite within two business days.")] }); + const realUpdates = store.updates; + store.update = async (id, patch) => { + realUpdates.push({ id, patch }); + return null; + }; + const llm = makeLlm({ + onDedupBatch: () => ({ results: [{ index: 1, decision: "merge", match_index: 1, reason: "adds detail" }] }), + onMergeBatch: () => ({ results: [{ index: 1, abstract: "merged abstract", overview: "o", content: "merged content" }] }), + }); + const extractor = makeExtractor(store, llm); + + const { stats } = await extractor.persistGatedCandidates( + [auditedReflectionItem("Schedule the incident retro invite inside two business days of closure.")], + { targetScope: "agent:probe", scopeFilter: ["agent:probe"], sessionKey: "refl-test" }, + ); + + assert.equal(stats.merged ?? 0, 0, "a null update result must not be reported as a merge"); + assert.equal(stats.created, 1, "the admitted row falls back to a create"); + assert.equal(store.bulkStored.length, 1, "the caller-built row lands instead of disappearing"); + const meta = JSON.parse(store.bulkStored[0].metadata); + assert.equal(meta.marker, "reflection-metadata-preserved"); + }); + + it("falls back to the caller-built row when a support target has vanished", async () => { + const store = makeStore({ neighbors: [neighborRow("row-1", "Keep the sandbox image list mirrored in the platform wiki.")] }); + const realGet = store.getById.bind(store); + store.getById = async (id, scopeFilter) => (id === "row-1" ? null : realGet(id, scopeFilter)); + const llm = makeLlm({ + onDedupBatch: () => ({ results: [{ index: 1, decision: "support", match_index: 1, reason: "same fact restated" }] }), + }); + const extractor = makeExtractor(store, llm); + + const { stats } = await extractor.persistGatedCandidates( + [auditedReflectionItem("Mirror every sandbox image name into the platform wiki page.")], + { targetScope: "agent:probe", scopeFilter: ["agent:probe"], sessionKey: "refl-test" }, + ); + + assert.equal(stats.supported ?? 0, 0, "a vanished support target must not count as supported"); + assert.equal(stats.created, 1, "the admitted row falls back to a create"); + assert.equal(store.bulkStored.length, 1, "the caller-built row lands instead of disappearing"); + const meta = JSON.parse(store.bulkStored[0].metadata); + assert.equal(meta.marker, "reflection-metadata-preserved"); + }); + + it("collapses same-burst near-duplicate mapped rows to a single create", async () => { + const store = makeStore({ neighbors: [] }); + const llm = makeLlm({}); + const extractor = makeExtractor(store, llm); + const text = "Archive finished experiment notebooks into the research index."; + + const { stats } = await extractor.persistGatedCandidates( + [reflectionItem(text), reflectionItem(text)], + { targetScope: "agent:probe", scopeFilter: ["agent:probe"], sessionKey: "refl-test" }, + ); + + assert.equal(store.bulkStored.length, 1, "twin rows in one burst must collapse to one create"); + assert.equal(stats.created, 1); + assert.equal(stats.skipped, 1, "the dropped twin is accounted as skipped"); + }); + + it("a merged row stays reachable through its own category-filtered list view (real filter semantics)", async () => { + const target = neighborRow("row-1", "Review the failing check output before re-running the pipeline."); + const store = makeStore({ neighbors: [target] }); + const persistedSources = []; + const llm = makeLlm({ + onDedupBatch: () => ({ results: [{ index: 1, decision: "merge", match_index: 1, reason: "same practice, richer detail" }] }), + onMergeBatch: () => ({ results: [{ index: 1, abstract: "merged abstract", overview: "o", content: "merged content" }] }), + }); + const extractor = makeExtractor(store, llm, { + onPersisted: (entry, info) => { persistedSources.push(info.source); }, + }); + + await extractor.persistGatedCandidates( + [auditedReflectionItem("Always review the failing check output before restarting the pipeline run.", { category: "preferences" })], + { targetScope: "agent:probe", scopeFilter: ["agent:probe"], sessionKey: "refl-test" }, + ); + + const contentUpdate = store.updates.find((u) => u.patch && u.patch.text); + assert.ok(contentUpdate, "the merge target must be updated"); + const mergedColumnCategory = target.category; + const mergedMetadata = contentUpdate.patch.metadata; + assert.ok( + resolveCategoryFilterCandidates("patterns").includes(mergedColumnCategory), + "the target's own view must still SQL-prefilter the row in", + ); + assert.ok( + matchesMemoryCategoryFilter(mergedColumnCategory, "patterns", mergedMetadata), + "the merged row must remain visible in the target's category view; a cross-category merge must not reclassify it out of both views", + ); + assert.ok( + !resolveCategoryFilterCandidates("preferences").includes(mergedColumnCategory), + "the incoming candidate's view never sees the target's column", + ); + assert.ok( + persistedSources.some((s) => typeof s === "string" && s.startsWith("reflection:")), + "a mapped-row merge must carry its reflection provenance in the persistence notification", + ); + }); + + it("routes a richer same-burst restatement through the semantic judge (MERGE verdict, one row)", async () => { + const store = makeStore({ neighbors: [] }); + const llm = makeLlm({ + onDedupBatch: () => ({ results: [{ index: 1, decision: "merge", match_index: 1, reason: "richer restatement of the sibling row" }] }), + onMergeBatch: () => ({ results: [{ index: 1, abstract: "merged digest practice", overview: "o", content: "merged digest practice content" }] }), + }); + const extractor = makeExtractor(store, llm); + const short = reflectionItem("Ship the weekly metrics digest on Mondays."); + const richer = reflectionItem("Ship the weekly metrics digest on Mondays, and attach the anomaly summary when a threshold tripped."); + richer.vector = [...short.vector]; + + const { stats } = await extractor.persistGatedCandidates( + [short, richer], + { targetScope: "agent:probe", scopeFilter: ["agent:probe"], sessionKey: "refl-test" }, + ); + + assert.ok(llm.calls.includes("dedup-decision-batch"), "same-lane burst pairs must reach the semantic judge"); + assert.equal(store.bulkStored.length, 1, "a MERGE verdict must prevent two unconditional creates"); + assert.equal(stats.created, 1); + assert.equal(stats.merged, 1, "the richer row merges into its sibling's stored row"); + const contentUpdate = store.updates.find((u) => u.patch && u.patch.text); + assert.ok(contentUpdate, "the merged content lands on the sibling's stored row"); + assert.equal(contentUpdate.id, "new-1"); + }); + + it("honors a same-burst SKIP verdict from the judge", async () => { + const store = makeStore({ neighbors: [] }); + const llm = makeLlm({ + onDedupBatch: () => ({ results: [{ index: 1, decision: "skip", match_index: 1, reason: "adds nothing beyond the sibling" }] }), + }); + const extractor = makeExtractor(store, llm); + const first = reflectionItem("Rotate the pager schedule at the sprint boundary."); + const restated = reflectionItem("The pager schedule rotates when a sprint boundary arrives."); + restated.vector = [...first.vector]; + + const { stats } = await extractor.persistGatedCandidates( + [first, restated], + { targetScope: "agent:probe", scopeFilter: ["agent:probe"], sessionKey: "refl-test" }, + ); + + assert.ok(llm.calls.includes("dedup-decision-batch"), "the skip must come from the judge, not a blind guard"); + assert.equal(store.bulkStored.length, 1, "a SKIP verdict must prevent the duplicate create"); + assert.equal(stats.created, 1); + assert.equal(stats.skipped, 1); + }); + + it("leaves dissimilar same-lane burst rows judge-free (below the sibling similarity threshold)", async () => { + const store = makeStore({ neighbors: [] }); + const llm = makeLlm({}); + const extractor = makeExtractor(store, llm); + const a = reflectionItem("Track quarterly budget variance in the shared finance sheet."); + const b = reflectionItem("Recycle stale sandbox images at the start of each month."); + a.vector = [1, 0, 0, 0]; + b.vector = [0, 1, 0, 0]; + + const { stats } = await extractor.persistGatedCandidates( + [a, b], + { targetScope: "agent:probe", scopeFilter: ["agent:probe"], sessionKey: "refl-test" }, + ); + + assert.equal(store.bulkStored.length, 2, "unrelated rows persist independently with no judge call"); + assert.equal(stats.created, 2); + assert.equal(stats.skipped ?? 0, 0); + }); + + it("keeps a lesson and a decision with identical text independent in one burst (shared candidate category)", async () => { + // Reflection lessons and decisions BOTH map to candidate category + // "cases" while carrying different mapped kinds, headings, importance, + // and decay policies. A category+text twin key silently drops whichever + // lane the slicer emits second. + const store = makeStore({ neighbors: [] }); + const llm = makeLlm({}); + const extractor = makeExtractor(store, llm); + const text = "Verify the backup restore end to end before rotating the encryption keys."; + const lessonRow = reflectionItem(text, { category: "cases", heading: "Lessons (durable)", mappedKind: "lesson" }); + const decisionRow = reflectionItem(text, { category: "cases", heading: "Decisions (durable)", mappedKind: "decision" }); + decisionRow.vector = [...lessonRow.vector]; + + const { stats } = await extractor.persistGatedCandidates( + [lessonRow, decisionRow], + { targetScope: "agent:probe", scopeFilter: ["agent:probe"], sessionKey: "refl-test" }, + ); + + assert.equal(store.bulkStored.length, 2, "identical text under lesson and decision is two lanes, not a twin"); + assert.equal(stats.created, 2); + assert.equal(stats.skipped ?? 0, 0, "neither lane's row may be silently discarded before semantic judging"); + const kinds = store.bulkStored + .map((e) => { try { return JSON.parse(e.metadata).mappedKind; } catch { return undefined; } }) + .sort(); + assert.deepEqual(kinds, ["decision", "lesson"], "both mapped kinds must persist"); + }); + + it("keeps same-text rows from different reflection sections independent in one burst", async () => { + const store = makeStore({ neighbors: [] }); + const llm = makeLlm({}); + const extractor = makeExtractor(store, llm); + const text = "Keep the changelog draft current while a release window is open."; + const patternsRow = reflectionItem(text, { category: "patterns" }); + const prefsRow = reflectionItem(text, { category: "preferences", heading: "User model deltas (about the user)" }); + prefsRow.vector = [...patternsRow.vector]; + + const { stats } = await extractor.persistGatedCandidates( + [patternsRow, prefsRow], + { targetScope: "agent:probe", scopeFilter: ["agent:probe"], sessionKey: "refl-test" }, + ); + + assert.equal(store.bulkStored.length, 2, "identical text under different categories is two facts, not a twin"); + assert.equal(stats.created, 2); + }); + + it("never lets a fail-open gate marker replace a target's complete admission audit", async () => { + const target = neighborRow("row-1", "Rotate the standby credentials during the maintenance window."); + const store = makeStore({ neighbors: [target] }); + const llm = makeLlm({ + onDedupBatch: () => ({ results: [{ index: 1, decision: "merge", match_index: 1, reason: "adds detail" }] }), + onMergeBatch: () => ({ results: [{ index: 1, abstract: "merged abstract", overview: "o", content: "merged content" }] }), + }); + const extractor = makeExtractor(store, llm, { admissionControl: { enabled: true } }); + + const item = reflectionItem("Rotate standby credentials inside the maintenance window and log the rotation."); + const build = item.buildEntry; + item.buildEntry = (v) => { + const entry = build(v); + const meta = JSON.parse(entry.metadata); + meta.admission_audit = JSON.stringify({ + provenance: "memory-reflection-mapped", + failedOpen: true, + reason: "gate error", + error: "synthetic outage", + }); + return { ...entry, metadata: JSON.stringify(meta) }; + }; + + await extractor.persistGatedCandidates( + [item], + { targetScope: "agent:probe", scopeFilter: ["agent:probe"], sessionKey: "refl-test" }, + ); + + for (const u of store.updates) { + let auditOnTarget; + try { auditOnTarget = JSON.parse(u.patch.metadata).admission_control; } catch { auditOnTarget = undefined; } + if (auditOnTarget !== undefined) { + assert.equal(auditOnTarget.version, "amac-v1", "only a complete audit record may land on a target, never a fail-open marker"); + } + } + }); +}); + +// Deferred same-burst verdicts resolve through store reads/writes that can +// throw; each verdict must degrade alone (fail open to its own create) so one +// storage failure never rejects the whole persistence call, discards queued +// follow-up work, or skips later verdicts. +describe("reflection mapped rows: deferred sibling-verdict fail-open", () => { + function siblingSupportPair() { + const anchor = reflectionItem("Confirm the failover runbook after every region switch."); + const restated = reflectionItem("After a region switch, always confirm the failover runbook."); + restated.vector = [...anchor.vector]; + return [anchor, restated]; + } + const supportVerdict = () => ({ + results: [{ index: 1, decision: "support", match_index: 1, reason: "same practice restated" }], + }); + + it("fails open to a create when the deferred support target read throws", async () => { + const store = makeStore({ neighbors: [] }); + store.getById = async () => { + throw new Error("read outage"); + }; + const llm = makeLlm({ onDedupBatch: supportVerdict }); + const extractor = makeExtractor(store, llm); + + const { stats } = await extractor.persistGatedCandidates( + siblingSupportPair(), + { targetScope: "agent:probe", scopeFilter: ["agent:probe"], sessionKey: "refl-test" }, + ); + + assert.equal(stats.created, 2, "a throwing getById must not reject the whole persistence call"); + assert.equal(stats.supported ?? 0, 0); + assert.equal(store.bulkStored.length, 2, "the caller-built row is enqueued exactly once"); + assert.equal(store.updates.length, 0, "no partial support write may land"); + }); + + it("fails open to a create when the deferred support write throws", async () => { + const store = makeStore({ neighbors: [] }); + store.update = async () => { + throw new Error("support write outage"); + }; + const llm = makeLlm({ onDedupBatch: supportVerdict }); + const extractor = makeExtractor(store, llm); + + const { stats } = await extractor.persistGatedCandidates( + siblingSupportPair(), + { targetScope: "agent:probe", scopeFilter: ["agent:probe"], sessionKey: "refl-test" }, + ); + + assert.equal(stats.created, 2, "a throwing store.update must not reject the whole persistence call"); + assert.equal(stats.supported ?? 0, 0); + assert.equal(store.bulkStored.length, 2, "the admitted row lands as a create, exactly once"); + }); + + it("isolates one deferred-verdict failure: earlier merges still flush and later verdicts still resolve", async () => { + const anchor = reflectionItem("Run the capacity check before enabling a new tenant."); + const mergeRow = reflectionItem("Run the capacity check before enabling a new tenant, and file the result in the intake ticket."); + const failingSupport = reflectionItem("Capacity checks precede any new tenant enablement."); + const laterSupport = reflectionItem("Before a tenant goes live, the capacity check must have run."); + // Geometry pins every verdict's top-scored sibling to the anchor with + // comfortable margins (anchor first, other siblings well below). + anchor.vector = [1, 0, 0, 0]; + mergeRow.vector = [0.9, 0.43588989, 0, 0]; + failingSupport.vector = [0.98, 0.19899749, 0, 0]; + laterSupport.vector = [0.9995, 0.0316186, 0, 0]; + + const store = makeStore({ neighbors: [] }); + const baseUpdate = store.update.bind(store); + let supportAttempts = 0; + store.update = async (id, patch) => { + if (!patch.text) { + supportAttempts++; + if (supportAttempts === 1) throw new Error("support write outage"); + } + return baseUpdate(id, patch); + }; + const llm = makeLlm({ + onDedupBatch: () => ({ + results: [ + { index: 1, decision: "merge", match_index: 1, reason: "richer restatement" }, + { index: 2, decision: "support", match_index: 1, reason: "same practice" }, + { index: 3, decision: "support", match_index: 1, reason: "same practice" }, + ], + }), + onMergeBatch: () => ({ + results: [{ index: 1, abstract: "merged capacity practice", overview: "o", content: "merged capacity practice content" }], + }), + }); + const extractor = makeExtractor(store, llm); + + const { stats } = await extractor.persistGatedCandidates( + [anchor, mergeRow, failingSupport, laterSupport], + { targetScope: "agent:probe", scopeFilter: ["agent:probe"], sessionKey: "refl-test" }, + ); + + assert.equal(stats.merged, 1, "the merge queued before the failure must still flush"); + assert.equal(stats.supported ?? 0, 1, "the verdict after the failure must still resolve"); + assert.equal(stats.created, 2, "the anchor plus exactly one fail-open create for the failed verdict"); + const contentUpdate = store.updates.find((u) => u.patch && u.patch.text); + assert.ok(contentUpdate, "the merged content still lands on the anchor row"); + assert.equal(contentUpdate.id, "new-1"); + assert.equal(store.bulkStored.length, 2, "no verdict may enqueue more than one caller-built row"); + }); + + it("binds a deferred verdict by lane identity, never to a same-text row from another lane", async () => { + const text = "Escalate stuck deploys to the on-call channel after two failed retries."; + const lessonTwin = reflectionItem(text, { category: "cases", heading: "Lessons (durable)", mappedKind: "lesson" }); + const decisionAnchor = reflectionItem(text, { category: "cases", heading: "Decisions (durable)", mappedKind: "decision" }); + const decisionSupport = reflectionItem( + "Two failed deploy retries mean an escalation to the on-call channel.", + { category: "cases", heading: "Decisions (durable)", mappedKind: "decision" }, + ); + decisionSupport.vector = [...decisionAnchor.vector]; + + // bulkStore drops the decision anchor, shifting positions so the + // deferred verdict has to resolve through the identity fallback. + const store = makeStore({ neighbors: [] }); + const baseBulkStore = store.bulkStore.bind(store); + let firstBulk = true; + store.bulkStore = async (entries) => { + if (!firstBulk) { + return baseBulkStore(entries); + } + firstBulk = false; + store.bulkStored.push(...entries); + const kept = entries.filter((e) => !String(e.metadata).includes('"mappedKind":"decision"')); + const stored = kept.map((e, i) => ({ ...e, id: `new-${store.rows.size + i + 1}`, timestamp: 1_700_000_500_000 })); + for (const s of stored) store.rows.set(s.id, s); + return stored; + }; + const llm = makeLlm({ + onDedupBatch: () => ({ + results: [{ index: 1, decision: "support", match_index: 1, reason: "same decision restated" }], + }), + }); + const extractor = makeExtractor(store, llm); + + const { stats } = await extractor.persistGatedCandidates( + [lessonTwin, decisionAnchor, decisionSupport], + { targetScope: "agent:probe", scopeFilter: ["agent:probe"], sessionKey: "refl-test" }, + ); + + assert.equal(stats.supported ?? 0, 0, "the support must not bind to the lesson row that merely shares the text"); + assert.equal(store.updates.length, 0, "no support write may land on the other lane's row"); + assert.equal(stats.created, 3, "the unresolvable verdict falls open to its own create"); + }); +}); diff --git a/test/smart-extractor-batch-admission.test.mjs b/test/smart-extractor-batch-admission.test.mjs index 50b46ff4e..8661a2147 100644 --- a/test/smart-extractor-batch-admission.test.mjs +++ b/test/smart-extractor-batch-admission.test.mjs @@ -228,6 +228,7 @@ function makeNeighborStore({ sharedTarget = false, neighbors = true } = {}) { }, async update(id, patch) { updates.push({ id, patch }); + return rows.get(id) ?? null; }, async store() {}, async bulkStore(entries) { diff --git a/test/smart-extractor-batch-embed.test.mjs b/test/smart-extractor-batch-embed.test.mjs index 25d77ea38..e313c3e65 100644 --- a/test/smart-extractor-batch-embed.test.mjs +++ b/test/smart-extractor-batch-embed.test.mjs @@ -95,6 +95,7 @@ function makeStore() { }, async update(_id, _patch, _scopeFilter) { entries.push({ action: "update", id: _id }); + return { id: _id }; }, async getById(_id, _scopeFilter) { return null; diff --git a/test/smart-extractor-merge-accounting.test.mjs b/test/smart-extractor-merge-accounting.test.mjs index 97eddc284..cebb49a9b 100644 --- a/test/smart-extractor-merge-accounting.test.mjs +++ b/test/smart-extractor-merge-accounting.test.mjs @@ -60,6 +60,7 @@ function makeStore({ getByIdThrows = false } = {}) { }, async update(id, patch, scopeFilter) { updates.push({ id, patch, scopeFilter }); + return EXISTING_ENTRY; }, async getById() { if (getByIdThrows) throw new Error("mock getById failure"); From 7278eebaaff85112f51ce2fe3c24f8749a6c64d0 Mon Sep 17 00:00:00 2001 From: Gorkem Date: Mon, 17 Aug 2026 11:43:56 +0300 Subject: [PATCH 2/7] fix: isolate supersede invalidation from the committed replacement; preserve fail-open admission evidence on merge/support A throwing or nothing-written invalidation no longer rejects past the already-committed superseding row: each failure is isolated per row, later invalidations and deferred sibling verdicts continue, and the outcome downgrades to a plain create with the replacement's supersedes claim stripped. Production-shaped fail-open admission markers are parsed as evidence instead of being dropped: merge/support targets keep their complete admission_control and gain an append-only admission_bypass_events record proving the mutation carried unevaluated content. Contextualize and contradict verify their target still exists and fall back to an ordinary create without a dangling relation when it vanished. --- dist/src/smart-extractor.js | 176 ++++++++---- src/smart-extractor.ts | 267 +++++++++++++----- ...eflection-mapped-uniform-pipeline.test.mjs | 174 ++++++++++++ 3 files changed, 498 insertions(+), 119 deletions(-) diff --git a/dist/src/smart-extractor.js b/dist/src/smart-extractor.js index 5f0d63e15..98d523767 100644 --- a/dist/src/smart-extractor.js +++ b/dist/src/smart-extractor.js @@ -15,15 +15,15 @@ import { classifyTemporal, inferExpiry } from "./temporal-classifier.js"; import { inferAtomicBrandItemPreferenceSlot } from "./preference-slots.js"; import { batchDedup } from "./batch-dedup.js"; import { buildBoundedTranscriptWithStats, } from "./auto-capture-cleanup.js"; -/** - * The caller's own admission audit, as carried inside an externally-built - * entry's metadata. Used by the gated-candidate lane so downstream verdict - * handling persists the real gate record, never a synthetic marker. - */ -function parseEntryAdmissionAudit(entry) { +const MAX_ADMISSION_BYPASS_EVENTS = 20; +function isFailOpenEvidence(value) { + return (value.failedOpen === true && + typeof value.provenance === "string"); +} +function parseEntryAdmissionEvidence(entry) { const raw = entry.metadata; if (typeof raw !== "string" || raw.length === 0) { - return undefined; + return {}; } try { const meta = JSON.parse(raw); @@ -31,19 +31,27 @@ function parseEntryAdmissionAudit(entry) { // gate record as a nested JSON string under admission_audit. const audit = meta.admission_audit; const parsed = typeof audit === "string" && audit.length > 0 ? JSON.parse(audit) : audit; - // Fail-open gate markers ({provenance, failedOpen, reason, error}) are - // evidence of a skipped evaluation, not an audit: adopting one would let - // MERGE/SUPPORT overwrite a target's complete audit with it. if (parsed && typeof parsed === "object" && parsed.version === "amac-v1" && typeof parsed.decision === "string") { - return parsed; + return { audit: parsed }; } - return undefined; + // Fail-open gate markers ({provenance, failedOpen, reason, error}) are + // evidence of a skipped evaluation, not an audit. They must never be + // adopted as admission_control, but dropping them entirely would leave a + // mutated target with no trace that unevaluated content became durable — + // carry them separately so write sites can append bypass evidence. + if (parsed && + typeof parsed === "object" && + parsed.failedOpen === true && + typeof parsed.provenance === "string") { + return { failOpen: parsed }; + } + return {}; } catch { - return undefined; + return {}; } } // ============================================================================ @@ -700,7 +708,7 @@ export class SmartExtractor { this.externalEntryBuilders.set(item.candidate, { build: item.buildEntry, prebuilt, - audit: parseEntryAdmissionAudit(prebuilt), + ...parseEntryAdmissionEvidence(prebuilt), }); } // Admission already ran in the caller's gate; the evaluation handed to @@ -930,7 +938,8 @@ export class SmartExtractor { const followupMerges = []; const followupCreates = []; for (const pending of pendingSiblingVerdicts) { - const audit = this.externalEntryBuilders.get(pending.candidate)?.audit; + const ext = this.externalEntryBuilders.get(pending.candidate); + const audit = ext?.audit ?? ext?.failOpen; const failOpenCreate = async (why) => { followupCreates.push(await this.externalOrBuiltFallbackEntry(pending.candidate, targetScope, sessionKey, pending.vector, audit)); stats.created++; @@ -1661,15 +1670,15 @@ export class SmartExtractor { const dedupResult = precomputedDedup ?? await this.deduplicate(candidate, vector, scopeFilter); switch (dedupResult.decision) { case "create": - createEntries?.push(this.buildStoreEntry(candidate, vector, sessionKey, targetScope, admission?.audit)); + createEntries?.push(this.buildStoreEntry(candidate, vector, sessionKey, targetScope, this.admissionWriteEvidenceFor(candidate, admission))); stats.created++; break; case "merge": if (dedupResult.matchId && MERGE_SUPPORTED_CATEGORIES.has(candidate.category)) { const mergeOutcome = pendingMerges - ? await this.queueMergeJob(pendingMerges, candidate, dedupResult.matchId, targetScope, scopeFilter, dedupResult.contextLabel, admission?.audit, createEntries, agentId) - : await this.handleMerge(candidate, dedupResult.matchId, targetScope, scopeFilter, dedupResult.contextLabel, admission?.audit, createEntries, agentId); + ? await this.queueMergeJob(pendingMerges, candidate, dedupResult.matchId, targetScope, scopeFilter, dedupResult.contextLabel, this.admissionWriteEvidenceFor(candidate, admission), createEntries, agentId) + : await this.handleMerge(candidate, dedupResult.matchId, targetScope, scopeFilter, dedupResult.contextLabel, this.admissionWriteEvidenceFor(candidate, admission), createEntries, agentId); if (mergeOutcome === "merged") { stats.merged++; } @@ -1682,7 +1691,7 @@ export class SmartExtractor { } else { // Category doesn't support merge → create instead - createEntries?.push(this.buildStoreEntry(candidate, vector, sessionKey, targetScope, admission?.audit)); + createEntries?.push(this.buildStoreEntry(candidate, vector, sessionKey, targetScope, this.admissionWriteEvidenceFor(candidate, admission))); stats.created++; } break; @@ -1693,40 +1702,40 @@ export class SmartExtractor { case "supersede": if (dedupResult.matchId && TEMPORAL_VERSIONED_CATEGORIES.has(candidate.category)) { - await this.handleSupersede(candidate, vector, dedupResult.matchId, sessionKey, targetScope, scopeFilter, admission?.audit, createEntries, pendingSupersedeInvalidations, agentId); + await this.handleSupersede(candidate, vector, dedupResult.matchId, sessionKey, targetScope, scopeFilter, this.admissionWriteEvidenceFor(candidate, admission), createEntries, pendingSupersedeInvalidations, agentId); stats.created++; stats.superseded = (stats.superseded ?? 0) + 1; } else { - createEntries?.push(this.buildStoreEntry(candidate, vector, sessionKey, targetScope, admission?.audit)); + createEntries?.push(this.buildStoreEntry(candidate, vector, sessionKey, targetScope, this.admissionWriteEvidenceFor(candidate, admission))); stats.created++; } break; case "support": if (dedupResult.matchId) { - const supportOutcome = await this.handleSupport(dedupResult.matchId, { session: sessionKey, timestamp: Date.now() }, dedupResult.reason, dedupResult.contextLabel, scopeFilter, admission?.audit); + const supportOutcome = await this.handleSupport(dedupResult.matchId, { session: sessionKey, timestamp: Date.now() }, dedupResult.reason, dedupResult.contextLabel, scopeFilter, this.admissionWriteEvidenceFor(candidate, admission)); if (supportOutcome === "supported") { stats.supported = (stats.supported ?? 0) + 1; } else { // Target vanished mid-flight: same semantics as a support verdict // with no target — the candidate lands as a new row. - createEntries?.push(await this.externalOrBuiltFallbackEntry(candidate, targetScope, sessionKey, vector, admission?.audit)); + createEntries?.push(await this.externalOrBuiltFallbackEntry(candidate, targetScope, sessionKey, vector, this.admissionWriteEvidenceFor(candidate, admission))); stats.created++; } } else { - createEntries?.push(this.buildStoreEntry(candidate, vector, sessionKey, targetScope, admission?.audit)); + createEntries?.push(this.buildStoreEntry(candidate, vector, sessionKey, targetScope, this.admissionWriteEvidenceFor(candidate, admission))); stats.created++; } break; case "contextualize": if (dedupResult.matchId) { - await this.handleContextualize(candidate, vector, dedupResult.matchId, sessionKey, targetScope, scopeFilter, dedupResult.contextLabel, admission?.audit, createEntries, agentId); + await this.handleContextualize(candidate, vector, dedupResult.matchId, sessionKey, targetScope, scopeFilter, dedupResult.contextLabel, this.admissionWriteEvidenceFor(candidate, admission), createEntries, agentId); stats.created++; } else { - createEntries?.push(this.buildStoreEntry(candidate, vector, sessionKey, targetScope, admission?.audit)); + createEntries?.push(this.buildStoreEntry(candidate, vector, sessionKey, targetScope, this.admissionWriteEvidenceFor(candidate, admission))); stats.created++; } break; @@ -1734,17 +1743,17 @@ export class SmartExtractor { if (dedupResult.matchId) { if (TEMPORAL_VERSIONED_CATEGORIES.has(candidate.category) && dedupResult.contextLabel === "general") { - await this.handleSupersede(candidate, vector, dedupResult.matchId, sessionKey, targetScope, scopeFilter, admission?.audit, createEntries, pendingSupersedeInvalidations, agentId); + await this.handleSupersede(candidate, vector, dedupResult.matchId, sessionKey, targetScope, scopeFilter, this.admissionWriteEvidenceFor(candidate, admission), createEntries, pendingSupersedeInvalidations, agentId); stats.created++; stats.superseded = (stats.superseded ?? 0) + 1; } else { - await this.handleContradict(candidate, vector, dedupResult.matchId, sessionKey, targetScope, scopeFilter, dedupResult.contextLabel, admission?.audit, createEntries, agentId); + await this.handleContradict(candidate, vector, dedupResult.matchId, sessionKey, targetScope, scopeFilter, dedupResult.contextLabel, this.admissionWriteEvidenceFor(candidate, admission), createEntries, agentId); stats.created++; } } else { - createEntries?.push(this.buildStoreEntry(candidate, vector, sessionKey, targetScope, admission?.audit)); + createEntries?.push(this.buildStoreEntry(candidate, vector, sessionKey, targetScope, this.admissionWriteEvidenceFor(candidate, admission))); stats.created++; } break; @@ -2360,9 +2369,11 @@ export class SmartExtractor { return; } const created = await this.store.store(entry); - await this.invalidateSupersededMemory(matchId, existing, factKey, created.id, scopeFilter); + const invalidated = await this.invalidateSupersededMemory(matchId, existing, factKey, created, scopeFilter); await this.notifyPersisted({ text: created.text, category: created.category, scope: created.scope, timestamp: created.timestamp }, "smart-extraction", agentId); - this.log(`memory-pro: smart-extractor: superseded [${candidate.category}] ${matchId.slice(0, 8)} -> ${created.id.slice(0, 8)}`); + if (invalidated) { + this.log(`memory-pro: smart-extractor: superseded [${candidate.category}] ${matchId.slice(0, 8)} -> ${created.id.slice(0, 8)}`); + } } async applyPendingSupersedeInvalidations(createdEntries, pendingSupersedeInvalidations) { for (const pending of pendingSupersedeInvalidations) { @@ -2371,22 +2382,56 @@ export class SmartExtractor { this.log(`memory-pro: smart-extractor: supersede invalidation skipped for ${pending.matchId.slice(0, 8)} because batch create returned no matching entry`); continue; } - await this.invalidateSupersededMemory(pending.matchId, pending.existing, pending.factKey, created.id, pending.scopeFilter); - this.log(`memory-pro: smart-extractor: superseded ${pending.matchId.slice(0, 8)} -> ${created.id.slice(0, 8)}`); + const invalidated = await this.invalidateSupersededMemory(pending.matchId, pending.existing, pending.factKey, created, pending.scopeFilter); + if (invalidated) { + this.log(`memory-pro: smart-extractor: superseded ${pending.matchId.slice(0, 8)} -> ${created.id.slice(0, 8)}`); + } } } - async invalidateSupersededMemory(matchId, existing, factKey, createdId, scopeFilter) { - const existingMeta = parseSmartMetadata(existing.metadata, existing); - const invalidatedMetadata = buildSmartMetadata(existing, { - fact_key: factKey, - invalidated_at: Date.now(), - superseded_by: createdId, - relations: appendRelation(existingMeta.relations, { - type: "superseded_by", - targetId: createdId, - }), - }); - await this.store.update(matchId, { metadata: stringifySmartMetadata(invalidatedMetadata) }, scopeFilter); + /** + * Invalidate the superseded row AFTER its replacement is committed. The + * replacement commit is irrevocable by this point, so this step must never + * reject past it: any failure (thrown read/update, or an update that + * reports nothing written) is isolated per row, the outcome downgrades to + * a plain CREATE, and the replacement's supersedes claim is stripped + * best-effort so the pair never reports a supersede that did not happen. + * Returns true only when the old row was actually invalidated. + */ + async invalidateSupersededMemory(matchId, existing, factKey, created, scopeFilter) { + try { + const existingMeta = parseSmartMetadata(existing.metadata, existing); + const invalidatedMetadata = buildSmartMetadata(existing, { + fact_key: factKey, + invalidated_at: Date.now(), + superseded_by: created.id, + relations: appendRelation(existingMeta.relations, { + type: "superseded_by", + targetId: created.id, + }), + }); + const written = await this.store.update(matchId, { metadata: stringifySmartMetadata(invalidatedMetadata) }, scopeFilter); + if (written) { + return true; + } + await this.downgradeSupersedeToCreate(matchId, created, scopeFilter, "update wrote nothing"); + return false; + } + catch (err) { + await this.downgradeSupersedeToCreate(matchId, created, scopeFilter, String(err)); + return false; + } + } + async downgradeSupersedeToCreate(matchId, created, scopeFilter, cause) { + this.log(`memory-pro: smart-extractor: supersede invalidation failed for ${matchId.slice(0, 8)} (${cause}) — outcome downgraded to plain CREATE, old row remains active`); + try { + const createdMeta = parseSmartMetadata(created.metadata, created); + delete createdMeta.supersedes; + createdMeta.relations = (createdMeta.relations ?? []).filter((r) => !(r.type === "supersedes" && r.targetId === matchId)); + await this.store.update(created.id, { metadata: stringifySmartMetadata(createdMeta) }, scopeFilter); + } + catch (stripErr) { + this.log(`memory-pro: smart-extractor: supersede-claim strip failed for ${created.id.slice(0, 8)}: ${String(stripErr)}`); + } } // -------------------------------------------------------------------------- // Context-Aware Handlers (support / contextualize / contradict) @@ -2417,6 +2462,15 @@ export class SmartExtractor { * linked to the original via a relation in metadata. */ async handleContextualize(candidate, vector, matchId, sessionKey, targetScope, scopeFilter, contextLabel, admissionAudit, createEntries, agentId) { + // A vanished target downgrades to an ordinary create: never persist a + // relation to a row that no longer exists. + const targetExists = Boolean(await this.store.getById(matchId, scopeFilter)); + if (!targetExists) { + this.log(`memory-pro: smart-extractor: contextualize target ${matchId.slice(0, 8)} no longer exists — storing as ordinary create without a relation`); + } + const contextualizeRelations = targetExists + ? [{ type: "contextualizes", targetId: matchId }] + : []; const storeCategory = this.mapToStoreCategory(candidate.category); const metadata = stringifySmartMetadata(this.withAdmissionAudit({ l0_abstract: candidate.abstract, @@ -2435,12 +2489,12 @@ export class SmartExtractor { bad_recall_count: 0, suppressed_until_turn: 0, contexts: contextLabel ? [contextLabel] : [], - relations: [{ type: "contextualizes", targetId: matchId }], + relations: contextualizeRelations, }, admissionAudit)); const entry_c = this.externalVerdictEntry(candidate, { state: "confirmed", contexts: contextLabel ? [contextLabel] : [], - relations: [{ type: "contextualizes", targetId: matchId }], + relations: contextualizeRelations, }) ?? { text: candidate.abstract, vector, @@ -2472,7 +2526,13 @@ export class SmartExtractor { meta.support_info = updated; await this.store.update(matchId, { metadata: stringifySmartMetadata(meta) }, scopeFilter); } - // 2. Store the contradicting entry as a new memory + else { + this.log(`memory-pro: smart-extractor: contradict target ${matchId.slice(0, 8)} no longer exists — storing as ordinary create without a relation`); + } + // 2. Store the contradicting entry as a new memory. A vanished target + // downgrades to an ordinary create: never persist a relation to a row + // that no longer exists. + const contradictRelations = existing ? [{ type: "contradicts", targetId: matchId }] : []; const storeCategory = this.mapToStoreCategory(candidate.category); const metadata = stringifySmartMetadata(this.withAdmissionAudit({ l0_abstract: candidate.abstract, @@ -2491,12 +2551,12 @@ export class SmartExtractor { bad_recall_count: 0, suppressed_until_turn: 0, contexts: contextLabel ? [contextLabel] : [], - relations: [{ type: "contradicts", targetId: matchId }], + relations: contradictRelations, }, admissionAudit)); const entry_d = this.externalVerdictEntry(candidate, { state: "confirmed", contexts: contextLabel ? [contextLabel] : [], - relations: [{ type: "contradicts", targetId: matchId }], + relations: contradictRelations, }) ?? { text: candidate.abstract, vector, @@ -2649,10 +2709,26 @@ export class SmartExtractor { /** * Embed admission audit record into metadata if audit persistence is enabled. */ + admissionWriteEvidenceFor(candidate, admission) { + return admission?.audit ?? this.externalEntryBuilders.get(candidate)?.failOpen; + } withAdmissionAudit(metadata, admissionAudit) { if (!admissionAudit || !this.persistAdmissionAudit) { return metadata; } + if (isFailOpenEvidence(admissionAudit)) { + // A fail-open marker proves this mutation carried unevaluated content. + // Preserve whatever complete audit the target already has and append + // the marker as bypass evidence (append-only, capped to the newest). + const prior = Array.isArray(metadata.admission_bypass_events) + ? metadata.admission_bypass_events + : []; + const events = [...prior, { at: Date.now(), ...admissionAudit }].slice(-MAX_ADMISSION_BYPASS_EVENTS); + return { + ...metadata, + admission_bypass_events: events, + }; + } return { ...metadata, admission_control: admissionAudit }; } /** diff --git a/src/smart-extractor.ts b/src/smart-extractor.ts index fda5ab9f9..62429768d 100644 --- a/src/smart-extractor.ts +++ b/src/smart-extractor.ts @@ -71,7 +71,7 @@ type StoreEntry = Omit; type PendingMergeAddition = { candidate: CandidateMemory; contextLabel?: string; - admissionAudit?: AdmissionAuditRecord; + admissionAudit?: AdmissionWriteEvidence; }; /** @@ -79,12 +79,39 @@ type PendingMergeAddition = { * entry's metadata. Used by the gated-candidate lane so downstream verdict * handling persists the real gate record, never a synthetic marker. */ -function parseEntryAdmissionAudit( +type AdmissionFailOpenEvidence = { + provenance: string; + failedOpen: true; + reason?: string; + error?: string; +}; + +/** + * The evidence a mutation may carry to its write site: either the caller's + * complete gate audit, or a fail-open marker proving the content was never + * evaluated. The write site (withAdmissionAudit) discriminates: complete + * audits replace admission_control as before; fail-open markers never touch + * an existing audit and are appended to admission_bypass_events instead. + */ +type AdmissionWriteEvidence = AdmissionAuditRecord | AdmissionFailOpenEvidence; + +const MAX_ADMISSION_BYPASS_EVENTS = 20; + +function isFailOpenEvidence( + value: AdmissionWriteEvidence, +): value is AdmissionFailOpenEvidence { + return ( + (value as AdmissionFailOpenEvidence).failedOpen === true && + typeof (value as AdmissionFailOpenEvidence).provenance === "string" + ); +} + +function parseEntryAdmissionEvidence( entry: Omit, -): AdmissionAuditRecord | undefined { +): { audit?: AdmissionAuditRecord; failOpen?: AdmissionFailOpenEvidence } { const raw = entry.metadata; if (typeof raw !== "string" || raw.length === 0) { - return undefined; + return {}; } try { const meta = JSON.parse(raw); @@ -92,20 +119,30 @@ function parseEntryAdmissionAudit( // gate record as a nested JSON string under admission_audit. const audit = meta.admission_audit; const parsed = typeof audit === "string" && audit.length > 0 ? JSON.parse(audit) : audit; - // Fail-open gate markers ({provenance, failedOpen, reason, error}) are - // evidence of a skipped evaluation, not an audit: adopting one would let - // MERGE/SUPPORT overwrite a target's complete audit with it. if ( parsed && typeof parsed === "object" && (parsed as Record).version === "amac-v1" && typeof (parsed as Record).decision === "string" ) { - return parsed as AdmissionAuditRecord; + return { audit: parsed as AdmissionAuditRecord }; } - return undefined; + // Fail-open gate markers ({provenance, failedOpen, reason, error}) are + // evidence of a skipped evaluation, not an audit. They must never be + // adopted as admission_control, but dropping them entirely would leave a + // mutated target with no trace that unevaluated content became durable — + // carry them separately so write sites can append bypass evidence. + if ( + parsed && + typeof parsed === "object" && + (parsed as Record).failedOpen === true && + typeof (parsed as Record).provenance === "string" + ) { + return { failOpen: parsed as AdmissionFailOpenEvidence }; + } + return {}; } catch { - return undefined; + return {}; } } /** @@ -1008,7 +1045,7 @@ export class SmartExtractor { this.externalEntryBuilders.set(item.candidate, { build: item.buildEntry, prebuilt, - audit: parseEntryAdmissionAudit(prebuilt), + ...parseEntryAdmissionEvidence(prebuilt), }); } @@ -1287,7 +1324,8 @@ export class SmartExtractor { const followupMerges: PendingMergeJob[] = []; const followupCreates: StoreEntry[] = []; for (const pending of pendingSiblingVerdicts) { - const audit = this.externalEntryBuilders.get(pending.candidate)?.audit; + const ext = this.externalEntryBuilders.get(pending.candidate); + const audit: AdmissionWriteEvidence | undefined = ext?.audit ?? ext?.failOpen; const failOpenCreate = async (why: string) => { followupCreates.push( await this.externalOrBuiltFallbackEntry(pending.candidate, targetScope, sessionKey, pending.vector, audit), @@ -2227,7 +2265,7 @@ export class SmartExtractor { switch (dedupResult.decision) { case "create": - createEntries?.push(this.buildStoreEntry(candidate, vector, sessionKey, targetScope, admission?.audit)); + createEntries?.push(this.buildStoreEntry(candidate, vector, sessionKey, targetScope, this.admissionWriteEvidenceFor(candidate, admission))); stats.created++; break; @@ -2244,7 +2282,7 @@ export class SmartExtractor { targetScope, scopeFilter, dedupResult.contextLabel, - admission?.audit, + this.admissionWriteEvidenceFor(candidate, admission), createEntries, agentId, ) @@ -2254,7 +2292,7 @@ export class SmartExtractor { targetScope, scopeFilter, dedupResult.contextLabel, - admission?.audit, + this.admissionWriteEvidenceFor(candidate, admission), createEntries, agentId, ); @@ -2268,7 +2306,7 @@ export class SmartExtractor { // "queued": accounted when the batched merge writer flushes. } else { // Category doesn't support merge → create instead - createEntries?.push(this.buildStoreEntry(candidate, vector, sessionKey, targetScope, admission?.audit)); + createEntries?.push(this.buildStoreEntry(candidate, vector, sessionKey, targetScope, this.admissionWriteEvidenceFor(candidate, admission))); stats.created++; } break; @@ -2292,7 +2330,7 @@ export class SmartExtractor { sessionKey, targetScope, scopeFilter, - admission?.audit, + this.admissionWriteEvidenceFor(candidate, admission), createEntries, pendingSupersedeInvalidations, agentId, @@ -2300,34 +2338,34 @@ export class SmartExtractor { stats.created++; stats.superseded = (stats.superseded ?? 0) + 1; } else { - createEntries?.push(this.buildStoreEntry(candidate, vector, sessionKey, targetScope, admission?.audit)); + createEntries?.push(this.buildStoreEntry(candidate, vector, sessionKey, targetScope, this.admissionWriteEvidenceFor(candidate, admission))); stats.created++; } break; case "support": if (dedupResult.matchId) { - const supportOutcome = await this.handleSupport(dedupResult.matchId, { session: sessionKey, timestamp: Date.now() }, dedupResult.reason, dedupResult.contextLabel, scopeFilter, admission?.audit); + const supportOutcome = await this.handleSupport(dedupResult.matchId, { session: sessionKey, timestamp: Date.now() }, dedupResult.reason, dedupResult.contextLabel, scopeFilter, this.admissionWriteEvidenceFor(candidate, admission)); if (supportOutcome === "supported") { stats.supported = (stats.supported ?? 0) + 1; } else { // Target vanished mid-flight: same semantics as a support verdict // with no target — the candidate lands as a new row. - createEntries?.push(await this.externalOrBuiltFallbackEntry(candidate, targetScope, sessionKey, vector, admission?.audit)); + createEntries?.push(await this.externalOrBuiltFallbackEntry(candidate, targetScope, sessionKey, vector, this.admissionWriteEvidenceFor(candidate, admission))); stats.created++; } } else { - createEntries?.push(this.buildStoreEntry(candidate, vector, sessionKey, targetScope, admission?.audit)); + createEntries?.push(this.buildStoreEntry(candidate, vector, sessionKey, targetScope, this.admissionWriteEvidenceFor(candidate, admission))); stats.created++; } break; case "contextualize": if (dedupResult.matchId) { - await this.handleContextualize(candidate, vector, dedupResult.matchId, sessionKey, targetScope, scopeFilter, dedupResult.contextLabel, admission?.audit, createEntries, agentId); + await this.handleContextualize(candidate, vector, dedupResult.matchId, sessionKey, targetScope, scopeFilter, dedupResult.contextLabel, this.admissionWriteEvidenceFor(candidate, admission), createEntries, agentId); stats.created++; } else { - createEntries?.push(this.buildStoreEntry(candidate, vector, sessionKey, targetScope, admission?.audit)); + createEntries?.push(this.buildStoreEntry(candidate, vector, sessionKey, targetScope, this.admissionWriteEvidenceFor(candidate, admission))); stats.created++; } break; @@ -2345,7 +2383,7 @@ export class SmartExtractor { sessionKey, targetScope, scopeFilter, - admission?.audit, + this.admissionWriteEvidenceFor(candidate, admission), createEntries, pendingSupersedeInvalidations, agentId, @@ -2353,11 +2391,11 @@ export class SmartExtractor { stats.created++; stats.superseded = (stats.superseded ?? 0) + 1; } else { - await this.handleContradict(candidate, vector, dedupResult.matchId, sessionKey, targetScope, scopeFilter, dedupResult.contextLabel, admission?.audit, createEntries, agentId); + await this.handleContradict(candidate, vector, dedupResult.matchId, sessionKey, targetScope, scopeFilter, dedupResult.contextLabel, this.admissionWriteEvidenceFor(candidate, admission), createEntries, agentId); stats.created++; } } else { - createEntries?.push(this.buildStoreEntry(candidate, vector, sessionKey, targetScope, admission?.audit)); + createEntries?.push(this.buildStoreEntry(candidate, vector, sessionKey, targetScope, this.admissionWriteEvidenceFor(candidate, admission))); stats.created++; } break; @@ -2627,7 +2665,7 @@ export class SmartExtractor { sessionKey: string, targetScope: string, scopeFilter?: string[], - admissionAudit?: AdmissionAuditRecord, + admissionAudit?: AdmissionWriteEvidence, createEntries?: StoreEntry[], agentId?: string, pendingMerges?: PendingMergeJob[], @@ -2739,7 +2777,7 @@ export class SmartExtractor { targetScope: string, scopeFilter?: string[], contextLabel?: string, - admissionAudit?: AdmissionAuditRecord, + admissionAudit?: AdmissionWriteEvidence, createEntries?: StoreEntry[], agentId?: string, ): Promise<"merged" | "created" | "llm-failed"> { @@ -2801,7 +2839,7 @@ export class SmartExtractor { targetScope: string, sessionLabel: string, vector?: number[], - admissionAudit?: AdmissionAuditRecord, + admissionAudit?: AdmissionWriteEvidence, ): Promise { const ext = this.externalEntryBuilders.get(candidate); if (ext?.prebuilt) { @@ -2866,7 +2904,7 @@ export class SmartExtractor { targetScope: string, scopeFilter?: string[], contextLabel?: string, - admissionAudit?: AdmissionAuditRecord, + admissionAudit?: AdmissionWriteEvidence, createEntries?: StoreEntry[], agentId?: string, ): Promise<"queued" | "created"> { @@ -3058,7 +3096,7 @@ export class SmartExtractor { targetScope: string, scopeFilter: string[] | undefined, contextLabels: Array, - admissionAudit: AdmissionAuditRecord | undefined, + admissionAudit: AdmissionWriteEvidence | undefined, agentId: string | undefined, mirrorSource: string = "smart-extraction", ): Promise<"updated" | "target-missing"> { @@ -3159,7 +3197,7 @@ export class SmartExtractor { sessionKey: string, targetScope: string, scopeFilter?: string[], - admissionAudit?: AdmissionAuditRecord, + admissionAudit?: AdmissionWriteEvidence, createEntries?: StoreEntry[], pendingSupersedeInvalidations?: PendingSupersedeInvalidation[], agentId?: string, @@ -3241,11 +3279,11 @@ export class SmartExtractor { } const created = await this.store.store(entry); - await this.invalidateSupersededMemory( + const invalidated = await this.invalidateSupersededMemory( matchId, existing, factKey, - created.id, + created, scopeFilter, ); await this.notifyPersisted( @@ -3254,9 +3292,11 @@ export class SmartExtractor { agentId, ); - this.log( - `memory-pro: smart-extractor: superseded [${candidate.category}] ${matchId.slice(0, 8)} -> ${created.id.slice(0, 8)}`, - ); + if (invalidated) { + this.log( + `memory-pro: smart-extractor: superseded [${candidate.category}] ${matchId.slice(0, 8)} -> ${created.id.slice(0, 8)}`, + ); + } } private async applyPendingSupersedeInvalidations( @@ -3271,42 +3311,90 @@ export class SmartExtractor { ); continue; } - await this.invalidateSupersededMemory( + const invalidated = await this.invalidateSupersededMemory( pending.matchId, pending.existing, pending.factKey, - created.id, + created, pending.scopeFilter, ); - this.log( - `memory-pro: smart-extractor: superseded ${pending.matchId.slice(0, 8)} -> ${created.id.slice(0, 8)}`, - ); + if (invalidated) { + this.log( + `memory-pro: smart-extractor: superseded ${pending.matchId.slice(0, 8)} -> ${created.id.slice(0, 8)}`, + ); + } } } + /** + * Invalidate the superseded row AFTER its replacement is committed. The + * replacement commit is irrevocable by this point, so this step must never + * reject past it: any failure (thrown read/update, or an update that + * reports nothing written) is isolated per row, the outcome downgrades to + * a plain CREATE, and the replacement's supersedes claim is stripped + * best-effort so the pair never reports a supersede that did not happen. + * Returns true only when the old row was actually invalidated. + */ private async invalidateSupersededMemory( matchId: string, existing: MemoryEntry, factKey: string, - createdId: string, + created: MemoryEntry, scopeFilter?: string[], - ): Promise { - const existingMeta = parseSmartMetadata(existing.metadata, existing); - const invalidatedMetadata = buildSmartMetadata(existing, { - fact_key: factKey, - invalidated_at: Date.now(), - superseded_by: createdId, - relations: appendRelation(existingMeta.relations, { - type: "superseded_by", - targetId: createdId, - }), - }); + ): Promise { + try { + const existingMeta = parseSmartMetadata(existing.metadata, existing); + const invalidatedMetadata = buildSmartMetadata(existing, { + fact_key: factKey, + invalidated_at: Date.now(), + superseded_by: created.id, + relations: appendRelation(existingMeta.relations, { + type: "superseded_by", + targetId: created.id, + }), + }); - await this.store.update( - matchId, - { metadata: stringifySmartMetadata(invalidatedMetadata) }, - scopeFilter, + const written = await this.store.update( + matchId, + { metadata: stringifySmartMetadata(invalidatedMetadata) }, + scopeFilter, + ); + if (written) { + return true; + } + await this.downgradeSupersedeToCreate(matchId, created, scopeFilter, "update wrote nothing"); + return false; + } catch (err) { + await this.downgradeSupersedeToCreate(matchId, created, scopeFilter, String(err)); + return false; + } + } + + private async downgradeSupersedeToCreate( + matchId: string, + created: MemoryEntry, + scopeFilter: string[] | undefined, + cause: string, + ): Promise { + this.log( + `memory-pro: smart-extractor: supersede invalidation failed for ${matchId.slice(0, 8)} (${cause}) — outcome downgraded to plain CREATE, old row remains active`, ); + try { + const createdMeta = parseSmartMetadata(created.metadata, created); + delete (createdMeta as Record).supersedes; + createdMeta.relations = (createdMeta.relations ?? []).filter( + (r) => !(r.type === "supersedes" && r.targetId === matchId), + ); + await this.store.update( + created.id, + { metadata: stringifySmartMetadata(createdMeta) }, + scopeFilter, + ); + } catch (stripErr) { + this.log( + `memory-pro: smart-extractor: supersede-claim strip failed for ${created.id.slice(0, 8)}: ${String(stripErr)}`, + ); + } } // -------------------------------------------------------------------------- @@ -3322,7 +3410,7 @@ export class SmartExtractor { reason: string, contextLabel?: string, scopeFilter?: string[], - admissionAudit?: AdmissionAuditRecord, + admissionAudit?: AdmissionWriteEvidence, ): Promise<"supported" | "target-missing"> { const existing = await this.store.getById(matchId, scopeFilter); if (!existing) { @@ -3367,10 +3455,21 @@ export class SmartExtractor { targetScope: string, scopeFilter?: string[], contextLabel?: string, - admissionAudit?: AdmissionAuditRecord, + admissionAudit?: AdmissionWriteEvidence, createEntries?: StoreEntry[], agentId?: string, ): Promise { + // A vanished target downgrades to an ordinary create: never persist a + // relation to a row that no longer exists. + const targetExists = Boolean(await this.store.getById(matchId, scopeFilter)); + if (!targetExists) { + this.log( + `memory-pro: smart-extractor: contextualize target ${matchId.slice(0, 8)} no longer exists — storing as ordinary create without a relation`, + ); + } + const contextualizeRelations = targetExists + ? [{ type: "contextualizes", targetId: matchId }] + : []; const storeCategory = this.mapToStoreCategory(candidate.category); const metadata = stringifySmartMetadata(this.withAdmissionAudit({ l0_abstract: candidate.abstract, @@ -3389,13 +3488,13 @@ export class SmartExtractor { bad_recall_count: 0, suppressed_until_turn: 0, contexts: contextLabel ? [contextLabel] : [], - relations: [{ type: "contextualizes", targetId: matchId }], + relations: contextualizeRelations, }, admissionAudit)); const entry_c: StoreEntry = this.externalVerdictEntry(candidate, { state: "confirmed", contexts: contextLabel ? [contextLabel] : [], - relations: [{ type: "contextualizes", targetId: matchId }], + relations: contextualizeRelations, }) ?? { text: candidate.abstract, vector, @@ -3432,7 +3531,7 @@ export class SmartExtractor { targetScope: string, scopeFilter?: string[], contextLabel?: string, - admissionAudit?: AdmissionAuditRecord, + admissionAudit?: AdmissionWriteEvidence, createEntries?: StoreEntry[], agentId?: string, ): Promise { @@ -3448,9 +3547,16 @@ export class SmartExtractor { { metadata: stringifySmartMetadata(meta) }, scopeFilter, ); + } else { + this.log( + `memory-pro: smart-extractor: contradict target ${matchId.slice(0, 8)} no longer exists — storing as ordinary create without a relation`, + ); } - // 2. Store the contradicting entry as a new memory + // 2. Store the contradicting entry as a new memory. A vanished target + // downgrades to an ordinary create: never persist a relation to a row + // that no longer exists. + const contradictRelations = existing ? [{ type: "contradicts", targetId: matchId }] : []; const storeCategory = this.mapToStoreCategory(candidate.category); const metadata = stringifySmartMetadata(this.withAdmissionAudit({ l0_abstract: candidate.abstract, @@ -3469,13 +3575,13 @@ export class SmartExtractor { bad_recall_count: 0, suppressed_until_turn: 0, contexts: contextLabel ? [contextLabel] : [], - relations: [{ type: "contradicts", targetId: matchId }], + relations: contradictRelations, }, admissionAudit)); const entry_d: StoreEntry = this.externalVerdictEntry(candidate, { state: "confirmed", contexts: contextLabel ? [contextLabel] : [], - relations: [{ type: "contradicts", targetId: matchId }], + relations: contradictRelations, }) ?? { text: candidate.abstract, vector, @@ -3518,6 +3624,7 @@ export class SmartExtractor { build: (vector: number[]) => StoreEntry; prebuilt?: StoreEntry; audit?: AdmissionAuditRecord; + failOpen?: AdmissionFailOpenEvidence; } >(); @@ -3557,7 +3664,7 @@ export class SmartExtractor { vector: number[], sessionKey: string, targetScope: string, - admissionAudit?: AdmissionAuditRecord, + admissionAudit?: AdmissionWriteEvidence, ): Omit { const external = this.externalEntryBuilders.get(candidate); if (external) { @@ -3618,7 +3725,7 @@ export class SmartExtractor { vector: number[], sessionKey: string, targetScope: string, - admissionAudit?: AdmissionAuditRecord, + admissionAudit?: AdmissionWriteEvidence, ): Promise { const entry = this.buildStoreEntry(candidate, vector, sessionKey, targetScope, admissionAudit); await this.store.store(entry); @@ -3675,13 +3782,35 @@ export class SmartExtractor { /** * Embed admission audit record into metadata if audit persistence is enabled. */ + private admissionWriteEvidenceFor( + candidate: CandidateMemory, + admission?: { audit?: AdmissionAuditRecord }, + ): AdmissionWriteEvidence | undefined { + return admission?.audit ?? this.externalEntryBuilders.get(candidate)?.failOpen; + } + private withAdmissionAudit>( metadata: T, - admissionAudit?: AdmissionAuditRecord, + admissionAudit?: AdmissionWriteEvidence, ): T & { admission_control?: AdmissionAuditRecord } { if (!admissionAudit || !this.persistAdmissionAudit) { return metadata as T & { admission_control?: AdmissionAuditRecord }; } + if (isFailOpenEvidence(admissionAudit)) { + // A fail-open marker proves this mutation carried unevaluated content. + // Preserve whatever complete audit the target already has and append + // the marker as bypass evidence (append-only, capped to the newest). + const prior = Array.isArray((metadata as Record).admission_bypass_events) + ? ((metadata as Record).admission_bypass_events as unknown[]) + : []; + const events = [...prior, { at: Date.now(), ...admissionAudit }].slice( + -MAX_ADMISSION_BYPASS_EVENTS, + ); + return { + ...metadata, + admission_bypass_events: events, + } as T & { admission_control?: AdmissionAuditRecord }; + } return { ...metadata, admission_control: admissionAudit }; } diff --git a/test/reflection-mapped-uniform-pipeline.test.mjs b/test/reflection-mapped-uniform-pipeline.test.mjs index 1bbfa50e6..2c702862c 100644 --- a/test/reflection-mapped-uniform-pipeline.test.mjs +++ b/test/reflection-mapped-uniform-pipeline.test.mjs @@ -792,3 +792,177 @@ describe("reflection mapped rows: deferred sibling-verdict fail-open", () => { assert.equal(stats.created, 3, "the unresolvable verdict falls open to its own create"); }); }); + +// Round-5 review regressions: supersede invalidation is isolated from the +// already-committed replacement, and fail-open admission markers survive as +// bypass evidence on MERGE/SUPPORT targets instead of vanishing. +// Fixtures are entirely synthetic; no real conversation data. +const PRODUCTION_FAIL_OPEN_MARKER = { + provenance: "memory-reflection-mapped", + failedOpen: true, + reason: "admission evaluation failed open", + error: "Error: gate outage", +}; + +function failOpenReflectionItem(text, opts = {}) { + const item = reflectionItem(text, opts); + const build = item.buildEntry; + item.buildEntry = (v) => { + const entry = build(v); + const meta = JSON.parse(entry.metadata); + meta.admission_audit = JSON.stringify(PRODUCTION_FAIL_OPEN_MARKER); + return { ...entry, metadata: JSON.stringify(meta) }; + }; + return item; +} + +function auditedNeighborRow(id, text) { + const row = neighborRow(id, text); + const meta = JSON.parse(row.metadata); + meta.admission_control = PRODUCTION_MAPPED_AUDIT; + row.metadata = JSON.stringify(meta); + return row; +} + +describe("reflection mapped rows: round-5 data-integrity regressions", () => { + it("isolates a throwing supersede invalidation: replacement stays, claim stripped, later invalidations run", async () => { + const store = makeStore({ + neighbors: [ + neighborRow("row-1", "The nightly export job writes into the archive bucket."), + neighborRow("row-2", "Weekly metrics roll up on Mondays before standup."), + ], + }); + const baseUpdate = store.update.bind(store); + store.update = async (id, patch, scopeFilter) => { + if (id === "row-1" && patch?.metadata?.includes("superseded_by")) { + throw new Error("invalidate outage"); + } + return baseUpdate(id, patch, scopeFilter); + }; + const llm = makeLlm({ + onDedupBatch: () => ({ + results: [ + { index: 1, decision: "supersede", match_index: 1, reason: "newer fact" }, + { index: 2, decision: "supersede", match_index: 2, reason: "newer fact" }, + ], + }), + }); + const extractor = makeExtractor(store, llm); + + const { stats, createdEntries } = await extractor.persistGatedCandidates( + [ + reflectionItem("The nightly export job now writes into the cold-storage bucket instead.", { category: "preferences" }), + reflectionItem("Weekly metrics now roll up on Fridays after the retro instead.", { category: "entities" }), + ], + { targetScope: "agent:probe", scopeFilter: ["agent:probe"], sessionKey: "refl-test" }, + ); + + assert.equal(stats.created, 2, "both replacements are committed"); + assert.equal(createdEntries.length, 2, "the batch resolves instead of rejecting past the commit"); + const oldRowMeta = JSON.parse(store.rows.get("row-1").metadata); + assert.equal(oldRowMeta.superseded_by, undefined, "the old row stays active after the failed invalidation"); + const row2Invalidation = store.updates.find( + (u) => u.id === "row-2" && u.patch?.metadata?.includes("superseded_by"), + ); + assert.ok(row2Invalidation, "the later invalidation still runs after the earlier one failed"); + assert.equal( + store.updates.some((u) => u.id === "row-1" && u.patch?.metadata?.includes("superseded_by")), + false, + "the failed invalidation never lands on the old row", + ); + const stripWrite = store.updates.find((u) => u.id === "new-3" && u.patch?.metadata); + assert.ok(stripWrite, "the downgrade rewrites the failed pair's replacement row"); + const strippedMeta = JSON.parse(stripWrite.patch.metadata); + assert.equal(strippedMeta.supersedes, undefined, "the supersedes claim is stripped"); + assert.equal( + (strippedMeta.relations ?? []).some((r) => r.type === "supersedes"), + false, + "the supersedes relation is stripped", + ); + }); + + it("treats an invalidation update that writes nothing as a failure and downgrades to a plain create", async () => { + const store = makeStore({ + neighbors: [neighborRow("row-1", "The canary check gates every rollout stage.")], + }); + const baseUpdate = store.update.bind(store); + store.update = async (id, patch, scopeFilter) => { + if (id === "row-1" && patch?.metadata?.includes("superseded_by")) { + return null; + } + return baseUpdate(id, patch, scopeFilter); + }; + const llm = makeLlm({ + onDedupBatch: () => ({ + results: [{ index: 1, decision: "supersede", match_index: 1, reason: "newer fact" }], + }), + }); + const extractor = makeExtractor(store, llm); + + const { createdEntries } = await extractor.persistGatedCandidates( + [reflectionItem("The canary check now gates only the final rollout stage.", { category: "preferences" })], + { targetScope: "agent:probe", scopeFilter: ["agent:probe"], sessionKey: "refl-test" }, + ); + + assert.equal(createdEntries.length, 1, "the replacement row stays committed"); + const oldRowMeta = JSON.parse(store.rows.get("row-1").metadata); + assert.equal(oldRowMeta.superseded_by, undefined, "a nothing-written update must not count as invalidated"); + const stripWrite = store.updates.find((u) => u.id === "new-2" && u.patch?.metadata); + assert.ok(stripWrite, "the outcome downgrades to a plain create"); + assert.equal(JSON.parse(stripWrite.patch.metadata).supersedes, undefined, "the supersedes claim is stripped"); + }); + + it("MERGE with a production fail-open marker preserves the target audit and appends bypass evidence", async () => { + const store = makeStore({ + neighbors: [auditedNeighborRow("row-1", "Keep the sandbox image list inside the platform handbook.")], + }); + const llm = makeLlm({ + onDedupBatch: () => ({ results: [{ index: 1, decision: "merge", match_index: 1, reason: "adds detail" }] }), + onMergeBatch: () => ({ results: [{ index: 1, abstract: "merged abstract", overview: "o", content: "merged content" }] }), + }); + const extractor = makeExtractor(store, llm, { admissionControl: { enabled: true } }); + + await extractor.persistGatedCandidates( + [failOpenReflectionItem("List every sandbox image in the platform handbook appendix as well.")], + { targetScope: "agent:probe", scopeFilter: ["agent:probe"], sessionKey: "refl-test" }, + ); + + const targetWrite = store.updates.find((u) => u.id === "row-1" && u.patch?.metadata); + assert.ok(targetWrite, "the merge must update its target"); + const meta = JSON.parse(targetWrite.patch.metadata); + assert.deepEqual( + meta.admission_control, + PRODUCTION_MAPPED_AUDIT, + "the target's complete audit must be preserved, never replaced by the marker", + ); + assert.ok(Array.isArray(meta.admission_bypass_events), "bypass evidence must be recorded"); + assert.equal(meta.admission_bypass_events.length, 1); + assert.equal(meta.admission_bypass_events[0].failedOpen, true); + assert.equal(meta.admission_bypass_events[0].provenance, "memory-reflection-mapped"); + assert.equal(typeof meta.admission_bypass_events[0].at, "number"); + }); + + it("SUPPORT with a production fail-open marker preserves the target audit and appends bypass evidence", async () => { + const store = makeStore({ + neighbors: [auditedNeighborRow("row-1", "Run the schema linter before publishing any config change.")], + }); + const llm = makeLlm({ + onDedupBatch: () => ({ results: [{ index: 1, decision: "support", match_index: 1, reason: "same practice" }] }), + }); + const extractor = makeExtractor(store, llm, { admissionControl: { enabled: true } }); + + const { stats } = await extractor.persistGatedCandidates( + [failOpenReflectionItem("Always run the schema linter ahead of publishing configuration changes.")], + { targetScope: "agent:probe", scopeFilter: ["agent:probe"], sessionKey: "refl-test" }, + ); + + assert.equal(stats.skipped >= 1 || stats.merged >= 1 || stats.created === 0, true, "support resolves against the target"); + const targetWrite = store.updates.find((u) => u.id === "row-1" && u.patch?.metadata); + assert.ok(targetWrite, "the support must update its target"); + const meta = JSON.parse(targetWrite.patch.metadata); + assert.deepEqual(meta.admission_control, PRODUCTION_MAPPED_AUDIT, "the complete audit survives"); + assert.equal(meta.admission_bypass_events?.length, 1, "the bypass marker is appended once"); + assert.equal(meta.admission_bypass_events[0].failedOpen, true); + assert.ok(meta.support_info, "the support stats still update"); + }); +}); From c6ca9f9e7aa8b4c34c12d36a9dd06b099a4e3b2b Mon Sep 17 00:00:00 2001 From: Gorkem Date: Wed, 19 Aug 2026 09:07:08 +0300 Subject: [PATCH 3/7] fix: bind deferred supersede repair to entry identity; aggregate grouped-merge evidence; guard contextualize/contradict fallbacks --- dist/src/smart-extractor.js | 154 +++++++++--- src/smart-extractor.ts | 214 ++++++++++++---- ...eflection-mapped-uniform-pipeline.test.mjs | 234 ++++++++++++++++++ 3 files changed, 514 insertions(+), 88 deletions(-) diff --git a/dist/src/smart-extractor.js b/dist/src/smart-extractor.js index 98d523767..d7d9305c1 100644 --- a/dist/src/smart-extractor.js +++ b/dist/src/smart-extractor.js @@ -657,7 +657,7 @@ export class SmartExtractor { if (createEntries.length > 0) { const createdEntries = await this.bulkStoreAndValidate(createEntries); if (createdEntries) { - await this.applyPendingSupersedeInvalidations(createdEntries, pendingSupersedeInvalidations); + await this.applyPendingSupersedeInvalidations(createEntries, createdEntries, pendingSupersedeInvalidations, stats); for (const created of createdEntries) { await this.notifyPersisted({ text: created.text, @@ -896,7 +896,7 @@ export class SmartExtractor { const stored = await this.bulkStoreAndValidate(createEntries); if (stored) { createdEntries = stored; - await this.applyPendingSupersedeInvalidations(stored, pendingSupersedeInvalidations); + await this.applyPendingSupersedeInvalidations(createEntries, stored, pendingSupersedeInvalidations, stats); } else if (pendingSupersedeInvalidations.length > 0) { this.log("memory-pro: smart-extractor: gated-candidate supersede invalidation skipped because bulkStore() did not return created entries"); @@ -1702,9 +1702,14 @@ export class SmartExtractor { case "supersede": if (dedupResult.matchId && TEMPORAL_VERSIONED_CATEGORIES.has(candidate.category)) { - await this.handleSupersede(candidate, vector, dedupResult.matchId, sessionKey, targetScope, scopeFilter, this.admissionWriteEvidenceFor(candidate, admission), createEntries, pendingSupersedeInvalidations, agentId); + const supersedeOutcome = await this.handleSupersede(candidate, vector, dedupResult.matchId, sessionKey, targetScope, scopeFilter, this.admissionWriteEvidenceFor(candidate, admission), createEntries, pendingSupersedeInvalidations, agentId); stats.created++; - stats.superseded = (stats.superseded ?? 0) + 1; + // Deferred invalidations count only after they are CONFIRMED in + // applyPendingSupersedeInvalidations; a failed/downgraded supersede + // is a plain create and must not inflate the superseded stat. + if (supersedeOutcome === "superseded") { + stats.superseded = (stats.superseded ?? 0) + 1; + } } else { createEntries?.push(this.buildStoreEntry(candidate, vector, sessionKey, targetScope, this.admissionWriteEvidenceFor(candidate, admission))); @@ -1743,9 +1748,11 @@ export class SmartExtractor { if (dedupResult.matchId) { if (TEMPORAL_VERSIONED_CATEGORIES.has(candidate.category) && dedupResult.contextLabel === "general") { - await this.handleSupersede(candidate, vector, dedupResult.matchId, sessionKey, targetScope, scopeFilter, this.admissionWriteEvidenceFor(candidate, admission), createEntries, pendingSupersedeInvalidations, agentId); + const contradictSupersede = await this.handleSupersede(candidate, vector, dedupResult.matchId, sessionKey, targetScope, scopeFilter, this.admissionWriteEvidenceFor(candidate, admission), createEntries, pendingSupersedeInvalidations, agentId); stats.created++; - stats.superseded = (stats.superseded ?? 0) + 1; + if (contradictSupersede === "superseded") { + stats.superseded = (stats.superseded ?? 0) + 1; + } } else { await this.handleContradict(candidate, vector, dedupResult.matchId, sessionKey, targetScope, scopeFilter, dedupResult.contextLabel, this.admissionWriteEvidenceFor(candidate, admission), createEntries, agentId); @@ -2023,7 +2030,7 @@ export class SmartExtractor { this.log("memory-pro: smart-extractor: merge LLM failed, skipping merge"); return "llm-failed"; } - const applied = await this.applyMergedContent(matchId, candidate.category, merged, targetScope, scopeFilter, [contextLabel], admissionAudit, agentId); + const applied = await this.applyMergedContent(matchId, candidate.category, merged, targetScope, scopeFilter, [contextLabel], [admissionAudit], agentId); if (applied === "target-missing") { createEntries?.push(await this.externalOrBuiltFallbackEntry(candidate, targetScope, "merge-fallback", undefined, admissionAudit)); return "created"; @@ -2168,7 +2175,7 @@ export class SmartExtractor { continue; } try { - const applied = await this.applyMergedContent(job.matchId, job.category, merged, job.targetScope, job.scopeFilter, job.additions.map((a) => a.contextLabel), job.additions[0]?.admissionAudit, job.agentId, mirrorSourceFor(job)); + const applied = await this.applyMergedContent(job.matchId, job.category, merged, job.targetScope, job.scopeFilter, job.additions.map((a) => a.contextLabel), job.additions.map((a) => a.admissionAudit), job.agentId, mirrorSourceFor(job)); if (applied === "target-missing") { failOpenAdditions(job, "target vanished"); continue; @@ -2235,7 +2242,7 @@ export class SmartExtractor { * stats update once per merged-in candidate. Shared by the inline * single-call merge path and the batched merge writer. */ - async applyMergedContent(matchId, category, merged, targetScope, scopeFilter, contextLabels, admissionAudit, agentId, mirrorSource = "smart-extraction") { + async applyMergedContent(matchId, category, merged, targetScope, scopeFilter, contextLabels, admissionEvidence, agentId, mirrorSource = "smart-extraction") { // Re-embed the merged content const mergedText = `${merged.abstract} ${merged.content}`; const newVector = await this.embedder.embed(mergedText); @@ -2256,14 +2263,18 @@ export class SmartExtractor { // category views on a cross-category merge. const existingMeta = parseSmartMetadata(existing.metadata, existing); const targetCategory = existingMeta.memory_category || category; - const metadata = stringifySmartMetadata(this.withAdmissionAudit(buildSmartMetadata(existing, { + // A grouped merge folds N additions into one write: the first addition + // keeps the historical single-evidence semantics, and every later + // addition's audit/fail-open evidence is appended to the capped + // append-only field so no row's admission provenance is dropped. + const metadata = stringifySmartMetadata(this.appendAdditionalAdmissionEvidence(this.withAdmissionAudit(buildSmartMetadata(existing, { l0_abstract: merged.abstract, l1_overview: merged.overview, l2_content: merged.content, memory_category: targetCategory, tier: "working", confidence: 0.8, - }), admissionAudit)); + }), admissionEvidence?.[0]), admissionEvidence?.slice(1) ?? [])); const updated = await this.store.update(matchId, { text: merged.abstract, vector: newVector, @@ -2306,7 +2317,7 @@ export class SmartExtractor { const existing = await this.store.getById(matchId, scopeFilter); if (!existing) { createEntries?.push(await this.externalOrBuiltFallbackEntry(candidate, targetScope, sessionKey, vector, admissionAudit)); - return; + return "create-only"; } const now = Date.now(); const existingMeta = parseSmartMetadata(existing.metadata, existing); @@ -2361,29 +2372,52 @@ export class SmartExtractor { createEntries.push(entry); pendingSupersedeInvalidations.push({ entryIndex, + entry, matchId, existing, factKey, scopeFilter, }); - return; + return "deferred"; } const created = await this.store.store(entry); const invalidated = await this.invalidateSupersededMemory(matchId, existing, factKey, created, scopeFilter); await this.notifyPersisted({ text: created.text, category: created.category, scope: created.scope, timestamp: created.timestamp }, "smart-extraction", agentId); if (invalidated) { this.log(`memory-pro: smart-extractor: superseded [${candidate.category}] ${matchId.slice(0, 8)} -> ${created.id.slice(0, 8)}`); + return "superseded"; } + return "create-only"; } - async applyPendingSupersedeInvalidations(createdEntries, pendingSupersedeInvalidations) { + async applyPendingSupersedeInvalidations(queuedEntries, createdEntries, pendingSupersedeInvalidations, stats) { + const claimedIds = new Set(); + const resolveCreated = (pending) => { + if (createdEntries.length === queuedEntries.length) { + return createdEntries[pending.entryIndex]; + } + // bulkStore accepted fewer entries than were queued, so positions have + // shifted: bind by stable entry identity (the same fallback the + // sibling-verdict resolver uses) and never invalidate unless the exact + // replacement row is found — a positional read here could point the + // old row's superseded_by at an unrelated create. + const want = pending.entry; + return createdEntries.find((e) => e.text === want.text && + e.category === want.category && + laneFromMetadata(e.metadata) === laneFromMetadata(want.metadata) && + !claimedIds.has(e.id)); + }; for (const pending of pendingSupersedeInvalidations) { - const created = createdEntries[pending.entryIndex]; + const created = resolveCreated(pending); if (!created) { - this.log(`memory-pro: smart-extractor: supersede invalidation skipped for ${pending.matchId.slice(0, 8)} because batch create returned no matching entry`); + this.log(`memory-pro: smart-extractor: supersede invalidation skipped for ${pending.matchId.slice(0, 8)} because batch create returned no matching replacement entry`); continue; } + claimedIds.add(created.id); const invalidated = await this.invalidateSupersededMemory(pending.matchId, pending.existing, pending.factKey, created, pending.scopeFilter); if (invalidated) { + if (stats) { + stats.superseded = (stats.superseded ?? 0) + 1; + } this.log(`memory-pro: smart-extractor: superseded ${pending.matchId.slice(0, 8)} -> ${created.id.slice(0, 8)}`); } } @@ -2422,16 +2456,25 @@ export class SmartExtractor { } } async downgradeSupersedeToCreate(matchId, created, scopeFilter, cause) { - this.log(`memory-pro: smart-extractor: supersede invalidation failed for ${matchId.slice(0, 8)} (${cause}) — outcome downgraded to plain CREATE, old row remains active`); + let stripped = false; try { const createdMeta = parseSmartMetadata(created.metadata, created); delete createdMeta.supersedes; createdMeta.relations = (createdMeta.relations ?? []).filter((r) => !(r.type === "supersedes" && r.targetId === matchId)); - await this.store.update(created.id, { metadata: stringifySmartMetadata(createdMeta) }, scopeFilter); + stripped = Boolean(await this.store.update(created.id, { metadata: stringifySmartMetadata(createdMeta) }, scopeFilter)); } catch (stripErr) { this.log(`memory-pro: smart-extractor: supersede-claim strip failed for ${created.id.slice(0, 8)}: ${String(stripErr)}`); } + if (stripped) { + this.log(`memory-pro: smart-extractor: supersede invalidation failed for ${matchId.slice(0, 8)} (${cause}) — outcome downgraded to plain CREATE, old row remains active`); + } + else { + // The downgrade itself could not be confirmed: the old row is still + // active AND the replacement still carries a durable supersedes claim. + // Surface the unresolved repair state instead of a success-style log. + this.log(`memory-pro: smart-extractor: UNRESOLVED supersede repair for ${matchId.slice(0, 8)} (${cause}) — old row remains active and replacement ${created.id.slice(0, 8)} still carries its supersedes claim (strip unconfirmed)`); + } } // -------------------------------------------------------------------------- // Context-Aware Handlers (support / contextualize / contradict) @@ -2463,10 +2506,18 @@ export class SmartExtractor { */ async handleContextualize(candidate, vector, matchId, sessionKey, targetScope, scopeFilter, contextLabel, admissionAudit, createEntries, agentId) { // A vanished target downgrades to an ordinary create: never persist a - // relation to a row that no longer exists. - const targetExists = Boolean(await this.store.getById(matchId, scopeFilter)); - if (!targetExists) { - this.log(`memory-pro: smart-extractor: contextualize target ${matchId.slice(0, 8)} no longer exists — storing as ordinary create without a relation`); + // relation to a row that no longer exists. A THROWING read gets the same + // treatment — dropping an admitted candidate over a transient store + // failure would silently lose it. + let targetExists = false; + try { + targetExists = Boolean(await this.store.getById(matchId, scopeFilter)); + if (!targetExists) { + this.log(`memory-pro: smart-extractor: contextualize target ${matchId.slice(0, 8)} no longer exists — storing as ordinary create without a relation`); + } + } + catch (readErr) { + this.log(`memory-pro: smart-extractor: contextualize target read failed for ${matchId.slice(0, 8)} (${String(readErr)}) — storing as ordinary create without a relation`); } const contextualizeRelations = targetExists ? [{ type: "contextualizes", targetId: matchId }] @@ -2517,22 +2568,35 @@ export class SmartExtractor { * on the original memory's support stats. */ async handleContradict(candidate, vector, matchId, sessionKey, targetScope, scopeFilter, contextLabel, admissionAudit, createEntries, agentId) { - // 1. Record contradiction on the existing memory - const existing = await this.store.getById(matchId, scopeFilter); - if (existing) { - const meta = parseSmartMetadata(existing.metadata, existing); - const supportInfo = parseSupportInfo(meta.support_info); - const updated = updateSupportStats(supportInfo, contextLabel, "contradict"); - meta.support_info = updated; - await this.store.update(matchId, { metadata: stringifySmartMetadata(meta) }, scopeFilter); + // 1. Record contradiction on the existing memory. The relation below is + // persisted only when this evidence write CONFIRMS the target still + // exists — a read/update that throws or reports nothing written means + // the target is gone (or unprovable), and a relation to it would dangle. + let targetLinked = false; + try { + const existing = await this.store.getById(matchId, scopeFilter); + if (existing) { + const meta = parseSmartMetadata(existing.metadata, existing); + const supportInfo = parseSupportInfo(meta.support_info); + const updated = updateSupportStats(supportInfo, contextLabel, "contradict"); + meta.support_info = updated; + const written = await this.store.update(matchId, { metadata: stringifySmartMetadata(meta) }, scopeFilter); + if (written) { + targetLinked = true; + } + else { + this.log(`memory-pro: smart-extractor: contradict target ${matchId.slice(0, 8)} vanished during update — storing as ordinary create without a relation`); + } + } + else { + this.log(`memory-pro: smart-extractor: contradict target ${matchId.slice(0, 8)} no longer exists — storing as ordinary create without a relation`); + } } - else { - this.log(`memory-pro: smart-extractor: contradict target ${matchId.slice(0, 8)} no longer exists — storing as ordinary create without a relation`); + catch (evidenceErr) { + this.log(`memory-pro: smart-extractor: contradict target read/update failed for ${matchId.slice(0, 8)} (${String(evidenceErr)}) — storing as ordinary create without a relation`); } - // 2. Store the contradicting entry as a new memory. A vanished target - // downgrades to an ordinary create: never persist a relation to a row - // that no longer exists. - const contradictRelations = existing ? [{ type: "contradicts", targetId: matchId }] : []; + // 2. Store the contradicting entry as a new memory. + const contradictRelations = targetLinked ? [{ type: "contradicts", targetId: matchId }] : []; const storeCategory = this.mapToStoreCategory(candidate.category); const metadata = stringifySmartMetadata(this.withAdmissionAudit({ l0_abstract: candidate.abstract, @@ -2731,6 +2795,24 @@ export class SmartExtractor { } return { ...metadata, admission_control: admissionAudit }; } + /** + * Append every provided evidence record (full audits and fail-open markers + * alike) to the capped append-only bypass field. Used for grouped merges, + * where additions beyond the first would otherwise lose their admission + * provenance entirely. + */ + appendAdditionalAdmissionEvidence(metadata, evidence) { + const additional = evidence.filter((e) => Boolean(e)); + if (additional.length === 0 || !this.persistAdmissionAudit) { + return metadata; + } + const prior = Array.isArray(metadata.admission_bypass_events) + ? metadata.admission_bypass_events + : []; + const now = Date.now(); + const events = [...prior, ...additional.map((e) => ({ at: now, ...e }))].slice(-MAX_ADMISSION_BYPASS_EVENTS); + return { ...metadata, admission_bypass_events: events }; + } /** * Record a rejected admission to the durable audit log. */ diff --git a/src/smart-extractor.ts b/src/smart-extractor.ts index 62429768d..cc20cc687 100644 --- a/src/smart-extractor.ts +++ b/src/smart-extractor.ts @@ -162,6 +162,10 @@ type PendingMergeJob = { }; type PendingSupersedeInvalidation = { entryIndex: number; + // The exact queued entry object: bulkStore may filter entries and return a + // shorter array, shifting positions, so resolution falls back to stable + // entry identity (text + category + lane) instead of trusting the index. + entry: StoreEntry; matchId: string; existing: MemoryEntry; factKey: string; @@ -971,8 +975,10 @@ export class SmartExtractor { const createdEntries = await this.bulkStoreAndValidate(createEntries); if (createdEntries) { await this.applyPendingSupersedeInvalidations( + createEntries, createdEntries, pendingSupersedeInvalidations, + stats, ); for (const created of createdEntries) { await this.notifyPersisted( @@ -1277,7 +1283,7 @@ export class SmartExtractor { const stored = await this.bulkStoreAndValidate(createEntries); if (stored) { createdEntries = stored; - await this.applyPendingSupersedeInvalidations(stored, pendingSupersedeInvalidations); + await this.applyPendingSupersedeInvalidations(createEntries, stored, pendingSupersedeInvalidations, stats); } else if (pendingSupersedeInvalidations.length > 0) { this.log( "memory-pro: smart-extractor: gated-candidate supersede invalidation skipped because bulkStore() did not return created entries", @@ -2323,7 +2329,7 @@ export class SmartExtractor { dedupResult.matchId && TEMPORAL_VERSIONED_CATEGORIES.has(candidate.category) ) { - await this.handleSupersede( + const supersedeOutcome = await this.handleSupersede( candidate, vector, dedupResult.matchId, @@ -2336,7 +2342,12 @@ export class SmartExtractor { agentId, ); stats.created++; - stats.superseded = (stats.superseded ?? 0) + 1; + // Deferred invalidations count only after they are CONFIRMED in + // applyPendingSupersedeInvalidations; a failed/downgraded supersede + // is a plain create and must not inflate the superseded stat. + if (supersedeOutcome === "superseded") { + stats.superseded = (stats.superseded ?? 0) + 1; + } } else { createEntries?.push(this.buildStoreEntry(candidate, vector, sessionKey, targetScope, this.admissionWriteEvidenceFor(candidate, admission))); stats.created++; @@ -2376,7 +2387,7 @@ export class SmartExtractor { TEMPORAL_VERSIONED_CATEGORIES.has(candidate.category) && dedupResult.contextLabel === "general" ) { - await this.handleSupersede( + const contradictSupersede = await this.handleSupersede( candidate, vector, dedupResult.matchId, @@ -2389,7 +2400,9 @@ export class SmartExtractor { agentId, ); stats.created++; - stats.superseded = (stats.superseded ?? 0) + 1; + if (contradictSupersede === "superseded") { + stats.superseded = (stats.superseded ?? 0) + 1; + } } else { await this.handleContradict(candidate, vector, dedupResult.matchId, sessionKey, targetScope, scopeFilter, dedupResult.contextLabel, this.admissionWriteEvidenceFor(candidate, admission), createEntries, agentId); stats.created++; @@ -2810,7 +2823,7 @@ export class SmartExtractor { targetScope, scopeFilter, [contextLabel], - admissionAudit, + [admissionAudit], agentId, ); if (applied === "target-missing") { @@ -3006,7 +3019,7 @@ export class SmartExtractor { job.targetScope, job.scopeFilter, job.additions.map((a) => a.contextLabel), - job.additions[0]?.admissionAudit, + job.additions.map((a) => a.admissionAudit), job.agentId, mirrorSourceFor(job), ); @@ -3096,7 +3109,7 @@ export class SmartExtractor { targetScope: string, scopeFilter: string[] | undefined, contextLabels: Array, - admissionAudit: AdmissionWriteEvidence | undefined, + admissionEvidence: Array | undefined, agentId: string | undefined, mirrorSource: string = "smart-extraction", ): Promise<"updated" | "target-missing"> { @@ -3123,17 +3136,24 @@ export class SmartExtractor { // category views on a cross-category merge. const existingMeta = parseSmartMetadata(existing.metadata, existing); const targetCategory = (existingMeta.memory_category as MemoryCategory) || category; + // A grouped merge folds N additions into one write: the first addition + // keeps the historical single-evidence semantics, and every later + // addition's audit/fail-open evidence is appended to the capped + // append-only field so no row's admission provenance is dropped. const metadata = stringifySmartMetadata( - this.withAdmissionAudit( - buildSmartMetadata(existing, { - l0_abstract: merged.abstract, - l1_overview: merged.overview, - l2_content: merged.content, - memory_category: targetCategory, - tier: "working", - confidence: 0.8, - }), - admissionAudit, + this.appendAdditionalAdmissionEvidence( + this.withAdmissionAudit( + buildSmartMetadata(existing, { + l0_abstract: merged.abstract, + l1_overview: merged.overview, + l2_content: merged.content, + memory_category: targetCategory, + tier: "working", + confidence: 0.8, + }), + admissionEvidence?.[0], + ), + admissionEvidence?.slice(1) ?? [], ), ); @@ -3201,13 +3221,13 @@ export class SmartExtractor { createEntries?: StoreEntry[], pendingSupersedeInvalidations?: PendingSupersedeInvalidation[], agentId?: string, - ): Promise { + ): Promise<"deferred" | "superseded" | "create-only"> { const existing = await this.store.getById(matchId, scopeFilter); if (!existing) { createEntries?.push( await this.externalOrBuiltFallbackEntry(candidate, targetScope, sessionKey, vector, admissionAudit), ); - return; + return "create-only"; } const now = Date.now(); @@ -3270,12 +3290,13 @@ export class SmartExtractor { createEntries.push(entry); pendingSupersedeInvalidations.push({ entryIndex, + entry, matchId, existing, factKey, scopeFilter, }); - return; + return "deferred"; } const created = await this.store.store(entry); @@ -3296,21 +3317,47 @@ export class SmartExtractor { this.log( `memory-pro: smart-extractor: superseded [${candidate.category}] ${matchId.slice(0, 8)} -> ${created.id.slice(0, 8)}`, ); + return "superseded"; } + return "create-only"; } private async applyPendingSupersedeInvalidations( + queuedEntries: StoreEntry[], createdEntries: MemoryEntry[], pendingSupersedeInvalidations: PendingSupersedeInvalidation[], + stats?: { superseded?: number }, ): Promise { + const claimedIds = new Set(); + const resolveCreated = ( + pending: PendingSupersedeInvalidation, + ): MemoryEntry | undefined => { + if (createdEntries.length === queuedEntries.length) { + return createdEntries[pending.entryIndex]; + } + // bulkStore accepted fewer entries than were queued, so positions have + // shifted: bind by stable entry identity (the same fallback the + // sibling-verdict resolver uses) and never invalidate unless the exact + // replacement row is found — a positional read here could point the + // old row's superseded_by at an unrelated create. + const want = pending.entry; + return createdEntries.find( + (e) => + e.text === want.text && + e.category === want.category && + laneFromMetadata(e.metadata) === laneFromMetadata(want.metadata) && + !claimedIds.has(e.id), + ); + }; for (const pending of pendingSupersedeInvalidations) { - const created = createdEntries[pending.entryIndex]; + const created = resolveCreated(pending); if (!created) { this.log( - `memory-pro: smart-extractor: supersede invalidation skipped for ${pending.matchId.slice(0, 8)} because batch create returned no matching entry`, + `memory-pro: smart-extractor: supersede invalidation skipped for ${pending.matchId.slice(0, 8)} because batch create returned no matching replacement entry`, ); continue; } + claimedIds.add(created.id); const invalidated = await this.invalidateSupersededMemory( pending.matchId, pending.existing, @@ -3319,6 +3366,9 @@ export class SmartExtractor { pending.scopeFilter, ); if (invalidated) { + if (stats) { + stats.superseded = (stats.superseded ?? 0) + 1; + } this.log( `memory-pro: smart-extractor: superseded ${pending.matchId.slice(0, 8)} -> ${created.id.slice(0, 8)}`, ); @@ -3376,25 +3426,37 @@ export class SmartExtractor { scopeFilter: string[] | undefined, cause: string, ): Promise { - this.log( - `memory-pro: smart-extractor: supersede invalidation failed for ${matchId.slice(0, 8)} (${cause}) — outcome downgraded to plain CREATE, old row remains active`, - ); + let stripped = false; try { const createdMeta = parseSmartMetadata(created.metadata, created); delete (createdMeta as Record).supersedes; createdMeta.relations = (createdMeta.relations ?? []).filter( (r) => !(r.type === "supersedes" && r.targetId === matchId), ); - await this.store.update( - created.id, - { metadata: stringifySmartMetadata(createdMeta) }, - scopeFilter, + stripped = Boolean( + await this.store.update( + created.id, + { metadata: stringifySmartMetadata(createdMeta) }, + scopeFilter, + ), ); } catch (stripErr) { this.log( `memory-pro: smart-extractor: supersede-claim strip failed for ${created.id.slice(0, 8)}: ${String(stripErr)}`, ); } + if (stripped) { + this.log( + `memory-pro: smart-extractor: supersede invalidation failed for ${matchId.slice(0, 8)} (${cause}) — outcome downgraded to plain CREATE, old row remains active`, + ); + } else { + // The downgrade itself could not be confirmed: the old row is still + // active AND the replacement still carries a durable supersedes claim. + // Surface the unresolved repair state instead of a success-style log. + this.log( + `memory-pro: smart-extractor: UNRESOLVED supersede repair for ${matchId.slice(0, 8)} (${cause}) — old row remains active and replacement ${created.id.slice(0, 8)} still carries its supersedes claim (strip unconfirmed)`, + ); + } } // -------------------------------------------------------------------------- @@ -3460,11 +3522,20 @@ export class SmartExtractor { agentId?: string, ): Promise { // A vanished target downgrades to an ordinary create: never persist a - // relation to a row that no longer exists. - const targetExists = Boolean(await this.store.getById(matchId, scopeFilter)); - if (!targetExists) { + // relation to a row that no longer exists. A THROWING read gets the same + // treatment — dropping an admitted candidate over a transient store + // failure would silently lose it. + let targetExists = false; + try { + targetExists = Boolean(await this.store.getById(matchId, scopeFilter)); + if (!targetExists) { + this.log( + `memory-pro: smart-extractor: contextualize target ${matchId.slice(0, 8)} no longer exists — storing as ordinary create without a relation`, + ); + } + } catch (readErr) { this.log( - `memory-pro: smart-extractor: contextualize target ${matchId.slice(0, 8)} no longer exists — storing as ordinary create without a relation`, + `memory-pro: smart-extractor: contextualize target read failed for ${matchId.slice(0, 8)} (${String(readErr)}) — storing as ordinary create without a relation`, ); } const contextualizeRelations = targetExists @@ -3535,28 +3606,43 @@ export class SmartExtractor { createEntries?: StoreEntry[], agentId?: string, ): Promise { - // 1. Record contradiction on the existing memory - const existing = await this.store.getById(matchId, scopeFilter); - if (existing) { - const meta = parseSmartMetadata(existing.metadata, existing); - const supportInfo = parseSupportInfo(meta.support_info); - const updated = updateSupportStats(supportInfo, contextLabel, "contradict"); - meta.support_info = updated; - await this.store.update( - matchId, - { metadata: stringifySmartMetadata(meta) }, - scopeFilter, - ); - } else { + // 1. Record contradiction on the existing memory. The relation below is + // persisted only when this evidence write CONFIRMS the target still + // exists — a read/update that throws or reports nothing written means + // the target is gone (or unprovable), and a relation to it would dangle. + let targetLinked = false; + try { + const existing = await this.store.getById(matchId, scopeFilter); + if (existing) { + const meta = parseSmartMetadata(existing.metadata, existing); + const supportInfo = parseSupportInfo(meta.support_info); + const updated = updateSupportStats(supportInfo, contextLabel, "contradict"); + meta.support_info = updated; + const written = await this.store.update( + matchId, + { metadata: stringifySmartMetadata(meta) }, + scopeFilter, + ); + if (written) { + targetLinked = true; + } else { + this.log( + `memory-pro: smart-extractor: contradict target ${matchId.slice(0, 8)} vanished during update — storing as ordinary create without a relation`, + ); + } + } else { + this.log( + `memory-pro: smart-extractor: contradict target ${matchId.slice(0, 8)} no longer exists — storing as ordinary create without a relation`, + ); + } + } catch (evidenceErr) { this.log( - `memory-pro: smart-extractor: contradict target ${matchId.slice(0, 8)} no longer exists — storing as ordinary create without a relation`, + `memory-pro: smart-extractor: contradict target read/update failed for ${matchId.slice(0, 8)} (${String(evidenceErr)}) — storing as ordinary create without a relation`, ); } - // 2. Store the contradicting entry as a new memory. A vanished target - // downgrades to an ordinary create: never persist a relation to a row - // that no longer exists. - const contradictRelations = existing ? [{ type: "contradicts", targetId: matchId }] : []; + // 2. Store the contradicting entry as a new memory. + const contradictRelations = targetLinked ? [{ type: "contradicts", targetId: matchId }] : []; const storeCategory = this.mapToStoreCategory(candidate.category); const metadata = stringifySmartMetadata(this.withAdmissionAudit({ l0_abstract: candidate.abstract, @@ -3814,6 +3900,30 @@ export class SmartExtractor { return { ...metadata, admission_control: admissionAudit }; } + /** + * Append every provided evidence record (full audits and fail-open markers + * alike) to the capped append-only bypass field. Used for grouped merges, + * where additions beyond the first would otherwise lose their admission + * provenance entirely. + */ + private appendAdditionalAdmissionEvidence>( + metadata: T, + evidence: Array, + ): T { + const additional = evidence.filter((e): e is AdmissionWriteEvidence => Boolean(e)); + if (additional.length === 0 || !this.persistAdmissionAudit) { + return metadata; + } + const prior = Array.isArray((metadata as Record).admission_bypass_events) + ? ((metadata as Record).admission_bypass_events as unknown[]) + : []; + const now = Date.now(); + const events = [...prior, ...additional.map((e) => ({ at: now, ...e }))].slice( + -MAX_ADMISSION_BYPASS_EVENTS, + ); + return { ...metadata, admission_bypass_events: events }; + } + /** * Record a rejected admission to the durable audit log. */ diff --git a/test/reflection-mapped-uniform-pipeline.test.mjs b/test/reflection-mapped-uniform-pipeline.test.mjs index 2c702862c..d53e01857 100644 --- a/test/reflection-mapped-uniform-pipeline.test.mjs +++ b/test/reflection-mapped-uniform-pipeline.test.mjs @@ -966,3 +966,237 @@ describe("reflection mapped rows: round-5 data-integrity regressions", () => { assert.ok(meta.support_info, "the support stats still update"); }); }); + +// Round-6 review regressions: failure-path integrity — identity-stable +// deferred invalidation under bulkStore filtering, complete evidence on +// grouped merges, guarded contextualize/contradict fallbacks, confirmed-only +// downgrade and statistics. Fixtures are entirely synthetic. +describe("reflection mapped rows: round-6 failure-path integrity", () => { + it("binds a deferred supersede invalidation by entry identity when bulkStore filters an earlier row", async () => { + const store = makeStore({ + neighbors: [neighborRow("row-1", "The staging balancer drains connections before each maintenance window.")], + }); + const filteredText = "Rotate the artifact signing key at the start of every quarter."; + const baseBulkStore = store.bulkStore.bind(store); + store.bulkStore = async (entries) => { + const accepted = entries.filter((e) => e.text !== filteredText); + store.bulkStored.push(...entries); + const stored = accepted.map((e, i) => ({ ...e, id: `new-${i + 2}`, timestamp: 1_700_000_500_000 })); + for (const s of stored) store.rows.set(s.id, s); + return stored; + }; + void baseBulkStore; + const llm = makeLlm({ + onDedupBatch: () => ({ + results: [ + { index: 1, decision: "create", reason: "novel" }, + { index: 2, decision: "supersede", match_index: 1, reason: "newer fact" }, + { index: 3, decision: "create", reason: "novel" }, + ], + }), + }); + const extractor = makeExtractor(store, llm); + + const { stats } = await extractor.persistGatedCandidates( + [ + reflectionItem(filteredText), + reflectionItem("The staging balancer now drains connections only during the overnight window.", { category: "preferences" }), + reflectionItem("Publish the deprecation calendar to the shared operations wiki."), + ], + { targetScope: "agent:probe", scopeFilter: ["agent:probe"], sessionKey: "refl-test" }, + ); + + const invalidation = store.updates.find( + (u) => u.id === "row-1" && u.patch?.metadata?.includes("superseded_by"), + ); + assert.ok(invalidation, "the surviving replacement still invalidates its target"); + const oldMeta = JSON.parse(invalidation.patch.metadata); + assert.equal( + oldMeta.superseded_by, + "new-2", + "superseded_by must point at the actual replacement, not the row that shifted into its position", + ); + assert.notEqual(oldMeta.superseded_by, "new-3", "an unrelated create must never claim the supersede"); + assert.equal(stats.superseded, 1); + }); + + it("skips a deferred invalidation entirely when bulkStore filtered the replacement itself", async () => { + const store = makeStore({ + neighbors: [neighborRow("row-1", "Cache warmers replay the top queries after every deploy completes.")], + }); + const filteredText = "Cache warmers now replay only the checkout queries after deploys."; + store.bulkStore = async (entries) => { + const accepted = entries.filter((e) => e.text !== filteredText); + store.bulkStored.push(...entries); + const stored = accepted.map((e, i) => ({ ...e, id: `new-${i + 2}`, timestamp: 1_700_000_500_000 })); + for (const s of stored) store.rows.set(s.id, s); + return stored; + }; + const llm = makeLlm({ + onDedupBatch: () => ({ + results: [ + { index: 1, decision: "supersede", match_index: 1, reason: "newer fact" }, + { index: 2, decision: "create", reason: "novel" }, + ], + }), + }); + const extractor = makeExtractor(store, llm); + + const { stats } = await extractor.persistGatedCandidates( + [ + reflectionItem(filteredText, { category: "preferences" }), + reflectionItem("Route the weekly digest through the notifications relay instead of direct send."), + ], + { targetScope: "agent:probe", scopeFilter: ["agent:probe"], sessionKey: "refl-test" }, + ); + + assert.equal( + store.updates.some((u) => u.id === "row-1" && u.patch?.metadata?.includes("superseded_by")), + false, + "no replacement row exists, so the old row must stay untouched", + ); + assert.equal(stats.superseded ?? 0, 0, "an unconfirmed supersede must not be counted"); + }); + + it("aggregates every merge addition's fail-open evidence, not only the first addition's", async () => { + const store = makeStore({ + neighbors: [auditedNeighborRow("row-1", "Keep the incident timeline template pinned in the response channel.")], + }); + const llm = makeLlm({ + onDedupBatch: () => ({ + results: [ + { index: 1, decision: "merge", match_index: 1, reason: "adds detail" }, + { index: 2, decision: "merge", match_index: 1, reason: "adds detail" }, + ], + }), + onMergeBatch: () => ({ + results: [{ index: 1, abstract: "merged abstract", overview: "o", content: "merged content" }], + }), + }); + const extractor = makeExtractor(store, llm, { admissionControl: { enabled: true } }); + + await extractor.persistGatedCandidates( + [ + reflectionItem("Pin the incident timeline template near the top of the response channel."), + failOpenReflectionItem("Link the incident timeline template from the escalation runbook too."), + ], + { targetScope: "agent:probe", scopeFilter: ["agent:probe"], sessionKey: "refl-test" }, + ); + + const targetWrite = store.updates.find((u) => u.id === "row-1" && u.patch?.metadata?.includes("l0_abstract")); + assert.ok(targetWrite, "the grouped merge must update its target"); + const meta = JSON.parse(targetWrite.patch.metadata); + assert.deepEqual(meta.admission_control, PRODUCTION_MAPPED_AUDIT, "the target's complete audit survives"); + assert.ok(Array.isArray(meta.admission_bypass_events), "the second addition's marker must be recorded"); + assert.equal(meta.admission_bypass_events.length, 1); + assert.equal(meta.admission_bypass_events[0].failedOpen, true, "the fail-open marker from the non-first addition survives"); + }); + + it("falls back to an unlinked create when the contextualize target read throws", async () => { + const logs = []; + const store = makeStore({ + neighbors: [neighborRow("row-1", "Pre-warm the reporting cluster ahead of the month-end close.")], + }); + store.getById = async (id) => { + if (id === "row-1") throw new Error("read outage"); + return store.rows.get(id) ?? null; + }; + const llm = makeLlm({ + onDedupBatch: () => ({ + results: [{ index: 1, decision: "contextualize", match_index: 1, reason: "adds nuance" }], + }), + }); + const extractor = makeExtractor(store, llm, { log: (m) => logs.push(m) }); + + const { createdEntries } = await extractor.persistGatedCandidates( + [reflectionItem("Pre-warming matters most when the close lands right after a long weekend.")], + { targetScope: "agent:probe", scopeFilter: ["agent:probe"], sessionKey: "refl-test" }, + ); + + assert.equal(createdEntries.length, 1, "the admitted candidate must still land"); + assert.ok( + logs.some((m) => m.includes("contextualize target read failed")), + "the read failure resolves through the guarded contextualize fallback, not the generic processing-failure catch", + ); + const meta = JSON.parse(store.bulkStored[0].metadata); + assert.equal( + (meta.relations ?? []).some((r) => r.type === "contextualizes"), + false, + "no relation may point at an unreadable target", + ); + }); + + it("stores a contradict candidate without a relation when the evidence update writes nothing", async () => { + const logs = []; + const store = makeStore({ + neighbors: [neighborRow("row-1", "Roll access reviews on the first business day of each month.")], + }); + const baseUpdate = store.update.bind(store); + store.update = async (id, patch, scopeFilter) => { + if (id === "row-1") return null; + return baseUpdate(id, patch, scopeFilter); + }; + const llm = makeLlm({ + onDedupBatch: () => ({ + results: [{ index: 1, decision: "contradict", match_index: 1, reason: "opposite practice" }], + }), + }); + const extractor = makeExtractor(store, llm, { log: (m) => logs.push(m) }); + + const { createdEntries } = await extractor.persistGatedCandidates( + [reflectionItem("Access reviews actually roll on the last business day of each month.")], + { targetScope: "agent:probe", scopeFilter: ["agent:probe"], sessionKey: "refl-test" }, + ); + + assert.equal(createdEntries.length, 1, "the contradicting row still lands"); + const meta = JSON.parse(store.bulkStored[0].metadata); + assert.equal( + (meta.relations ?? []).some((r) => r.type === "contradicts"), + false, + "a nothing-written evidence update must not leave a dangling contradicts relation", + ); + assert.ok( + logs.some((m) => m.includes("vanished during update")), + "the null update is surfaced as a vanished target", + ); + }); + + it("surfaces an unresolved repair when the downgrade strip also writes nothing, and counts only confirmed supersedes", async () => { + const logs = []; + const store = makeStore({ + neighbors: [ + neighborRow("row-1", "Mirror the build artifacts into the secondary region nightly."), + neighborRow("row-2", "Contract tests run against the recorded provider snapshots."), + ], + }); + const baseUpdate = store.update.bind(store); + store.update = async (id, patch, scopeFilter) => { + if (id === "row-1" && patch?.metadata?.includes("superseded_by")) return null; + if (id === "new-3" ) return null; + return baseUpdate(id, patch, scopeFilter); + }; + const llm = makeLlm({ + onDedupBatch: () => ({ + results: [ + { index: 1, decision: "supersede", match_index: 1, reason: "newer fact" }, + { index: 2, decision: "supersede", match_index: 2, reason: "newer fact" }, + ], + }), + }); + const extractor = makeExtractor(store, llm, { log: (m) => logs.push(m) }); + + const { stats } = await extractor.persistGatedCandidates( + [ + reflectionItem("Build artifacts now mirror into the secondary region twice a day.", { category: "preferences" }), + reflectionItem("Contract tests now run against live provider sandboxes instead.", { category: "entities" }), + ], + { targetScope: "agent:probe", scopeFilter: ["agent:probe"], sessionKey: "refl-test" }, + ); + + assert.ok( + logs.some((m) => m.includes("UNRESOLVED supersede repair")), + "an unconfirmed strip must surface the unresolved repair state, never a success-style downgrade log", + ); + assert.equal(stats.superseded, 1, "only the confirmed invalidation is counted"); + }); +}); From dc49f36afbbfa726bf8b944ebcbd4450d5d8229e Mon Sep 17 00:00:00 2001 From: Gorkem Date: Thu, 20 Aug 2026 05:58:28 +0300 Subject: [PATCH 4/7] fix: reuse the resolved surviving anchor across deferred verdicts; split complete audits out of the bypass history - storedIdForSurviving caches each surviving index's resolution so a second MERGE/SUPPORT verdict on the same sibling anchor reuses the row after bulkStore shortens the result; claimedIds only separates distinct surviving entries in the identity fallback. - appendAdditionalAdmissionEvidence keeps admission_bypass_events exclusive to fail-open markers; additional complete audits append to the new capped admission_control_history field so pass audits can never evict genuine bypass evidence. - interpretDedupVerdict drops the redundant casts around context_label. Regressions: the combined filtered-result/shared-anchor case (two supports on one surviving anchor with an unrelated row filtered) and a grouped-merge evidence-routing case; both red-proofed against the unfixed source. --- dist/src/smart-extractor.js | 87 +++++++++------ src/smart-extractor.ts | 102 +++++++++++------- ...eflection-mapped-uniform-pipeline.test.mjs | 97 +++++++++++++++++ 3 files changed, 216 insertions(+), 70 deletions(-) diff --git a/dist/src/smart-extractor.js b/dist/src/smart-extractor.js index d7d9305c1..f2e81d4cf 100644 --- a/dist/src/smart-extractor.js +++ b/dist/src/smart-extractor.js @@ -16,6 +16,7 @@ import { inferAtomicBrandItemPreferenceSlot } from "./preference-slots.js"; import { batchDedup } from "./batch-dedup.js"; import { buildBoundedTranscriptWithStats, } from "./auto-capture-cleanup.js"; const MAX_ADMISSION_BYPASS_EVENTS = 20; +const MAX_ADMISSION_CONTROL_HISTORY = 20; function isFailOpenEvidence(value) { return (value.failedOpen === true && typeof value.provenance === "string"); @@ -907,33 +908,43 @@ export class SmartExtractor { // cannot be resolved keeps the caller's row (fail open to create). if (pendingSiblingVerdicts.length > 0) { const claimedIds = new Set(); + const resolvedIdBySurviving = new Map(); const storedIdForSurviving = (survivingIndex) => { - const slot = createSlotBySurviving.get(survivingIndex); - if (slot === undefined) { - return undefined; + // Verdicts may share one surviving anchor: the first resolution is + // cached per index so every later verdict reuses the same row. + // claimedIds only keeps DISTINCT surviving entries apart in the + // filtered-result fallback — it must never exclude the row an index + // already resolved. + if (resolvedIdBySurviving.has(survivingIndex)) { + return resolvedIdBySurviving.get(survivingIndex); } - if (createdEntries.length === createEntries.length) { - const id = createdEntries[slot]?.id; - if (id) { - claimedIds.add(id); + const resolve = () => { + const slot = createSlotBySurviving.get(survivingIndex); + if (slot === undefined) { + return undefined; } - return id; - } - // bulkStore may filter entries, shifting positions: fall back to the - // first unclaimed row with the same text AND the same lane/category - // identity — identical text is legal across lanes, so a text-only - // match could bind the verdict to another lane's row. - const want = createEntries[slot]; - const hit = want - ? createdEntries.find((e) => e.text === want.text && - e.category === want.category && - laneFromMetadata(e.metadata) === laneFromMetadata(want.metadata) && - !claimedIds.has(e.id)) - : undefined; - if (hit) { - claimedIds.add(hit.id); + if (createdEntries.length === createEntries.length) { + return createdEntries[slot]?.id; + } + // bulkStore may filter entries, shifting positions: fall back to the + // first unclaimed row with the same text AND the same lane/category + // identity — identical text is legal across lanes, so a text-only + // match could bind the verdict to another lane's row. + const want = createEntries[slot]; + const hit = want + ? createdEntries.find((e) => e.text === want.text && + e.category === want.category && + laneFromMetadata(e.metadata) === laneFromMetadata(want.metadata) && + !claimedIds.has(e.id)) + : undefined; + return hit?.id; + }; + const id = resolve(); + if (id) { + claimedIds.add(id); } - return hit?.id; + resolvedIdBySurviving.set(survivingIndex, id); + return id; }; const followupMerges = []; const followupCreates = []; @@ -2796,22 +2807,34 @@ export class SmartExtractor { return { ...metadata, admission_control: admissionAudit }; } /** - * Append every provided evidence record (full audits and fail-open markers - * alike) to the capped append-only bypass field. Used for grouped merges, - * where additions beyond the first would otherwise lose their admission - * provenance entirely. + * Append every non-first addition's evidence from a grouped mutation to a + * capped append-only history, so no addition loses its admission + * provenance. The two evidence kinds stay in separate fields: + * admission_bypass_events remains exclusive to fail-open markers + * (unevaluated content), while complete audits append to + * admission_control_history — a pass audit must never consume the bypass + * cap and evict genuine fail-open evidence. */ appendAdditionalAdmissionEvidence(metadata, evidence) { const additional = evidence.filter((e) => Boolean(e)); if (additional.length === 0 || !this.persistAdmissionAudit) { return metadata; } - const prior = Array.isArray(metadata.admission_bypass_events) - ? metadata.admission_bypass_events - : []; const now = Date.now(); - const events = [...prior, ...additional.map((e) => ({ at: now, ...e }))].slice(-MAX_ADMISSION_BYPASS_EVENTS); - return { ...metadata, admission_bypass_events: events }; + const priorOf = (field) => Array.isArray(metadata[field]) + ? metadata[field] + : []; + const stamped = (records) => records.map((e) => ({ at: now, ...e })); + const bypassMarkers = additional.filter((e) => isFailOpenEvidence(e)); + const completeAudits = additional.filter((e) => !isFailOpenEvidence(e)); + const next = { ...metadata }; + if (bypassMarkers.length > 0) { + next.admission_bypass_events = [...priorOf("admission_bypass_events"), ...stamped(bypassMarkers)].slice(-MAX_ADMISSION_BYPASS_EVENTS); + } + if (completeAudits.length > 0) { + next.admission_control_history = [...priorOf("admission_control_history"), ...stamped(completeAudits)].slice(-MAX_ADMISSION_CONTROL_HISTORY); + } + return next; } /** * Record a rejected admission to the durable audit log. diff --git a/src/smart-extractor.ts b/src/smart-extractor.ts index cc20cc687..a60fb4cde 100644 --- a/src/smart-extractor.ts +++ b/src/smart-extractor.ts @@ -96,6 +96,7 @@ type AdmissionFailOpenEvidence = { type AdmissionWriteEvidence = AdmissionAuditRecord | AdmissionFailOpenEvidence; const MAX_ADMISSION_BYPASS_EVENTS = 20; +const MAX_ADMISSION_CONTROL_HISTORY = 20; function isFailOpenEvidence( value: AdmissionWriteEvidence, @@ -1296,36 +1297,46 @@ export class SmartExtractor { // cannot be resolved keeps the caller's row (fail open to create). if (pendingSiblingVerdicts.length > 0) { const claimedIds = new Set(); + const resolvedIdBySurviving = new Map(); const storedIdForSurviving = (survivingIndex: number): string | undefined => { - const slot = createSlotBySurviving.get(survivingIndex); - if (slot === undefined) { - return undefined; + // Verdicts may share one surviving anchor: the first resolution is + // cached per index so every later verdict reuses the same row. + // claimedIds only keeps DISTINCT surviving entries apart in the + // filtered-result fallback — it must never exclude the row an index + // already resolved. + if (resolvedIdBySurviving.has(survivingIndex)) { + return resolvedIdBySurviving.get(survivingIndex); } - if (createdEntries.length === createEntries.length) { - const id = createdEntries[slot]?.id; - if (id) { - claimedIds.add(id); + const resolve = (): string | undefined => { + const slot = createSlotBySurviving.get(survivingIndex); + if (slot === undefined) { + return undefined; } - return id; - } - // bulkStore may filter entries, shifting positions: fall back to the - // first unclaimed row with the same text AND the same lane/category - // identity — identical text is legal across lanes, so a text-only - // match could bind the verdict to another lane's row. - const want = createEntries[slot]; - const hit = want - ? createdEntries.find( - (e) => - e.text === want.text && - e.category === want.category && - laneFromMetadata(e.metadata) === laneFromMetadata(want.metadata) && - !claimedIds.has(e.id), - ) - : undefined; - if (hit) { - claimedIds.add(hit.id); + if (createdEntries.length === createEntries.length) { + return createdEntries[slot]?.id; + } + // bulkStore may filter entries, shifting positions: fall back to the + // first unclaimed row with the same text AND the same lane/category + // identity — identical text is legal across lanes, so a text-only + // match could bind the verdict to another lane's row. + const want = createEntries[slot]; + const hit = want + ? createdEntries.find( + (e) => + e.text === want.text && + e.category === want.category && + laneFromMetadata(e.metadata) === laneFromMetadata(want.metadata) && + !claimedIds.has(e.id), + ) + : undefined; + return hit?.id; + }; + const id = resolve(); + if (id) { + claimedIds.add(id); } - return hit?.id; + resolvedIdBySurviving.set(survivingIndex, id); + return id; }; const followupMerges: PendingMergeJob[] = []; const followupCreates: StoreEntry[] = []; @@ -2661,7 +2672,7 @@ export class SmartExtractor { decision, reason: data.reason ?? "", matchId: ["merge", "support", "contextualize", "contradict", "supersede"].includes(decision) ? matchEntry?.entry.id : undefined, - contextLabel: typeof (data as any).context_label === "string" ? (data as any).context_label : undefined, + contextLabel: typeof data.context_label === "string" ? data.context_label : undefined, }; } @@ -3901,10 +3912,13 @@ export class SmartExtractor { } /** - * Append every provided evidence record (full audits and fail-open markers - * alike) to the capped append-only bypass field. Used for grouped merges, - * where additions beyond the first would otherwise lose their admission - * provenance entirely. + * Append every non-first addition's evidence from a grouped mutation to a + * capped append-only history, so no addition loses its admission + * provenance. The two evidence kinds stay in separate fields: + * admission_bypass_events remains exclusive to fail-open markers + * (unevaluated content), while complete audits append to + * admission_control_history — a pass audit must never consume the bypass + * cap and evict genuine fail-open evidence. */ private appendAdditionalAdmissionEvidence>( metadata: T, @@ -3914,14 +3928,26 @@ export class SmartExtractor { if (additional.length === 0 || !this.persistAdmissionAudit) { return metadata; } - const prior = Array.isArray((metadata as Record).admission_bypass_events) - ? ((metadata as Record).admission_bypass_events as unknown[]) - : []; const now = Date.now(); - const events = [...prior, ...additional.map((e) => ({ at: now, ...e }))].slice( - -MAX_ADMISSION_BYPASS_EVENTS, - ); - return { ...metadata, admission_bypass_events: events }; + const priorOf = (field: string): unknown[] => + Array.isArray((metadata as Record)[field]) + ? ((metadata as Record)[field] as unknown[]) + : []; + const stamped = (records: AdmissionWriteEvidence[]) => records.map((e) => ({ at: now, ...e })); + const bypassMarkers = additional.filter((e) => isFailOpenEvidence(e)); + const completeAudits = additional.filter((e) => !isFailOpenEvidence(e)); + const next: Record = { ...metadata }; + if (bypassMarkers.length > 0) { + next.admission_bypass_events = [...priorOf("admission_bypass_events"), ...stamped(bypassMarkers)].slice( + -MAX_ADMISSION_BYPASS_EVENTS, + ); + } + if (completeAudits.length > 0) { + next.admission_control_history = [...priorOf("admission_control_history"), ...stamped(completeAudits)].slice( + -MAX_ADMISSION_CONTROL_HISTORY, + ); + } + return next as T; } /** diff --git a/test/reflection-mapped-uniform-pipeline.test.mjs b/test/reflection-mapped-uniform-pipeline.test.mjs index d53e01857..85249436d 100644 --- a/test/reflection-mapped-uniform-pipeline.test.mjs +++ b/test/reflection-mapped-uniform-pipeline.test.mjs @@ -1200,3 +1200,100 @@ describe("reflection mapped rows: round-6 failure-path integrity", () => { assert.equal(stats.superseded, 1, "only the confirmed invalidation is counted"); }); }); + +// Round-7 review regressions: a partial bulkStore result must not break +// anchor sharing between deferred verdicts, and the bypass history stays +// exclusive to fail-open markers so pass audits can never evict them. +// Fixtures are entirely synthetic; no real conversation data. +describe("reflection mapped rows: round-7 partial-batch and evidence-history integrity", () => { + it("lets multiple deferred verdicts share one surviving anchor after bulkStore filters an unrelated row", async () => { + const filtered = reflectionItem("Digest emails render with the compact template from now on.", { category: "preferences" }); + const anchor = reflectionItem("Rotate the standby database credentials during the monthly window."); + const firstSupport = reflectionItem("Standby database credentials rotate in the monthly window."); + const secondSupport = reflectionItem("Credential rotation for the standby database happens monthly."); + // Geometry: the filtered row is orthogonal to the cluster; both support + // rows score the anchor as their top sibling with comfortable margins. + filtered.vector = [0, 1, 0, 0]; + anchor.vector = [1, 0, 0, 0]; + firstSupport.vector = [0.98, 0.19899749, 0, 0]; + secondSupport.vector = [0.9995, 0.0316186, 0, 0]; + + // bulkStore filters the unrelated row, shortening the result so the + // anchor resolves through the identity fallback for BOTH verdicts. + const store = makeStore({ neighbors: [] }); + const baseBulkStore = store.bulkStore.bind(store); + let firstBulk = true; + store.bulkStore = async (entries) => { + if (!firstBulk) { + return baseBulkStore(entries); + } + firstBulk = false; + store.bulkStored.push(...entries); + const kept = entries.filter((e) => !String(e.metadata).includes('"memory_category":"preferences"')); + const stored = kept.map((e, i) => ({ ...e, id: `new-${store.rows.size + i + 1}`, timestamp: 1_700_000_500_000 })); + for (const s of stored) store.rows.set(s.id, s); + return stored; + }; + const llm = makeLlm({ + onDedupBatch: () => ({ + results: [ + { index: 1, decision: "support", match_index: 1, reason: "same practice restated" }, + { index: 2, decision: "support", match_index: 1, reason: "same practice restated" }, + ], + }), + }); + const extractor = makeExtractor(store, llm); + + const { stats } = await extractor.persistGatedCandidates( + [filtered, anchor, firstSupport, secondSupport], + { targetScope: "agent:probe", scopeFilter: ["agent:probe"], sessionKey: "refl-test" }, + ); + + assert.equal(stats.supported ?? 0, 2, "both verdicts must resolve through the shared surviving anchor"); + assert.equal(stats.created, 2, "no duplicate fail-open create may land beside the anchor"); + assert.equal(store.bulkStored.length, 2, "only the two create rows are ever enqueued"); + const supportWrites = store.updates.filter((u) => u.patch && !u.patch.text); + assert.equal(supportWrites.length, 2, "both support writes must land"); + assert.ok(supportWrites.every((u) => u.id === "new-1"), "both supports bind to the same surviving anchor row"); + }); + + it("keeps the bypass history exclusive to fail-open markers and routes complete audits to admission_control_history", async () => { + const store = makeStore({ + neighbors: [auditedNeighborRow("row-1", "Keep the retro action list pinned to the team wiki page.")], + }); + const llm = makeLlm({ + onDedupBatch: () => ({ + results: [ + { index: 1, decision: "merge", match_index: 1, reason: "adds detail" }, + { index: 2, decision: "merge", match_index: 1, reason: "adds detail" }, + { index: 3, decision: "merge", match_index: 1, reason: "adds detail" }, + ], + }), + onMergeBatch: () => ({ + results: [{ index: 1, abstract: "merged abstract", overview: "o", content: "merged content" }], + }), + }); + const extractor = makeExtractor(store, llm, { admissionControl: { enabled: true } }); + + await extractor.persistGatedCandidates( + [ + reflectionItem("Pin the retro action list at the top of the team wiki."), + auditedReflectionItem("Retro action items belong on the pinned wiki list as well."), + failOpenReflectionItem("Link the retro action list from the sprint board too."), + ], + { targetScope: "agent:probe", scopeFilter: ["agent:probe"], sessionKey: "refl-test" }, + ); + + const targetWrite = store.updates.find((u) => u.id === "row-1" && u.patch?.metadata?.includes("l0_abstract")); + assert.ok(targetWrite, "the grouped merge must update its target"); + const meta = JSON.parse(targetWrite.patch.metadata); + assert.deepEqual(meta.admission_control, PRODUCTION_MAPPED_AUDIT, "the target's own complete audit survives untouched"); + assert.equal(meta.admission_bypass_events?.length ?? 0, 1, "only the fail-open marker may occupy the bypass history"); + assert.equal(meta.admission_bypass_events[0].failedOpen, true, "the bypass record is the fail-open marker"); + assert.equal(meta.admission_control_history?.length ?? 0, 1, "the additional complete audit lands in its own history"); + assert.equal(meta.admission_control_history[0].version, "amac-v1"); + assert.equal(meta.admission_control_history[0].decision, "pass_to_dedup"); + assert.equal(typeof meta.admission_control_history[0].at, "number"); + assert.equal(meta.admission_control_history[0].failedOpen, undefined, "no fail-open marker may leak into the audit history"); + }); +}); From 8276926700ba910ed7759203229f3546dd158503 Mon Sep 17 00:00:00 2001 From: Gorkem Date: Fri, 21 Aug 2026 09:11:00 +0300 Subject: [PATCH 5/7] fix: resolve deferred verdict chains to their durable anchor; let create short-circuits yield to burst siblings - Survivors that merged/supported/skipped into an earlier sibling now record a sibling anchor, and ones whose verdict targeted an existing stored row record that row; storedIdForSurviving chases the chain transitively (indices strictly decrease) so B-merges-A / C-merges-B collapses into A's row, and chains ending at a stored row resolve there instead of falling open to a duplicate create. - A create-decision dedup short-circuit (nothing stored, or the preference-slot guard) is authoritative about stored rows only: eligible burst siblings now still reach the batch judge, alone, so same-item rewordings cannot double-create behind the guard. Regressions (red-proofed): transitive merge chain, sibling verdict through an anchor merged into a stored row, and preference-guard sibling adjudication. --- dist/src/smart-extractor.js | 95 +++++++++++---- src/smart-extractor.ts | 103 ++++++++++++----- ...eflection-mapped-uniform-pipeline.test.mjs | 108 ++++++++++++++++++ 3 files changed, 255 insertions(+), 51 deletions(-) diff --git a/dist/src/smart-extractor.js b/dist/src/smart-extractor.js index f2e81d4cf..9838f9111 100644 --- a/dist/src/smart-extractor.js +++ b/dist/src/smart-extractor.js @@ -812,11 +812,14 @@ export class SmartExtractor { const siblings = burstSiblingsFor(i); try { const prefilter = await this.dedupPrefilter(candidate, vector, scopeFilter); - const emptyStoreShortCircuit = prefilter.shortCircuit?.reason === NO_SIMILAR_MEMORIES_REASON; - if (prefilter.shortCircuit && !(siblings.length > 0 && emptyStoreShortCircuit)) { - // Domain short-circuits (e.g. the preference-slot guard) stay - // authoritative even when burst siblings exist; only the plain - // "nothing similar stored yet" bypass yields to sibling context. + const yieldsToSiblings = prefilter.shortCircuit?.decision === "create" && siblings.length > 0; + if (prefilter.shortCircuit && !yieldsToSiblings) { + // A create-decision short-circuit is authoritative about STORED + // rows only (nothing similar stored yet, or the preference guard + // ruled every stored row a different item slot) — it says nothing + // about burst siblings, so eligible siblings still get + // adjudicated, alone: the short circuit carries no stored rows. + // Non-create short-circuits resolve the candidate and stand. precomputedDedups.set(i, prefilter.shortCircuit); } else { @@ -841,6 +844,12 @@ export class SmartExtractor { const pendingMerges = []; const pendingSiblingVerdicts = []; const createSlotBySurviving = new Map(); + // Durable-anchor bookkeeping for deferred verdict chains: a survivor + // that itself merged/supported/skipped into an earlier sibling anchors + // at that sibling (chased transitively at resolution time), and one + // whose verdict targeted an existing stored row anchors at that row. + const anchorSiblingBySurviving = new Map(); + const anchorStoredIdBySurviving = new Map(); for (let i = 0; i < surviving.length; i++) { const { candidate, vector } = surviving[i]; const pre = precomputedDedups.get(i); @@ -848,11 +857,13 @@ export class SmartExtractor { const siblingIndex = Number(pre.matchId.slice(BURST_SIBLING_PREFIX.length)); const resolvable = Number.isInteger(siblingIndex) && siblingIndex >= 0 && siblingIndex < i; if (pre.decision === "skip" && resolvable) { + anchorSiblingBySurviving.set(i, siblingIndex); stats.skipped++; this.log(`memory-pro: smart-extractor: gated candidate judged same-burst duplicate of an earlier sibling [${candidate.category}]`); continue; } if ((pre.decision === "merge" || pre.decision === "support") && resolvable) { + anchorSiblingBySurviving.set(i, siblingIndex); pendingSiblingVerdicts.push({ candidate, vector, @@ -870,6 +881,11 @@ export class SmartExtractor { reason: "sibling verdict fallback (unsupported decision for a pending row)", }); } + if (pre?.matchId && + !pre.matchId.startsWith(BURST_SIBLING_PREFIX) && + (pre.decision === "merge" || pre.decision === "support")) { + anchorStoredIdBySurviving.set(i, pre.matchId); + } const createCountBefore = createEntries.length; try { await this.processCandidate(candidate, conversationText, sessionKey, stats, targetScope, scopeFilter, vector, createEntries, pendingSupersedeInvalidations, options.agentId, preGatedFor(candidate), precomputedDedups.get(i), pendingMerges); @@ -909,35 +925,66 @@ export class SmartExtractor { if (pendingSiblingVerdicts.length > 0) { const claimedIds = new Set(); const resolvedIdBySurviving = new Map(); + const resolveSlotId = (slot) => { + if (createdEntries.length === createEntries.length) { + return createdEntries[slot]?.id; + } + // bulkStore may filter entries, shifting positions: fall back to the + // first unclaimed row with the same text AND the same lane/category + // identity — identical text is legal across lanes, so a text-only + // match could bind the verdict to another lane's row. + const want = createEntries[slot]; + const hit = want + ? createdEntries.find((e) => e.text === want.text && + e.category === want.category && + laneFromMetadata(e.metadata) === laneFromMetadata(want.metadata) && + !claimedIds.has(e.id)) + : undefined; + return hit?.id; + }; const storedIdForSurviving = (survivingIndex) => { // Verdicts may share one surviving anchor: the first resolution is // cached per index so every later verdict reuses the same row. // claimedIds only keeps DISTINCT surviving entries apart in the // filtered-result fallback — it must never exclude the row an index - // already resolved. + // already resolved. A survivor without its own create slot resolves + // through its durable anchor: sibling chains are chased transitively + // (indices strictly decrease, so chains terminate) and may end at a + // created row or at an existing stored row. if (resolvedIdBySurviving.has(survivingIndex)) { return resolvedIdBySurviving.get(survivingIndex); } const resolve = () => { - const slot = createSlotBySurviving.get(survivingIndex); - if (slot === undefined) { - return undefined; - } - if (createdEntries.length === createEntries.length) { - return createdEntries[slot]?.id; + let index = survivingIndex; + for (let hops = 0; hops <= surviving.length; hops++) { + if (index !== survivingIndex && resolvedIdBySurviving.has(index)) { + return resolvedIdBySurviving.get(index); + } + const slot = createSlotBySurviving.get(index); + if (slot !== undefined) { + const id = resolveSlotId(slot); + if (index !== survivingIndex) { + // A chain terminal is a shared anchor: cache and claim it + // under its own index so other chains landing here reuse + // the row instead of consuming another twin. + if (id) { + claimedIds.add(id); + } + resolvedIdBySurviving.set(index, id); + } + return id; + } + const storedId = anchorStoredIdBySurviving.get(index); + if (storedId) { + return storedId; + } + const next = anchorSiblingBySurviving.get(index); + if (next === undefined) { + return undefined; + } + index = next; } - // bulkStore may filter entries, shifting positions: fall back to the - // first unclaimed row with the same text AND the same lane/category - // identity — identical text is legal across lanes, so a text-only - // match could bind the verdict to another lane's row. - const want = createEntries[slot]; - const hit = want - ? createdEntries.find((e) => e.text === want.text && - e.category === want.category && - laneFromMetadata(e.metadata) === laneFromMetadata(want.metadata) && - !claimedIds.has(e.id)) - : undefined; - return hit?.id; + return undefined; }; const id = resolve(); if (id) { diff --git a/src/smart-extractor.ts b/src/smart-extractor.ts index a60fb4cde..36ebc1b7f 100644 --- a/src/smart-extractor.ts +++ b/src/smart-extractor.ts @@ -1169,11 +1169,14 @@ export class SmartExtractor { const siblings = burstSiblingsFor(i); try { const prefilter = await this.dedupPrefilter(candidate, vector, scopeFilter); - const emptyStoreShortCircuit = prefilter.shortCircuit?.reason === NO_SIMILAR_MEMORIES_REASON; - if (prefilter.shortCircuit && !(siblings.length > 0 && emptyStoreShortCircuit)) { - // Domain short-circuits (e.g. the preference-slot guard) stay - // authoritative even when burst siblings exist; only the plain - // "nothing similar stored yet" bypass yields to sibling context. + const yieldsToSiblings = prefilter.shortCircuit?.decision === "create" && siblings.length > 0; + if (prefilter.shortCircuit && !yieldsToSiblings) { + // A create-decision short-circuit is authoritative about STORED + // rows only (nothing similar stored yet, or the preference guard + // ruled every stored row a different item slot) — it says nothing + // about burst siblings, so eligible siblings still get + // adjudicated, alone: the short circuit carries no stored rows. + // Non-create short-circuits resolve the candidate and stand. precomputedDedups.set(i, prefilter.shortCircuit); } else { const topSimilar = [...prefilter.topSimilar, ...siblings] @@ -1206,6 +1209,12 @@ export class SmartExtractor { contextLabel?: string; }> = []; const createSlotBySurviving = new Map(); + // Durable-anchor bookkeeping for deferred verdict chains: a survivor + // that itself merged/supported/skipped into an earlier sibling anchors + // at that sibling (chased transitively at resolution time), and one + // whose verdict targeted an existing stored row anchors at that row. + const anchorSiblingBySurviving = new Map(); + const anchorStoredIdBySurviving = new Map(); for (let i = 0; i < surviving.length; i++) { const { candidate, vector } = surviving[i]; @@ -1214,6 +1223,7 @@ export class SmartExtractor { const siblingIndex = Number(pre.matchId.slice(BURST_SIBLING_PREFIX.length)); const resolvable = Number.isInteger(siblingIndex) && siblingIndex >= 0 && siblingIndex < i; if (pre.decision === "skip" && resolvable) { + anchorSiblingBySurviving.set(i, siblingIndex); stats.skipped++; this.log( `memory-pro: smart-extractor: gated candidate judged same-burst duplicate of an earlier sibling [${candidate.category}]`, @@ -1221,6 +1231,7 @@ export class SmartExtractor { continue; } if ((pre.decision === "merge" || pre.decision === "support") && resolvable) { + anchorSiblingBySurviving.set(i, siblingIndex); pendingSiblingVerdicts.push({ candidate, vector, @@ -1238,6 +1249,13 @@ export class SmartExtractor { reason: "sibling verdict fallback (unsupported decision for a pending row)", }); } + if ( + pre?.matchId && + !pre.matchId.startsWith(BURST_SIBLING_PREFIX) && + (pre.decision === "merge" || pre.decision === "support") + ) { + anchorStoredIdBySurviving.set(i, pre.matchId); + } const createCountBefore = createEntries.length; try { await this.processCandidate( @@ -1298,38 +1316,69 @@ export class SmartExtractor { if (pendingSiblingVerdicts.length > 0) { const claimedIds = new Set(); const resolvedIdBySurviving = new Map(); + const resolveSlotId = (slot: number): string | undefined => { + if (createdEntries.length === createEntries.length) { + return createdEntries[slot]?.id; + } + // bulkStore may filter entries, shifting positions: fall back to the + // first unclaimed row with the same text AND the same lane/category + // identity — identical text is legal across lanes, so a text-only + // match could bind the verdict to another lane's row. + const want = createEntries[slot]; + const hit = want + ? createdEntries.find( + (e) => + e.text === want.text && + e.category === want.category && + laneFromMetadata(e.metadata) === laneFromMetadata(want.metadata) && + !claimedIds.has(e.id), + ) + : undefined; + return hit?.id; + }; const storedIdForSurviving = (survivingIndex: number): string | undefined => { // Verdicts may share one surviving anchor: the first resolution is // cached per index so every later verdict reuses the same row. // claimedIds only keeps DISTINCT surviving entries apart in the // filtered-result fallback — it must never exclude the row an index - // already resolved. + // already resolved. A survivor without its own create slot resolves + // through its durable anchor: sibling chains are chased transitively + // (indices strictly decrease, so chains terminate) and may end at a + // created row or at an existing stored row. if (resolvedIdBySurviving.has(survivingIndex)) { return resolvedIdBySurviving.get(survivingIndex); } const resolve = (): string | undefined => { - const slot = createSlotBySurviving.get(survivingIndex); - if (slot === undefined) { - return undefined; - } - if (createdEntries.length === createEntries.length) { - return createdEntries[slot]?.id; + let index = survivingIndex; + for (let hops = 0; hops <= surviving.length; hops++) { + if (index !== survivingIndex && resolvedIdBySurviving.has(index)) { + return resolvedIdBySurviving.get(index); + } + const slot = createSlotBySurviving.get(index); + if (slot !== undefined) { + const id = resolveSlotId(slot); + if (index !== survivingIndex) { + // A chain terminal is a shared anchor: cache and claim it + // under its own index so other chains landing here reuse + // the row instead of consuming another twin. + if (id) { + claimedIds.add(id); + } + resolvedIdBySurviving.set(index, id); + } + return id; + } + const storedId = anchorStoredIdBySurviving.get(index); + if (storedId) { + return storedId; + } + const next = anchorSiblingBySurviving.get(index); + if (next === undefined) { + return undefined; + } + index = next; } - // bulkStore may filter entries, shifting positions: fall back to the - // first unclaimed row with the same text AND the same lane/category - // identity — identical text is legal across lanes, so a text-only - // match could bind the verdict to another lane's row. - const want = createEntries[slot]; - const hit = want - ? createdEntries.find( - (e) => - e.text === want.text && - e.category === want.category && - laneFromMetadata(e.metadata) === laneFromMetadata(want.metadata) && - !claimedIds.has(e.id), - ) - : undefined; - return hit?.id; + return undefined; }; const id = resolve(); if (id) { diff --git a/test/reflection-mapped-uniform-pipeline.test.mjs b/test/reflection-mapped-uniform-pipeline.test.mjs index 85249436d..58ae70803 100644 --- a/test/reflection-mapped-uniform-pipeline.test.mjs +++ b/test/reflection-mapped-uniform-pipeline.test.mjs @@ -1297,3 +1297,111 @@ describe("reflection mapped rows: round-7 partial-batch and evidence-history int assert.equal(meta.admission_control_history[0].failedOpen, undefined, "no fail-open marker may leak into the audit history"); }); }); + +// Round-8 review regressions: deferred verdict chains resolve transitively +// to their durable anchor (created row or existing stored row), and a +// create-decision short circuit no longer suppresses burst-sibling +// adjudication. Fixtures are entirely synthetic; no real conversation data. +describe("reflection mapped rows: round-8 transitive anchors and short-circuit sibling adjudication", () => { + it("resolves a merge chain transitively: B merges into A, C merges into B, one row persists", async () => { + const anchor = reflectionItem("Archive the load-test artifacts to cold storage every Friday."); + const restated = reflectionItem("Load-test artifacts move to cold storage on Fridays."); + const restatedAgain = reflectionItem("Every Friday the load-test artifacts go into cold storage."); + anchor.vector = [1, 0, 0, 0]; + restated.vector = [0.98, 0.19899749, 0, 0]; + restatedAgain.vector = [0.97, 0.24310491, 0, 0]; + + const store = makeStore({ neighbors: [] }); + const llm = makeLlm({ + onDedupBatch: () => ({ + results: [ + { index: 1, decision: "merge", match_index: 1, reason: "same practice restated" }, + { index: 2, decision: "merge", match_index: 1, reason: "restates the restatement" }, + ], + }), + onMergeBatch: () => ({ + results: [{ index: 1, abstract: "merged cold-storage practice", overview: "o", content: "merged cold-storage practice content" }], + }), + }); + const extractor = makeExtractor(store, llm); + + const { stats } = await extractor.persistGatedCandidates( + [anchor, restated, restatedAgain], + { targetScope: "agent:probe", scopeFilter: ["agent:probe"], sessionKey: "refl-test" }, + ); + + assert.equal(stats.created, 1, "only the chain anchor may persist as a create"); + assert.equal(stats.merged, 2, "both chained merge verdicts must resolve"); + assert.equal(store.bulkStored.length, 1, "no fail-open duplicate may land beside the anchor"); + const contentUpdate = store.updates.find((u) => u.id === "new-1" && u.patch?.metadata?.includes("l0_abstract")); + assert.ok(contentUpdate, "the grouped merge lands on the anchor row"); + }); + + it("resolves a sibling verdict through an anchor that itself merged into an existing stored row", async () => { + const store = makeStore({ + neighbors: [neighborRow("row-1", "Rotate the audit log encryption key at the end of each quarter.")], + }); + const mergingAnchor = reflectionItem("The audit log encryption key rotates at quarter end."); + const supporter = reflectionItem("Quarter end is when the audit log encryption key gets rotated."); + supporter.vector = [...mergingAnchor.vector]; + + const llm = makeLlm({ + onDedupBatch: () => ({ + results: [ + { index: 1, decision: "merge", match_index: 1, reason: "adds nothing new beyond the stored row" }, + { index: 2, decision: "support", match_index: 1, reason: "same practice restated" }, + ], + }), + onMergeBatch: () => ({ + results: [{ index: 1, abstract: "merged rotation practice", overview: "o", content: "merged rotation practice content" }], + }), + }); + const extractor = makeExtractor(store, llm); + + const { stats } = await extractor.persistGatedCandidates( + [mergingAnchor, supporter], + { targetScope: "agent:probe", scopeFilter: ["agent:probe"], sessionKey: "refl-test" }, + ); + + assert.equal(store.bulkStored.length, 0, "neither candidate may create a row"); + assert.equal(stats.created ?? 0, 0, "the supporter must not fall open to a duplicate create"); + assert.equal(stats.merged, 1, "the anchor's merge into the stored row resolves"); + assert.equal(stats.supported ?? 0, 1, "the deferred support resolves through the anchor's stored target"); + const supportWrite = store.updates.find((u) => u.id === "row-1" && u.patch && !u.patch.text); + assert.ok(supportWrite, "the support evidence lands on the stored row the anchor merged into"); + }); + + it("adjudicates burst siblings when the preference-slot guard excludes every stored neighbor", async () => { + const storedPreference = { + ...neighborRow("row-1", "I really like the curly fries from Crown Grill."), + category: "preferences", + metadata: JSON.stringify({ + memory_category: "preferences", + l0_abstract: "I really like the curly fries from Crown Grill.", + l1_overview: "## Existing\nI really like the curly fries from Crown Grill.", + l2_content: "I really like the curly fries from Crown Grill.", + }), + }; + const store = makeStore({ neighbors: [storedPreference] }); + const firstWording = reflectionItem("I really like the smash burger from Crown Grill.", { category: "preferences" }); + const secondWording = reflectionItem("I still love the smash burger from Crown Grill.", { category: "preferences" }); + secondWording.vector = [...firstWording.vector]; + + const llm = makeLlm({ + onDedupBatch: () => ({ + results: [{ index: 1, decision: "skip", match_index: 1, reason: "same incoming preference reworded" }], + }), + }); + const extractor = makeExtractor(store, llm); + + const { stats } = await extractor.persistGatedCandidates( + [firstWording, secondWording], + { targetScope: "agent:probe", scopeFilter: ["agent:probe"], sessionKey: "refl-test" }, + ); + + assert.equal(llm.calls.filter((c) => c === "dedup-decision-batch").length, 1, "eligible burst siblings must reach the judge despite the guard"); + assert.equal(stats.created, 1, "only the first wording persists"); + assert.equal(stats.skipped ?? 0, 1, "the reworded twin collapses through sibling adjudication"); + assert.equal(store.bulkStored.length, 1, "the guard must not double-create same-item siblings"); + }); +}); From d7a10c94e17d23bfd959adeaff867d1411e4cb3f Mon Sep 17 00:00:00 2001 From: Gorkem Date: Sun, 23 Aug 2026 14:06:25 +0300 Subject: [PATCH 6/7] fix: carry the skip verdict's target so skipped survivors keep their durable anchor interpretDedupVerdict now includes skip in the matchId decisions: a same-burst SKIP records its sibling anchor (and a stored-row SKIP its row anchor), so later verdicts chaining through the skipped survivor resolve to the durable row instead of failing open to a duplicate create. The skip handler itself still ignores the target. Regressions (red-proofed): B-SKIP-A with C-MERGE-B, and the analogous SUPPORT-through-skipped-anchor case. --- dist/src/smart-extractor.js | 7 +- src/smart-extractor.ts | 7 +- ...eflection-mapped-uniform-pipeline.test.mjs | 68 +++++++++++++++++++ 3 files changed, 78 insertions(+), 4 deletions(-) diff --git a/dist/src/smart-extractor.js b/dist/src/smart-extractor.js index 9838f9111..b55a6d5f7 100644 --- a/dist/src/smart-extractor.js +++ b/dist/src/smart-extractor.js @@ -883,7 +883,7 @@ export class SmartExtractor { } if (pre?.matchId && !pre.matchId.startsWith(BURST_SIBLING_PREFIX) && - (pre.decision === "merge" || pre.decision === "support")) { + (pre.decision === "merge" || pre.decision === "support" || pre.decision === "skip")) { anchorStoredIdBySurviving.set(i, pre.matchId); } const createCountBefore = createEntries.length; @@ -1996,7 +1996,10 @@ export class SmartExtractor { return { decision, reason: data.reason ?? "", - matchId: ["merge", "support", "contextualize", "contradict", "supersede"].includes(decision) ? matchEntry?.entry.id : undefined, + // skip carries its target too: a same-burst SKIP anchors the skipped + // survivor at its duplicate, and later sibling verdicts must be able + // to resolve through that anchor (the skip handler itself ignores it). + matchId: ["merge", "support", "contextualize", "contradict", "supersede", "skip"].includes(decision) ? matchEntry?.entry.id : undefined, contextLabel: typeof data.context_label === "string" ? data.context_label : undefined, }; } diff --git a/src/smart-extractor.ts b/src/smart-extractor.ts index 36ebc1b7f..82f5076fe 100644 --- a/src/smart-extractor.ts +++ b/src/smart-extractor.ts @@ -1252,7 +1252,7 @@ export class SmartExtractor { if ( pre?.matchId && !pre.matchId.startsWith(BURST_SIBLING_PREFIX) && - (pre.decision === "merge" || pre.decision === "support") + (pre.decision === "merge" || pre.decision === "support" || pre.decision === "skip") ) { anchorStoredIdBySurviving.set(i, pre.matchId); } @@ -2720,7 +2720,10 @@ export class SmartExtractor { return { decision, reason: data.reason ?? "", - matchId: ["merge", "support", "contextualize", "contradict", "supersede"].includes(decision) ? matchEntry?.entry.id : undefined, + // skip carries its target too: a same-burst SKIP anchors the skipped + // survivor at its duplicate, and later sibling verdicts must be able + // to resolve through that anchor (the skip handler itself ignores it). + matchId: ["merge", "support", "contextualize", "contradict", "supersede", "skip"].includes(decision) ? matchEntry?.entry.id : undefined, contextLabel: typeof data.context_label === "string" ? data.context_label : undefined, }; } diff --git a/test/reflection-mapped-uniform-pipeline.test.mjs b/test/reflection-mapped-uniform-pipeline.test.mjs index 58ae70803..6d00c3943 100644 --- a/test/reflection-mapped-uniform-pipeline.test.mjs +++ b/test/reflection-mapped-uniform-pipeline.test.mjs @@ -1405,3 +1405,71 @@ describe("reflection mapped rows: round-8 transitive anchors and short-circuit s assert.equal(store.bulkStored.length, 1, "the guard must not double-create same-item siblings"); }); }); + +// Round-9 review regression: a same-burst SKIP must preserve its anchor so +// later sibling verdicts resolve through the skipped survivor. +// Fixtures are entirely synthetic; no real conversation data. +describe("reflection mapped rows: round-9 skip-anchored sibling chains", () => { + function skipChainItems() { + const anchor = reflectionItem("Publish the incident retro notes within two days of closure."); + const skippedTwin = reflectionItem("Incident retro notes go out within two days of closing."); + const chained = reflectionItem("Within two days of an incident closing, the retro notes get published."); + anchor.vector = [1, 0, 0, 0]; + skippedTwin.vector = [0.98, 0.19899749, 0, 0]; + chained.vector = [0.97, 0.24310491, 0, 0]; + return [anchor, skippedTwin, chained]; + } + + it("resolves a MERGE through a skipped sibling anchor: B skips against A, C merges into B", async () => { + const store = makeStore({ neighbors: [] }); + const llm = makeLlm({ + onDedupBatch: () => ({ + results: [ + { index: 1, decision: "skip", match_index: 1, reason: "same practice reworded" }, + { index: 2, decision: "merge", match_index: 1, reason: "adds phrasing detail" }, + ], + }), + onMergeBatch: () => ({ + results: [{ index: 1, abstract: "merged retro practice", overview: "o", content: "merged retro practice content" }], + }), + }); + const extractor = makeExtractor(store, llm); + + const { stats } = await extractor.persistGatedCandidates( + skipChainItems(), + { targetScope: "agent:probe", scopeFilter: ["agent:probe"], sessionKey: "refl-test" }, + ); + + assert.equal(stats.created, 1, "only the anchor persists"); + assert.equal(stats.skipped ?? 0, 1, "the twin collapses as a skip"); + assert.equal(stats.merged, 1, "the chained merge resolves through the skipped survivor to the anchor"); + assert.equal(store.bulkStored.length, 1, "no fail-open duplicate lands"); + const contentUpdate = store.updates.find((u) => u.id === "new-1" && u.patch?.metadata?.includes("l0_abstract")); + assert.ok(contentUpdate, "the merge lands on the anchor row"); + }); + + it("resolves a SUPPORT through a skipped sibling anchor: B skips against A, C supports B", async () => { + const store = makeStore({ neighbors: [] }); + const llm = makeLlm({ + onDedupBatch: () => ({ + results: [ + { index: 1, decision: "skip", match_index: 1, reason: "same practice reworded" }, + { index: 2, decision: "support", match_index: 1, reason: "restates the practice" }, + ], + }), + }); + const extractor = makeExtractor(store, llm); + + const { stats } = await extractor.persistGatedCandidates( + skipChainItems(), + { targetScope: "agent:probe", scopeFilter: ["agent:probe"], sessionKey: "refl-test" }, + ); + + assert.equal(stats.created, 1, "only the anchor persists"); + assert.equal(stats.skipped ?? 0, 1, "the twin collapses as a skip"); + assert.equal(stats.supported ?? 0, 1, "the support resolves through the skipped survivor to the anchor"); + assert.equal(store.bulkStored.length, 1, "no fail-open duplicate lands"); + const supportWrite = store.updates.find((u) => u.id === "new-1" && u.patch && !u.patch.text); + assert.ok(supportWrite, "the support evidence lands on the anchor row"); + }); +}); From a0d556ba5e4975c1ac54ebbc38c56c13041c1b84 Mon Sep 17 00:00:00 2001 From: Gorkem Date: Sun, 23 Aug 2026 14:15:07 +0300 Subject: [PATCH 7/7] chore: retrigger CI (storage-and-schema cancelled-parent flake on the LanceDB backend)