From 5fe40755d21edd16c85bbd541c401a73dbf82b71 Mon Sep 17 00:00:00 2001 From: Gorkem Date: Tue, 4 Aug 2026 18:48:30 +0300 Subject: [PATCH] fold: noise-bank validation gate family onto current tips (PR 959 tip, adapted to master's ok-result shape) --- dist/src/smart-extractor.js | 16 ++++- src/smart-extractor.ts | 23 +++++-- test/smart-extractor-noise-gating.test.mjs | 70 +++++++++++++++++++--- 3 files changed, 96 insertions(+), 13 deletions(-) diff --git a/dist/src/smart-extractor.js b/dist/src/smart-extractor.js index b55a6d5f..b442bbb3 100644 --- a/dist/src/smart-extractor.js +++ b/dist/src/smart-extractor.js @@ -462,12 +462,17 @@ export class SmartExtractor { // No LLM call was made, so the caller's rate limiter must not be charged. stats.skippedNoInput = true; } - if (extraction.status === "ok" && !extraction.groundingOrPolicyDropped) { + if (extraction.status === "ok" && extraction.rawCandidateCount === 0) { // LLM genuinely returned zero candidates → strongest noise signal → feedback to noise bank this.learnAsNoise(conversationText); } else if (extraction.status === "ok") { - this.debugLog("memory-pro: smart-extractor: skipping noise-bank learning (batch emptied by grounding/register/policy drops, not a genuine zero-extraction)"); + // The model DID emit candidates; validation, grounding, or the batch + // contradiction check dropped them all. That is a policy verdict about + // those candidates, not evidence the conversation is noise — learning + // it as noise would pre-filter similar real content away from future + // extractions. + this.log(`memory-pro: smart-extractor: skipping noise-bank learning (validation emptied the batch, raw=${extraction.rawCandidateCount})`); } else { this.debugLog(`memory-pro: smart-extractor: skipping noise-bank learning (status=${extraction.status})`); @@ -1643,10 +1648,17 @@ export class SmartExtractor { } } } + if (contradictionDemotedCount > 0) { + // At the standard log level so a fully-demoted batch is distinguishable + // from "model found nothing" without debug logging (mirrors the + // admission-rejection lines). + this.log(`memory-lancedb-pro: smart-extractor: batch contradiction demoted ${contradictionDemotedCount} real-tagged durable candidate(s) (register=${conversationRegister}, constructed siblings present)`); + } this.debugLog(`memory-lancedb-pro: smart-extractor: validation summary register=${conversationRegister}, accepted=${candidates.length}, invalidCategory=${invalidCategoryCount}, shortAbstract=${shortAbstractCount}, noiseAbstract=${noiseAbstractCount}, policyDropped=${policyDroppedCount}, constructedDropped=${constructedDroppedCount}, fictionRegisterDropped=${fictionRegisterDroppedCount}, contradictionDemoted=${contradictionDemotedCount}`); return { status: "ok", candidates, + rawCandidateCount: result.memories.length, // A batch emptied by grounding, register, or policy drops is NOT a // "the LLM found nothing here" signal — the LLM found plenty and the // filters excluded it — so the caller must not train the noise bank diff --git a/src/smart-extractor.ts b/src/smart-extractor.ts index 82f5076f..27c5c204 100644 --- a/src/smart-extractor.ts +++ b/src/smart-extractor.ts @@ -177,7 +177,7 @@ type PendingSupersedeInvalidation = { // tell a genuine "LLM found nothing" verdict from a gateway/model failure or // a malformed response shape — only the former is a real noise signal. type ExtractCandidatesResult = - | { status: "ok"; candidates: CandidateMemory[]; groundingOrPolicyDropped?: boolean } + | { status: "ok"; candidates: CandidateMemory[]; rawCandidateCount: number; groundingOrPolicyDropped?: boolean } | { status: "llm_failure"; candidates: [] } | { status: "malformed"; candidates: [] } | { status: "empty_input"; candidates: [] }; @@ -734,12 +734,17 @@ export class SmartExtractor { // No LLM call was made, so the caller's rate limiter must not be charged. stats.skippedNoInput = true; } - if (extraction.status === "ok" && !extraction.groundingOrPolicyDropped) { + if (extraction.status === "ok" && extraction.rawCandidateCount === 0) { // LLM genuinely returned zero candidates → strongest noise signal → feedback to noise bank this.learnAsNoise(conversationText); } else if (extraction.status === "ok") { - this.debugLog( - "memory-pro: smart-extractor: skipping noise-bank learning (batch emptied by grounding/register/policy drops, not a genuine zero-extraction)", + // The model DID emit candidates; validation, grounding, or the batch + // contradiction check dropped them all. That is a policy verdict about + // those candidates, not evidence the conversation is noise — learning + // it as noise would pre-filter similar real content away from future + // extractions. + this.log( + `memory-pro: smart-extractor: skipping noise-bank learning (validation emptied the batch, raw=${extraction.rawCandidateCount})`, ); } else { this.debugLog( @@ -2201,6 +2206,15 @@ export class SmartExtractor { } } + if (contradictionDemotedCount > 0) { + // At the standard log level so a fully-demoted batch is distinguishable + // from "model found nothing" without debug logging (mirrors the + // admission-rejection lines). + this.log( + `memory-lancedb-pro: smart-extractor: batch contradiction demoted ${contradictionDemotedCount} real-tagged durable candidate(s) (register=${conversationRegister}, constructed siblings present)`, + ); + } + this.debugLog( `memory-lancedb-pro: smart-extractor: validation summary register=${conversationRegister}, accepted=${candidates.length}, invalidCategory=${invalidCategoryCount}, shortAbstract=${shortAbstractCount}, noiseAbstract=${noiseAbstractCount}, policyDropped=${policyDroppedCount}, constructedDropped=${constructedDroppedCount}, fictionRegisterDropped=${fictionRegisterDroppedCount}, contradictionDemoted=${contradictionDemotedCount}`, ); @@ -2208,6 +2222,7 @@ export class SmartExtractor { return { status: "ok", candidates, + rawCandidateCount: result.memories.length, // A batch emptied by grounding, register, or policy drops is NOT a // "the LLM found nothing here" signal — the LLM found plenty and the // filters excluded it — so the caller must not train the noise bank diff --git a/test/smart-extractor-noise-gating.test.mjs b/test/smart-extractor-noise-gating.test.mjs index 8ebd290e..e18f0c79 100644 --- a/test/smart-extractor-noise-gating.test.mjs +++ b/test/smart-extractor-noise-gating.test.mjs @@ -4,10 +4,12 @@ * extractCandidates() can come back with zero candidates for four different * reasons: the LLM/gateway call failed outright, the response had an * unexpected shape, the LLM genuinely returned an empty memories list, or - * every parsed candidate was dropped by local validation. Only the latter - * two are a real "nothing to remember" signal — the first two are failures - * that must NOT be fed into the noise-prototype bank, or gateway outages - * silently poison future extraction/recall. + * every parsed candidate was dropped/demoted by local validation. ONLY the + * genuinely-empty list is a real "nothing to remember" signal. Failures must + * not train the bank (gateway outages would poison it), and a + * validation-emptied batch must not either: the model DID find candidates + * there, so teaching the bank "this conversation is noise" would pre-filter + * similar real content away from future extractions. */ import { describe, it } from "node:test"; @@ -42,6 +44,7 @@ function makeLlm(behavior) { if (behavior === "malformed_non_array") return { memories: "not-an-array" }; if (behavior === "empty") return { memories: [] }; if (Array.isArray(behavior)) return { memories: behavior }; + if (behavior && typeof behavior === "object" && behavior.raw) return behavior.raw; throw new Error(`unsupported test behavior: ${behavior}`); }, }; @@ -130,16 +133,69 @@ describe("SmartExtractor noise-bank learning gate", () => { assert.equal(noiseBank.learnCalls.length, 1, "a genuine empty list is a real noise signal"); }); - it("learns noise when every parsed candidate is dropped by local validation", async () => { + it("does NOT learn noise when validation drops every parsed candidate (policy verdict, not a noise signal)", async () => { const noiseBank = makeNoiseBank(); + const logs = []; // "hi" is a valid category but fails the length>=5 abstract check, so the - // parsed response is well-formed yet every candidate is filtered out. + // parsed response is well-formed yet every candidate is filtered out. The + // model DID emit a candidate, so the conversation must not train the bank. const llm = makeLlm([{ category: "preferences", abstract: "hi", overview: "", content: "" }]); + const extractor = makeExtractor(makeEmbedder(), llm, makeStore(), { noiseBank, log: (msg) => logs.push(msg) }); + + await extractor.extractAndPersist("some conversation text", "s1"); + await flushMicrotasks(); + + assert.equal(noiseBank.learnCalls.length, 0, "a validation-emptied batch must not train the noise bank"); + assert.ok( + logs.some((msg) => msg.includes("skipping noise-bank learning (validation emptied the batch")), + "the skip must be visible at the standard log level", + ); + }); + + it("does NOT learn noise when the batch contradiction check demotes every candidate", async () => { + const noiseBank = makeNoiseBank(); + // Mixed-register batch with a constructed sibling: the real-tagged durable + // is demoted by the batch contradiction check, the constructed one is + // dropped by grounding enforcement. Raw output had 2 candidates, so the + // conversation (which contains real facts the policy demoted) must not be + // fed to the noise bank as a noise exemplar. + const llm = makeLlm({ + raw: { + conversation_register: "mixed", + memories: [ + { category: "preferences", abstract: "User loves space operas", overview: "- pref", content: "", grounding: "real" }, + { category: "events", abstract: "The captain hid the artifact", overview: "- event", content: "", grounding: "constructed" }, + ], + }, + }); const extractor = makeExtractor(makeEmbedder(), llm, makeStore(), { noiseBank }); await extractor.extractAndPersist("some conversation text", "s1"); await flushMicrotasks(); - assert.equal(noiseBank.learnCalls.length, 1, "a validly-parsed-but-empty-after-filtering result is still a real noise signal"); + assert.equal(noiseBank.learnCalls.length, 0, "a demotion-emptied batch must not train the noise bank"); + }); + + it("logs the batch contradiction demotion at the standard log level, with count and register", async () => { + const noiseBank = makeNoiseBank(); + const logs = []; + const llm = makeLlm({ + raw: { + conversation_register: "mixed", + memories: [ + { category: "preferences", abstract: "User loves space operas", overview: "- pref", content: "", grounding: "real" }, + { category: "events", abstract: "The captain hid the artifact", overview: "- event", content: "", grounding: "constructed" }, + ], + }, + }); + const extractor = makeExtractor(makeEmbedder(), llm, makeStore(), { noiseBank, log: (msg) => logs.push(msg) }); + + await extractor.extractAndPersist("some conversation text", "s1"); + await flushMicrotasks(); + + const demotionLine = logs.find((msg) => msg.includes("batch contradiction demoted")); + assert.ok(demotionLine, "a fully-demoted batch must be distinguishable from 'model found nothing' without debug logging"); + assert.ok(demotionLine.includes("1"), "the demotion line must carry the demoted count"); + assert.ok(demotionLine.includes("mixed"), "the demotion line must carry the batch register"); }); });