fix(extraction): keep validation-emptied batches out of the noise bank

dist/src/smart-extractor.js

@@ -462,12 +462,17 @@ export class SmartExtractor {
                 // No LLM call was made, so the caller's rate limiter must not be charged.
                 stats.skippedNoInput = true;
             }
-            if (extraction.status === "ok" && !extraction.groundingOrPolicyDropped) {
+            if (extraction.status === "ok" && extraction.rawCandidateCount === 0) {
                 // LLM genuinely returned zero candidates → strongest noise signal → feedback to noise bank
                 this.learnAsNoise(conversationText);
             }
             else if (extraction.status === "ok") {
-                this.debugLog("memory-pro: smart-extractor: skipping noise-bank learning (batch emptied by grounding/register/policy drops, not a genuine zero-extraction)");
+                // The model DID emit candidates; validation, grounding, or the batch
+                // contradiction check dropped them all. That is a policy verdict about
+                // those candidates, not evidence the conversation is noise — learning
+                // it as noise would pre-filter similar real content away from future
+                // extractions.
+                this.log(`memory-pro: smart-extractor: skipping noise-bank learning (validation emptied the batch, raw=${extraction.rawCandidateCount})`);
             }
             else {
                 this.debugLog(`memory-pro: smart-extractor: skipping noise-bank learning (status=${extraction.status})`);
@@ -1643,10 +1648,17 @@ export class SmartExtractor {
                 }
             }
         }
+        if (contradictionDemotedCount > 0) {
+            // At the standard log level so a fully-demoted batch is distinguishable
+            // from "model found nothing" without debug logging (mirrors the
+            // admission-rejection lines).
+            this.log(`memory-lancedb-pro: smart-extractor: batch contradiction demoted ${contradictionDemotedCount} real-tagged durable candidate(s) (register=${conversationRegister}, constructed siblings present)`);
+        }
         this.debugLog(`memory-lancedb-pro: smart-extractor: validation summary register=${conversationRegister}, accepted=${candidates.length}, invalidCategory=${invalidCategoryCount}, shortAbstract=${shortAbstractCount}, noiseAbstract=${noiseAbstractCount}, policyDropped=${policyDroppedCount}, constructedDropped=${constructedDroppedCount}, fictionRegisterDropped=${fictionRegisterDroppedCount}, contradictionDemoted=${contradictionDemotedCount}`);
         return {
             status: "ok",
             candidates,
+            rawCandidateCount: result.memories.length,
             // A batch emptied by grounding, register, or policy drops is NOT a
             // "the LLM found nothing here" signal — the LLM found plenty and the
             // filters excluded it — so the caller must not train the noise bank

src/smart-extractor.ts

@@ -177,7 +177,7 @@ type PendingSupersedeInvalidation = {
 // tell a genuine "LLM found nothing" verdict from a gateway/model failure or
 // a malformed response shape — only the former is a real noise signal.
 type ExtractCandidatesResult =
-  | { status: "ok"; candidates: CandidateMemory[]; groundingOrPolicyDropped?: boolean }
+  | { status: "ok"; candidates: CandidateMemory[]; rawCandidateCount: number; groundingOrPolicyDropped?: boolean }
   | { status: "llm_failure"; candidates: [] }
   | { status: "malformed"; candidates: [] }
   | { status: "empty_input"; candidates: [] };
@@ -734,12 +734,17 @@ export class SmartExtractor {
         // No LLM call was made, so the caller's rate limiter must not be charged.
         stats.skippedNoInput = true;
       }
-      if (extraction.status === "ok" && !extraction.groundingOrPolicyDropped) {
+      if (extraction.status === "ok" && extraction.rawCandidateCount === 0) {
         // LLM genuinely returned zero candidates → strongest noise signal → feedback to noise bank
         this.learnAsNoise(conversationText);
       } else if (extraction.status === "ok") {
-        this.debugLog(
-          "memory-pro: smart-extractor: skipping noise-bank learning (batch emptied by grounding/register/policy drops, not a genuine zero-extraction)",
+        // The model DID emit candidates; validation, grounding, or the batch
+        // contradiction check dropped them all. That is a policy verdict about
+        // those candidates, not evidence the conversation is noise — learning
+        // it as noise would pre-filter similar real content away from future
+        // extractions.
+        this.log(
+          `memory-pro: smart-extractor: skipping noise-bank learning (validation emptied the batch, raw=${extraction.rawCandidateCount})`,
         );
       } else {
         this.debugLog(
@@ -2201,13 +2206,23 @@ export class SmartExtractor {
       }
     }
 
+    if (contradictionDemotedCount > 0) {
+      // At the standard log level so a fully-demoted batch is distinguishable
+      // from "model found nothing" without debug logging (mirrors the
+      // admission-rejection lines).
+      this.log(
+        `memory-lancedb-pro: smart-extractor: batch contradiction demoted ${contradictionDemotedCount} real-tagged durable candidate(s) (register=${conversationRegister}, constructed siblings present)`,
+      );
+    }
+
     this.debugLog(
       `memory-lancedb-pro: smart-extractor: validation summary register=${conversationRegister}, accepted=${candidates.length}, invalidCategory=${invalidCategoryCount}, shortAbstract=${shortAbstractCount}, noiseAbstract=${noiseAbstractCount}, policyDropped=${policyDroppedCount}, constructedDropped=${constructedDroppedCount}, fictionRegisterDropped=${fictionRegisterDroppedCount}, contradictionDemoted=${contradictionDemotedCount}`,
     );
 
     return {
       status: "ok",
       candidates,
+      rawCandidateCount: result.memories.length,
       // A batch emptied by grounding, register, or policy drops is NOT a
       // "the LLM found nothing here" signal — the LLM found plenty and the
       // filters excluded it — so the caller must not train the noise bank

test/smart-extractor-noise-gating.test.mjs

@@ -4,10 +4,12 @@
  * extractCandidates() can come back with zero candidates for four different
  * reasons: the LLM/gateway call failed outright, the response had an
  * unexpected shape, the LLM genuinely returned an empty memories list, or
- * every parsed candidate was dropped by local validation. Only the latter
- * two are a real "nothing to remember" signal — the first two are failures
- * that must NOT be fed into the noise-prototype bank, or gateway outages
- * silently poison future extraction/recall.
+ * every parsed candidate was dropped/demoted by local validation. ONLY the
+ * genuinely-empty list is a real "nothing to remember" signal. Failures must
+ * not train the bank (gateway outages would poison it), and a
+ * validation-emptied batch must not either: the model DID find candidates
+ * there, so teaching the bank "this conversation is noise" would pre-filter
+ * similar real content away from future extractions.
  */
 
 import { describe, it } from "node:test";
@@ -42,6 +44,7 @@ function makeLlm(behavior) {
       if (behavior === "malformed_non_array") return { memories: "not-an-array" };
       if (behavior === "empty") return { memories: [] };
       if (Array.isArray(behavior)) return { memories: behavior };
+      if (behavior && typeof behavior === "object" && behavior.raw) return behavior.raw;
       throw new Error(`unsupported test behavior: ${behavior}`);
     },
   };
@@ -130,16 +133,69 @@ describe("SmartExtractor noise-bank learning gate", () => {
     assert.equal(noiseBank.learnCalls.length, 1, "a genuine empty list is a real noise signal");
   });
 
-  it("learns noise when every parsed candidate is dropped by local validation", async () => {
+  it("does NOT learn noise when validation drops every parsed candidate (policy verdict, not a noise signal)", async () => {
     const noiseBank = makeNoiseBank();
+    const logs = [];
     // "hi" is a valid category but fails the length>=5 abstract check, so the
-    // parsed response is well-formed yet every candidate is filtered out.
+    // parsed response is well-formed yet every candidate is filtered out. The
+    // model DID emit a candidate, so the conversation must not train the bank.
     const llm = makeLlm([{ category: "preferences", abstract: "hi", overview: "", content: "" }]);
+    const extractor = makeExtractor(makeEmbedder(), llm, makeStore(), { noiseBank, log: (msg) => logs.push(msg) });
+
+    await extractor.extractAndPersist("some conversation text", "s1");
+    await flushMicrotasks();
+
+    assert.equal(noiseBank.learnCalls.length, 0, "a validation-emptied batch must not train the noise bank");
+    assert.ok(
+      logs.some((msg) => msg.includes("skipping noise-bank learning (validation emptied the batch")),
+      "the skip must be visible at the standard log level",
+    );
+  });
+
+  it("does NOT learn noise when the batch contradiction check demotes every candidate", async () => {
+    const noiseBank = makeNoiseBank();
+    // Mixed-register batch with a constructed sibling: the real-tagged durable
+    // is demoted by the batch contradiction check, the constructed one is
+    // dropped by grounding enforcement. Raw output had 2 candidates, so the
+    // conversation (which contains real facts the policy demoted) must not be
+    // fed to the noise bank as a noise exemplar.
+    const llm = makeLlm({
+      raw: {
+        conversation_register: "mixed",
+        memories: [
+          { category: "preferences", abstract: "User loves space operas", overview: "- pref", content: "", grounding: "real" },
+          { category: "events", abstract: "The captain hid the artifact", overview: "- event", content: "", grounding: "constructed" },
+        ],
+      },
+    });
     const extractor = makeExtractor(makeEmbedder(), llm, makeStore(), { noiseBank });
 
     await extractor.extractAndPersist("some conversation text", "s1");
     await flushMicrotasks();
 
-    assert.equal(noiseBank.learnCalls.length, 1, "a validly-parsed-but-empty-after-filtering result is still a real noise signal");
+    assert.equal(noiseBank.learnCalls.length, 0, "a demotion-emptied batch must not train the noise bank");
+  });
+
+  it("logs the batch contradiction demotion at the standard log level, with count and register", async () => {
+    const noiseBank = makeNoiseBank();
+    const logs = [];
+    const llm = makeLlm({
+      raw: {
+        conversation_register: "mixed",
+        memories: [
+          { category: "preferences", abstract: "User loves space operas", overview: "- pref", content: "", grounding: "real" },
+          { category: "events", abstract: "The captain hid the artifact", overview: "- event", content: "", grounding: "constructed" },
+        ],
+      },
+    });
+    const extractor = makeExtractor(makeEmbedder(), llm, makeStore(), { noiseBank, log: (msg) => logs.push(msg) });
+
+    await extractor.extractAndPersist("some conversation text", "s1");
+    await flushMicrotasks();
+
+    const demotionLine = logs.find((msg) => msg.includes("batch contradiction demoted"));
+    assert.ok(demotionLine, "a fully-demoted batch must be distinguishable from 'model found nothing' without debug logging");
+    assert.ok(demotionLine.includes("1"), "the demotion line must carry the demoted count");
+    assert.ok(demotionLine.includes("mixed"), "the demotion line must carry the batch register");
   });
 });