Fix #2336: Feedback classifier leaks raw LLM completion (think tags, special tokens) into p

apps/memos-local-plugin/core/llm/providers/anthropic.ts

@@ -7,6 +7,7 @@
 
 import { ERROR_CODES, MemosError } from "../../../agent-contract/errors.js";
 import { decodeSse, httpPostJson, httpPostStream } from "../fetcher.js";
+import { sanitizeCompletionText } from "../sanitize.js";
 import type {
   LlmMessage,
   LlmProvider,
@@ -74,10 +75,12 @@ export class AnthropicLlmProvider implements LlmProvider {
       log,
     });
 
-    const text = (json.content ?? [])
-      .filter((b) => b.type === "text" && typeof b.text === "string")
-      .map((b) => b.text ?? "")
-      .join("");
+    const text = sanitizeCompletionText(
+      (json.content ?? [])
+        .filter((b) => b.type === "text" && typeof b.text === "string")
+        .map((b) => b.text ?? "")
+        .join(""),
+    );
     return {
       text,
       finishReason: mapFinish(json.stop_reason),

apps/memos-local-plugin/core/llm/providers/bedrock.ts

@@ -13,6 +13,7 @@
 
 import { ERROR_CODES, MemosError } from "../../../agent-contract/errors.js";
 import { httpPostJson } from "../fetcher.js";
+import { sanitizeCompletionText } from "../sanitize.js";
 import type {
   LlmMessage,
   LlmProvider,
@@ -92,7 +93,7 @@ export class BedrockLlmProvider implements LlmProvider {
     });
 
     const blocks = json.output?.message?.content ?? [];
-    const text = blocks.map((b) => b.text ?? "").join("");
+    const text = sanitizeCompletionText(blocks.map((b) => b.text ?? "").join(""));
     return {
       text,
       finishReason: mapFinish(json.stopReason),

apps/memos-local-plugin/core/llm/providers/gemini.ts

@@ -7,6 +7,7 @@
 
 import { ERROR_CODES, MemosError } from "../../../agent-contract/errors.js";
 import { decodeSse, httpPostJson, httpPostStream } from "../fetcher.js";
+import { sanitizeCompletionText } from "../sanitize.js";
 import type {
   LlmMessage,
   LlmProvider,
@@ -66,7 +67,9 @@ export class GeminiLlmProvider implements LlmProvider {
     });
 
     const cand = json.candidates?.[0];
-    const text = cand?.content?.parts?.map((p) => p.text ?? "").join("") ?? "";
+    const text = sanitizeCompletionText(
+      cand?.content?.parts?.map((p) => p.text ?? "").join("") ?? "",
+    );
     return {
       text,
       finishReason: mapFinish(cand?.finishReason),
@@ -109,7 +112,20 @@ export class GeminiLlmProvider implements LlmProvider {
       log,
     });
 
+    // Buffer the SSE deltas until the stream terminates, then sanitize the
+    // full accumulated text before yielding it (issue #2336 follow-up).
+    //
+    // Rationale: `<think>...</think>` blocks and DeepSeek-style special
+    // tokens can be split across chunk boundaries, so a naive per-chunk
+    // `sanitizeCompletionText` would miss any artifact that straddles two
+    // SSE events. Callers accumulate `chunk.delta` (see
+    // `core/llm/client.ts::stream`), so we can safely emit one sanitized
+    // delta at the tail: `chunks.map(c => c.delta).join("")` still yields
+    // the sanitized full text, matching the contract exercised by the
+    // provider tests.
     let done = false;
+    let accumulated = "";
+    let pendingUsage: GemResp["usageMetadata"] | undefined;
     for await (const payload of decodeSse(resp.body!)) {
       let evt: GemResp;
       try {
@@ -120,25 +136,43 @@ export class GeminiLlmProvider implements LlmProvider {
       const cand = evt.candidates?.[0];
       const delta = cand?.content?.parts?.map((p) => p.text ?? "").join("") ?? "";
       const finish = cand?.finishReason;
-      if (delta.length > 0) yield { delta, done: false };
+      if (delta.length > 0) accumulated += delta;
+      if (evt.usageMetadata) pendingUsage = evt.usageMetadata;
       if (finish) {
         done = true;
+        const sanitized = sanitizeCompletionText(accumulated);
+        if (sanitized.length > 0) yield { delta: sanitized, done: false };
+        const usage = evt.usageMetadata ?? pendingUsage;
         yield {
           delta: "",
           done: true,
           finishReason: mapFinish(finish),
-          usage: evt.usageMetadata
+          usage: usage
             ? {
-                promptTokens: evt.usageMetadata.promptTokenCount,
-                completionTokens: evt.usageMetadata.candidatesTokenCount,
-                totalTokens: evt.usageMetadata.totalTokenCount,
+                promptTokens: usage.promptTokenCount,
+                completionTokens: usage.candidatesTokenCount,
+                totalTokens: usage.totalTokenCount,
               }
             : undefined,
         };
         return;
       }
     }
-    if (!done) yield { delta: "", done: true };
+    if (!done) {
+      const sanitized = sanitizeCompletionText(accumulated);
+      if (sanitized.length > 0) yield { delta: sanitized, done: false };
+      yield {
+        delta: "",
+        done: true,
+        usage: pendingUsage
+          ? {
+              promptTokens: pendingUsage.promptTokenCount,
+              completionTokens: pendingUsage.candidatesTokenCount,
+              totalTokens: pendingUsage.totalTokenCount,
+            }
+          : undefined,
+      };
+    }
   }
 }
 

apps/memos-local-plugin/core/llm/providers/openai.ts

@@ -8,6 +8,7 @@
 import { ERROR_CODES, MemosError } from "../../../agent-contract/errors.js";
 import { applyOpenRouterProviderRouting } from "../../openrouter.js";
 import { decodeSse, httpPostJson, httpPostStream } from "../fetcher.js";
+import { sanitizeCompletionText } from "../sanitize.js";
 import type {
   LlmMessage,
   LlmProvider,
@@ -100,7 +101,7 @@ export class OpenAiLlmProvider implements LlmProvider {
     });
 
     const choice = json.choices?.[0];
-    const text = choice?.message?.content ?? "";
+    const text = sanitizeCompletionText(choice?.message?.content ?? "");
     return {
       text,
       finishReason: mapFinish(choice?.finish_reason),
@@ -166,12 +167,37 @@ export class OpenAiLlmProvider implements LlmProvider {
       log,
     });
 
+    // Buffer the SSE deltas until the stream terminates, then sanitize the
+    // full accumulated text before yielding it (issue #2336 follow-up).
+    //
+    // Rationale: `<think>...</think>` blocks and DeepSeek special tokens
+    // (`<｜end▁of▁sentence｜>`, etc.) can be split across chunk boundaries,
+    // so a naive per-chunk `sanitizeCompletionText` would miss any artifact
+    // that straddles two SSE events. Callers accumulate `chunk.delta` (see
+    // `core/llm/client.ts::stream`), so we can safely emit one sanitized
+    // delta at the tail: `chunks.map(c => c.delta).join("")` still yields
+    // the sanitized full text, matching the contract exercised by the
+    // provider tests.
     let emittedDone = false;
+    let accumulated = "";
+    let pendingUsage: OaResp["usage"] | undefined;
     for await (const payload of decodeSse(resp.body!)) {
       if (payload === "[DONE]") {
         if (!emittedDone) {
           emittedDone = true;
-          yield { delta: "", done: true };
+          const sanitized = sanitizeCompletionText(accumulated);
+          if (sanitized.length > 0) yield { delta: sanitized, done: false };
+          yield {
+            delta: "",
+            done: true,
+            usage: pendingUsage
+              ? {
+                  promptTokens: pendingUsage.prompt_tokens,
+                  completionTokens: pendingUsage.completion_tokens,
+                  totalTokens: pendingUsage.total_tokens,
+                }
+              : undefined,
+          };
         }
         return;
       }
@@ -186,26 +212,44 @@ export class OpenAiLlmProvider implements LlmProvider {
       const delta = choice?.delta?.content ?? "";
       const finish = choice?.finish_reason;
       if (delta.length > 0) {
-        yield { delta, done: false };
+        accumulated += delta;
       }
+      if (parsed.usage) pendingUsage = parsed.usage;
       if (finish) {
         emittedDone = true;
+        const sanitized = sanitizeCompletionText(accumulated);
+        if (sanitized.length > 0) yield { delta: sanitized, done: false };
+        const usage = parsed.usage ?? pendingUsage;
         yield {
           delta: "",
           done: true,
           finishReason: mapFinish(finish),
-          usage: parsed.usage
+          usage: usage
             ? {
-                promptTokens: parsed.usage.prompt_tokens,
-                completionTokens: parsed.usage.completion_tokens,
-                totalTokens: parsed.usage.total_tokens,
+                promptTokens: usage.prompt_tokens,
+                completionTokens: usage.completion_tokens,
+                totalTokens: usage.total_tokens,
               }
             : undefined,
         };
         return;
       }
     }
-    if (!emittedDone) yield { delta: "", done: true };
+    if (!emittedDone) {
+      const sanitized = sanitizeCompletionText(accumulated);
+      if (sanitized.length > 0) yield { delta: sanitized, done: false };
+      yield {
+        delta: "",
+        done: true,
+        usage: pendingUsage
+          ? {
+              promptTokens: pendingUsage.prompt_tokens,
+              completionTokens: pendingUsage.completion_tokens,
+              totalTokens: pendingUsage.total_tokens,
+            }
+          : undefined,
+      };
+    }
   }
 }
 

apps/memos-local-plugin/core/llm/sanitize.ts

@@ -0,0 +1,73 @@
+/**
+ * Strips gateway artifacts from LLM completion text before it is returned
+ * from a provider — see issue #2336.
+ *
+ * Thinking-enabled DeepSeek-family models served through gateways that keep
+ * the model's `<think>...</think>` reasoning block inside
+ * `choice.message.content` (rather than surfacing it via a separate
+ * `reasoning_content` field) leak that block into `ProviderCompletion.text`.
+ * Ops that persist this text — most notably `feedback.classify`, which
+ * writes into `FeedbackRow.rationale` — then pollute retrieval-visible
+ * memory with strings like `"</think>\n<｜end▁of▁sentence｜>\n..."`, and
+ * that pollution flows back into the model's own context on future turns.
+ *
+ * This module centralizes the sanitizer so every provider strips the same
+ * set of artifacts. Fixing it here is O(1) call sites for O(N) ops.
+ */
+
+/**
+ * Matched `<think>...</think>` block. Non-greedy so multiple blocks are
+ * stripped independently; multi-line so newlines inside the block are
+ * consumed. Case-insensitive because some gateways upcase tag names.
+ */
+const THINK_BLOCK_RE = /<\s*think\b[^>]*>[\s\S]*?<\s*\/\s*think\s*>/gi;
+
+/**
+ * Orphan closing `</think>` fragment. The reporter's evidence in #2336
+ * starts with one — a gateway that truncates thinking-block output can
+ * drop the opener while leaving the closer intact.
+ */
+const ORPHAN_CLOSE_THINK_RE = /<\s*\/\s*think\s*>/gi;
+
+/**
+ * Orphan opening `<think>` fragment plus everything after it. Symmetric
+ * defense against the reverse failure mode: gateway keeps the opener but
+ * cuts before the closer, leaving reasoning trailing off the response.
+ * We drop everything from the opener onward because the model has already
+ * committed to reasoning-mode output — retaining it would still be
+ * unactionable thinking text.
+ */
+const ORPHAN_OPEN_THINK_RE = /<\s*think\b[^>]*>[\s\S]*$/i;
+
+/**
+ * DeepSeek-family special tokens. Uses the Chinese full-width `｜`
+ * (U+FF5C) that the model actually emits — NOT the ASCII `|`. Some
+ * gateways strip these before sending, others don't; strip them here
+ * unconditionally so the invariant is provider-agnostic.
+ */
+const DEEPSEEK_SPECIAL_TOKENS_RE =
+  /<｜(?:begin|end)▁(?:of▁sentence|of▁session)｜>/g;
+
+/**
+ * Collapse runs of ≥3 blank lines that the strip may have introduced.
+ * Two consecutive newlines (a paragraph break) are still allowed.
+ */
+const EXCESS_BLANK_LINES_RE = /\n{3,}/g;
+
+/**
+ * Remove `<think>` blocks and DeepSeek gateway tokens from LLM completion
+ * text. Idempotent and safe on empty input.
+ *
+ * Order matters: strip matched blocks first, THEN orphan fragments — the
+ * orphan patterns are broad and would eat surrounding content if a
+ * well-formed block hadn't been removed first.
+ */
+export function sanitizeCompletionText(text: string): string {
+  if (!text) return text;
+  let out = text.replace(THINK_BLOCK_RE, "");
+  out = out.replace(ORPHAN_OPEN_THINK_RE, "");
+  out = out.replace(ORPHAN_CLOSE_THINK_RE, "");
+  out = out.replace(DEEPSEEK_SPECIAL_TOKENS_RE, "");
+  out = out.replace(EXCESS_BLANK_LINES_RE, "\n\n");
+  return out.trim();
+}