From c2cf01b3acf99322ffd9bf382271f5b849f21e46 Mon Sep 17 00:00:00 2001 From: autodev Date: Thu, 3 Sep 2026 11:43:29 +0800 Subject: [PATCH 1/2] fix(l3): synthesise fallback title when abstractor returns empty (#2335) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit l3.abstraction previously threw LLM_OUTPUT_MALFORMED when the LLM returned an empty/whitespace `title`, aborting the entire world-model generation for the cluster. In production (v2.0.17) this turned a single flaky LLM response into a full pipeline failure. Soften the validator: keep the triple (environment / inference / constraints) as load-bearing but derive `title` from `domain_tags`, first environment label, or a static "Untitled world model" when the LLM leaves it empty. Emit `abstract.title_fallback` warn so ops still see the regression. Apply the same pattern to l2.induction — `title` is a display attribute while `trigger` and `procedure` remain load-bearing. --- .../core/memory/l2/induce.ts | 55 +++++++++++--- .../core/memory/l3/abstract.ts | 55 +++++++++++--- .../tests/unit/memory/l2/induce.test.ts | 32 +++++++- .../tests/unit/memory/l3/abstract.test.ts | 73 +++++++++++++++++++ 4 files changed, 193 insertions(+), 22 deletions(-) diff --git a/apps/memos-local-plugin/core/memory/l2/induce.ts b/apps/memos-local-plugin/core/memory/l2/induce.ts index 2ea750cd2..b967d4c59 100644 --- a/apps/memos-local-plugin/core/memory/l2/induce.ts +++ b/apps/memos-local-plugin/core/memory/l2/induce.ts @@ -110,14 +110,17 @@ export async function induceDraft( schemaHint: `{"title":"...","trigger":"...","procedure":"...","verification":"...","rationale":"...","caveats":["..."],"confidence":0..1,"support_trace_ids":["tr_..."]}`, validate: (v) => { const o = v as Record; - for (const k of ["title", "trigger"]) { - if (typeof o[k] !== "string" || !(o[k] as string).trim()) { - throw new MemosError( - ERROR_CODES.LLM_OUTPUT_MALFORMED, - `l2.induction: '${k}' must be a non-empty string`, - { got: o[k] }, - ); - } + // `trigger` is load-bearing — a policy without a trigger has + // no matching semantics. `title` is a display attribute; if + // empty, `normaliseDraft` synthesises a fallback from the + // pattern signature so a single flaky LLM response doesn't + // discard an otherwise-usable draft. + if (typeof o.trigger !== "string" || !(o.trigger as string).trim()) { + throw new MemosError( + ERROR_CODES.LLM_OUTPUT_MALFORMED, + `l2.induction: 'trigger' must be a non-empty string`, + { got: o.trigger }, + ); } if ( typeof o.procedure !== "string" && @@ -133,7 +136,18 @@ export async function induceDraft( }, ); - const draft = normaliseDraft(rsp.value, input.evidenceTraces.map((t) => t.id)); + const draft = normaliseDraft( + rsp.value, + input.evidenceTraces.map((t) => t.id), + input.signatureLabel, + ); + const rawTitle = (rsp.value as { title?: unknown }).title; + if (typeof rawTitle !== "string" || rawTitle.trim().length === 0) { + log.warn("induce.title_fallback", { + signatureLabel: input.signatureLabel, + synthesisedTitle: draft.title, + }); + } if (deps.validate) deps.validate(draft); return { ok: true, draft }; } catch (err) { @@ -230,7 +244,11 @@ function truncate(s: string, n: number): string { return s.slice(0, n - 1) + "…"; } -function normaliseDraft(value: Record, traceIds: readonly TraceId[]): InductionDraft { +function normaliseDraft( + value: Record, + traceIds: readonly TraceId[], + signatureLabel: string, +): InductionDraft { const procedure = typeof value.procedure === "string" ? (value.procedure as string) @@ -244,8 +262,12 @@ function normaliseDraft(value: Record, traceIds: readonly Trace const supportTraceIds = Array.isArray(value.support_trace_ids) ? (value.support_trace_ids as unknown[]).filter((x): x is string => typeof x === "string") : []; + const cleanedTitle = sanitizeDerivedText(value.title); + const title = cleanedTitle.length > 0 + ? cleanedTitle + : synthesiseTitle(signatureLabel); return { - title: sanitizeDerivedText(value.title), + title, trigger: sanitizeDerivedMarkdown(value.trigger), procedure: sanitizeDerivedMarkdown(procedure), verification: typeof value.verification === "string" ? sanitizeDerivedMarkdown(value.verification) : "", @@ -256,3 +278,14 @@ function normaliseDraft(value: Record, traceIds: readonly Trace supportTraceIds: supportTraceIds.length > 0 ? (supportTraceIds as TraceId[]) : Array.from(traceIds), }; } + +/** + * Build a display-quality title from the pattern signature when the LLM + * returns an empty `title`. Keeps the whole draft usable instead of + * throwing away the trigger/procedure that we did get. + */ +function synthesiseTitle(signatureLabel: string): string { + const clean = signatureLabel.trim(); + if (clean.length > 0) return clean.slice(0, 120); + return "Untitled policy"; +} diff --git a/apps/memos-local-plugin/core/memory/l3/abstract.ts b/apps/memos-local-plugin/core/memory/l3/abstract.ts index 0bde96fb3..dcee25b74 100644 --- a/apps/memos-local-plugin/core/memory/l3/abstract.ts +++ b/apps/memos-local-plugin/core/memory/l3/abstract.ts @@ -104,13 +104,11 @@ export async function abstractDraft( schemaHint: `{"title":"...","domain_tags":["..."],"environment":[{"label":"...","description":"...","evidenceIds":["..."]}],"inference":[...],"constraints":[...],"body":"markdown","confidence":0..1,"supersedes_world_ids":[]}`, validate: (v) => { const o = v as Record; - if (typeof o.title !== "string" || !(o.title as string).trim()) { - throw new MemosError( - ERROR_CODES.LLM_OUTPUT_MALFORMED, - "l3.abstraction: 'title' must be a non-empty string", - { got: o.title }, - ); - } + // Empty `title` is tolerated — `normaliseDraft` synthesises a + // fallback from `domain_tags` / first environment label so a + // single missing display attribute doesn't abort the entire + // world-model. Missing triple, however, means the draft has + // no usable payload and must still fail. const triple = ["environment", "inference", "constraints"]; for (const k of triple) { if (!Array.isArray(o[k])) { @@ -126,6 +124,12 @@ export async function abstractDraft( ); const draft = normaliseDraft(rsp.value); + if (isEmptyString((rsp.value as Record).title)) { + log.warn("abstract.title_fallback", { + clusterKey: input.cluster.key, + synthesisedTitle: draft.title, + }); + } if (deps.validate) deps.validate(draft); return { ok: true, draft }; } catch (err) { @@ -260,9 +264,14 @@ function packPolicy( function normaliseDraft(value: Record): L3AbstractionDraft { const triple = pickTriple(value); + const domainTags = normaliseTags(value.domain_tags); + const rawTitle = sanitizeDerivedText(value.title); + const title = rawTitle.length > 0 + ? rawTitle + : synthesiseTitle(domainTags, triple.environment); return { - title: sanitizeDerivedText(value.title), - domainTags: normaliseTags(value.domain_tags), + title, + domainTags, environment: triple.environment, inference: triple.inference, constraints: triple.constraints, @@ -276,6 +285,34 @@ function normaliseDraft(value: Record): L3AbstractionDraft { }; } +function isEmptyString(v: unknown): boolean { + return typeof v !== "string" || v.trim().length === 0; +} + +/** + * Build a display-quality title from whatever structure the draft carries. + * The abstractor's `title` is a display attribute; a missing one shouldn't + * lose the whole world model. See `docs/openspec/changes/…-2335-…/design.md`. + */ +function synthesiseTitle( + domainTags: readonly string[], + environment: readonly L3AbstractionDraftEntry[], +): string { + if (domainTags.length > 0) { + const joined = domainTags.slice(0, 3).map(titleCaseTag).join(" · "); + if (joined.trim().length > 0) return joined.slice(0, 160); + } + for (const e of environment) { + if (e.label && e.label.trim().length > 0) return e.label.slice(0, 160); + } + return "Untitled world model"; +} + +function titleCaseTag(tag: string): string { + if (tag.length === 0) return tag; + return tag.charAt(0).toUpperCase() + tag.slice(1); +} + function pickTriple(value: Record): { environment: L3AbstractionDraftEntry[]; inference: L3AbstractionDraftEntry[]; diff --git a/apps/memos-local-plugin/tests/unit/memory/l2/induce.test.ts b/apps/memos-local-plugin/tests/unit/memory/l2/induce.test.ts index 06d61459f..c3b2ce7be 100644 --- a/apps/memos-local-plugin/tests/unit/memory/l2/induce.test.ts +++ b/apps/memos-local-plugin/tests/unit/memory/l2/induce.test.ts @@ -146,10 +146,38 @@ describe("memory/l2/induce", () => { expect(res.reason).toBe("llm_failed"); }); - it("reason=llm_failed when the LLM draft is malformed (missing title)", async () => { + it("synthesises a fallback title from the pattern signature when the LLM omits it", async () => { + // Regression for #2335 (parity with l3.abstract): the LLM + // occasionally returns an empty `title`. That is a display attribute, + // not a load-bearing field — the whole draft shouldn't be discarded. const llm = fakeLlm({ completeJson: { - "l2.l2.induction.v2": { trigger: "no title", procedure: "..." }, + "l2.l2.induction.v2": { + title: " ", + trigger: "pip install fails in alpine", + procedure: "apk add + retry", + confidence: 0.6, + }, + }, + }); + const res = await induceDraft( + { + evidenceTraces: [mkTrace("tr_a", "ep_1", vec([1, 0]))], + episodeIds: ["ep_1"] as EpisodeId[], + signatureLabel: "docker|pip|MODULE_NOT_FOUND", + charCap: 1000, + }, + { llm, log }, + ); + expect(res.ok).toBe(true); + if (!res.ok) return; + expect(res.draft.title).toBe("docker|pip|MODULE_NOT_FOUND"); + }); + + it("reason=llm_failed when the LLM draft is malformed (missing trigger)", async () => { + const llm = fakeLlm({ + completeJson: { + "l2.l2.induction.v2": { title: "have title", procedure: "..." }, }, }); const res = await induceDraft( diff --git a/apps/memos-local-plugin/tests/unit/memory/l3/abstract.test.ts b/apps/memos-local-plugin/tests/unit/memory/l3/abstract.test.ts index ff759b17f..ec2c93dcd 100644 --- a/apps/memos-local-plugin/tests/unit/memory/l3/abstract.test.ts +++ b/apps/memos-local-plugin/tests/unit/memory/l3/abstract.test.ts @@ -180,6 +180,79 @@ describe("memory/l3/abstract", () => { expect(res.detail).toContain("boom"); }); + it("synthesises a fallback title from domain_tags when the LLM returns empty title", async () => { + const llm = fakeLlm({ + completeJson: { + [OP]: { + title: " ", + domain_tags: ["Alpine", "python", "pip"], + environment: [{ label: "Alpine", description: "musl libc" }], + inference: [{ label: "wheels fail", description: "musl" }], + constraints: [{ label: "avoid wheels", description: "no binary" }], + body: "body", + confidence: 0.6, + supersedes_world_ids: [], + }, + }, + }); + + const res = await abstractDraft( + { cluster: mkCluster(), evidenceByPolicy: new Map() }, + { llm, log, config: cfg() }, + ); + + expect(res.ok).toBe(true); + if (!res.ok) return; + expect(res.draft.title.trim().length).toBeGreaterThan(0); + expect(res.draft.title.toLowerCase()).toContain("alpine"); + }); + + it("falls back to environment label when title and domain_tags are empty", async () => { + const llm = fakeLlm({ + completeJson: { + [OP]: { + title: "", + domain_tags: [], + environment: [{ label: "Runtime", description: "runtime" }], + inference: [{ label: "x", description: "y" }], + constraints: [{ label: "z", description: "w" }], + body: "", + confidence: 0.5, + }, + }, + }); + const res = await abstractDraft( + { cluster: mkCluster(), evidenceByPolicy: new Map() }, + { llm, log, config: cfg() }, + ); + expect(res.ok).toBe(true); + if (!res.ok) return; + expect(res.draft.title).toBe("Runtime"); + }); + + it("falls back to a static title when title / tags / environment are all empty", async () => { + const llm = fakeLlm({ + completeJson: { + [OP]: { + title: "", + domain_tags: [], + environment: [{ label: "", description: "d" }], + inference: [{ label: "x", description: "y" }], + constraints: [{ label: "z", description: "w" }], + body: "", + confidence: 0.5, + }, + }, + }); + const res = await abstractDraft( + { cluster: mkCluster(), evidenceByPolicy: new Map() }, + { llm, log, config: cfg() }, + ); + expect(res.ok).toBe(true); + if (!res.ok) return; + expect(res.draft.title).toBe("Untitled world model"); + }); + it("returns llm_failed when the LLM returns missing triple", async () => { const llm = fakeLlm({ completeJson: { From ed61fb9928d9f1ac2b8ac159aaaea3c9a286cb2e Mon Sep 17 00:00:00 2001 From: autodev Date: Thu, 3 Sep 2026 12:01:07 +0800 Subject: [PATCH 2/2] refactor(l2/l3): address OCR review findings on title-fallback MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Inline the raw-title check in l3.abstract (parity with l2.induce); drop the now-unused isEmptyString helper. - Add rawTitleType to the induce/abstract title_fallback warn logs so operators can distinguish null vs. "" vs. non-string LLM responses. - Route l2.induce.synthesiseTitle through sanitizeDerivedText + the shared truncate() helper so the fallback path matches the normal sanitised/ellipsised path. - Make l3.abstract's titleCaseTag and synthesiseTitle Unicode-safe by iterating over code points instead of UTF-16 code units, so emoji or surrogate-pair tags no longer corrupt the display title. - Drop the outdated docs/openspec/…-2335-….md pointer from the synthesiseTitle JSDoc — the referenced path never landed. --- .../core/memory/l2/induce.ts | 5 +++-- .../core/memory/l3/abstract.ts | 19 ++++++++++--------- 2 files changed, 13 insertions(+), 11 deletions(-) diff --git a/apps/memos-local-plugin/core/memory/l2/induce.ts b/apps/memos-local-plugin/core/memory/l2/induce.ts index b967d4c59..31a1befa7 100644 --- a/apps/memos-local-plugin/core/memory/l2/induce.ts +++ b/apps/memos-local-plugin/core/memory/l2/induce.ts @@ -146,6 +146,7 @@ export async function induceDraft( log.warn("induce.title_fallback", { signatureLabel: input.signatureLabel, synthesisedTitle: draft.title, + rawTitleType: typeof rawTitle, }); } if (deps.validate) deps.validate(draft); @@ -285,7 +286,7 @@ function normaliseDraft( * throwing away the trigger/procedure that we did get. */ function synthesiseTitle(signatureLabel: string): string { - const clean = signatureLabel.trim(); - if (clean.length > 0) return clean.slice(0, 120); + const clean = sanitizeDerivedText(signatureLabel); + if (clean.length > 0) return truncate(clean, 120); return "Untitled policy"; } diff --git a/apps/memos-local-plugin/core/memory/l3/abstract.ts b/apps/memos-local-plugin/core/memory/l3/abstract.ts index dcee25b74..98eef5f76 100644 --- a/apps/memos-local-plugin/core/memory/l3/abstract.ts +++ b/apps/memos-local-plugin/core/memory/l3/abstract.ts @@ -124,10 +124,12 @@ export async function abstractDraft( ); const draft = normaliseDraft(rsp.value); - if (isEmptyString((rsp.value as Record).title)) { + const rawTitle = (rsp.value as { title?: unknown }).title; + if (typeof rawTitle !== "string" || rawTitle.trim().length === 0) { log.warn("abstract.title_fallback", { clusterKey: input.cluster.key, synthesisedTitle: draft.title, + rawTitleType: typeof rawTitle, }); } if (deps.validate) deps.validate(draft); @@ -285,14 +287,10 @@ function normaliseDraft(value: Record): L3AbstractionDraft { }; } -function isEmptyString(v: unknown): boolean { - return typeof v !== "string" || v.trim().length === 0; -} - /** * Build a display-quality title from whatever structure the draft carries. * The abstractor's `title` is a display attribute; a missing one shouldn't - * lose the whole world model. See `docs/openspec/changes/…-2335-…/design.md`. + * lose the whole world model. */ function synthesiseTitle( domainTags: readonly string[], @@ -300,17 +298,20 @@ function synthesiseTitle( ): string { if (domainTags.length > 0) { const joined = domainTags.slice(0, 3).map(titleCaseTag).join(" · "); - if (joined.trim().length > 0) return joined.slice(0, 160); + return [...joined].slice(0, 160).join(""); } for (const e of environment) { - if (e.label && e.label.trim().length > 0) return e.label.slice(0, 160); + if (e.label && e.label.trim().length > 0) { + return [...e.label].slice(0, 160).join(""); + } } return "Untitled world model"; } function titleCaseTag(tag: string): string { if (tag.length === 0) return tag; - return tag.charAt(0).toUpperCase() + tag.slice(1); + const chars = [...tag]; + return chars[0].toUpperCase() + chars.slice(1).join(""); } function pickTriple(value: Record): {