From 20cd0076d3eeac5670b967321f3712e78a3523ec Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 16 Oct 2026 17:10:50 +0000 Subject: [PATCH 1/5] fix(rag): treat required and needed as question wording in relevance terms The word required in "What assessment documentation is required?" was a core evidence term, so the Assessment Documentation guideline, which never says required, scored only a partial match while unrelated pages that say required scored direct and were listed ahead of it. Co-Authored-By: Claude Opus 5.5 (1M context) Claude-Session: https://claude.ai/code/session_01VkKjYakMrMXofU8foQCgbq --- src/lib/evidence-relevance.ts | 7 +++++++ tests/evidence-relevance.test.ts | 28 ++++++++++++++++++++++++++++ 2 files changed, 35 insertions(+) diff --git a/src/lib/evidence-relevance.ts b/src/lib/evidence-relevance.ts index d00bce61f1..fe0b8ae830 100644 --- a/src/lib/evidence-relevance.ts +++ b/src/lib/evidence-relevance.ts @@ -40,12 +40,19 @@ const genericQueryTerms = new Set([ "item", "list", "management", + // Requirement verbs frame the question like "should" does. Kept as terms, + // they let any passage that happens to say "required" outrank the guideline + // on the topic. "requirement" stays: it names a subject. + "need", + "needed", "overview", "passage", "patient", "policy", "question", "recommendation", + "require", + "required", "review", "reviewed", "reviewing", diff --git a/tests/evidence-relevance.test.ts b/tests/evidence-relevance.test.ts index 777aa5f340..6234786987 100644 --- a/tests/evidence-relevance.test.ts +++ b/tests/evidence-relevance.test.ts @@ -39,6 +39,34 @@ describe("evidence relevance", () => { expect(terms).not.toContain("reviewed"); }); + it("treats requirement verbs as question wording, not evidence terms", () => { + expect(queryCoreTerms("What assessment documentation is required?")).toEqual(["assessment", "documentation"]); + expect(queryCoreTerms("What monitoring does lithium require?")).not.toContain("require"); + expect(queryCoreTerms("What monitoring is needed for valproate?")).not.toContain("needed"); + expect(queryCoreTerms("What does the infant need?")).not.toContain("need"); + expect(queryCoreTerms("What documentation requirements apply?")).toContain("requirement"); + }); + + it("gives the on-topic guideline the same match level as a passage that merely says required", () => { + const query = "What assessment documentation is required?"; + const [onTopic, offTopic] = annotateSearchResults(query, [ + result({ + id: "assessment-documentation", + title: "Assessment Documentation", + content: + "Any outstanding assessments must be documented, and assessment documentation is completed on admission.", + }), + result({ + id: "admissions", + title: "Admissions Policy", + content: "Patients accepted for admission must have accompanying assessment documentation where required.", + }), + ]); + + expect(onTopic?.relevance?.verdict).toBe("direct"); + expect(offTopic?.relevance?.verdict).toBe("direct"); + }); + it("classifies weak lithium-style neighboring sources as nearby, not direct", () => { const relevance = buildEvidenceRelevance("What toxicity safety-net symptoms should be reviewed for lithium?", [ result({ From 03259b71fa26f4431ad2eab709b3a8697291add6 Mon Sep 17 00:00:00 2001 From: "qodo-code-review[bot]" <151058649+qodo-code-review[bot]@users.noreply.github.com> Date: Sat, 17 Oct 2026 01:45:44 +0000 Subject: [PATCH 2/5] fix: Preserve substantive uses of needs --- src/lib/evidence-relevance.ts | 19 +++++++++++++++++-- tests/evidence-relevance.test.ts | 10 ++++++++++ 2 files changed, 27 insertions(+), 2 deletions(-) diff --git a/src/lib/evidence-relevance.ts b/src/lib/evidence-relevance.ts index fe0b8ae830..f875c82626 100644 --- a/src/lib/evidence-relevance.ts +++ b/src/lib/evidence-relevance.ts @@ -116,6 +116,13 @@ function normalizeTerm(term: string) { return cleaned; } +// "Need" can frame a question ("what does the infant need?") or name the +// subject being asked about ("what are the carers' needs?"). Singularization +// makes both forms identical, so retain the latter before filtering generic +// question wording. +const substantiveNeedNounPattern = + /\b(?:[a-z][a-z-]*['’]s?\s+needs?|(?:support|clinical|care|health|service|practical|emotional|communication|additional|specific|individual|complex|unmet|patient|carer|carers)\s+needs?|needs?\s+(?:of|assessment|include|are|were|do|does|did|have|has))\b/i; + // Only this interrogative scaffold becomes a semantic cadence requirement. const monitoringFrequencyQuestion = /\bHow often (?:is|are) ([^?.]+?) (?:checked|monitored|measured|reviewed)(?=[?.]|$|\s+(?:in|for|with|without|among)\b)/i; @@ -143,8 +150,16 @@ export function queryCoreTerms(query: string) { const normalized = normalizedClinicalSearchTokens(semanticQuery).map(normalizeTerm).filter(Boolean); const raw = (semanticQuery.toLowerCase().match(/[a-z0-9]+/g) ?? []).map(normalizeTerm).filter(Boolean); const candidates = uniq([...normalized, ...raw], 14); - const specific = candidates.filter((term) => term.length >= 3 && !genericQueryTerms.has(term)); - return specific.length ? specific.slice(0, 10) : candidates.filter((term) => term.length >= 3).slice(0, 10); + const hasSubstantiveNeedNoun = substantiveNeedNounPattern.test(semanticQuery); + const specific = candidates.filter( + (term) => + term.length >= 3 && + (!genericQueryTerms.has(term) || (term === "need" && hasSubstantiveNeedNoun)), + ); + const fallback = candidates.filter( + (term) => term.length >= 3 && (term !== "need" || hasSubstantiveNeedNoun), + ); + return specific.length ? specific.slice(0, 10) : fallback.slice(0, 10); } function textIncludesTerm(text: string, term: string) { diff --git a/tests/evidence-relevance.test.ts b/tests/evidence-relevance.test.ts index 6234786987..51bd5ff061 100644 --- a/tests/evidence-relevance.test.ts +++ b/tests/evidence-relevance.test.ts @@ -44,9 +44,19 @@ describe("evidence relevance", () => { expect(queryCoreTerms("What monitoring does lithium require?")).not.toContain("require"); expect(queryCoreTerms("What monitoring is needed for valproate?")).not.toContain("needed"); expect(queryCoreTerms("What does the infant need?")).not.toContain("need"); + expect(queryCoreTerms("What are the carers' support needs?")).toContain("need"); expect(queryCoreTerms("What documentation requirements apply?")).toContain("requirement"); }); + it("does not treat a carers-only passage as direct evidence for their needs", () => { + const relevance = buildSourceRelevance("What are the carers' support needs?", result({ + content: "Carers are involved in discharge planning and ward communication.", + })); + + expect(relevance.missingTerms).toContain("need"); + expect(relevance.verdict).not.toBe("direct"); + }); + it("gives the on-topic guideline the same match level as a passage that merely says required", () => { const query = "What assessment documentation is required?"; const [onTopic, offTopic] = annotateSearchResults(query, [ From 7ffdfc772d9e95a2e1c46364665e2829c1cf7811 Mon Sep 17 00:00:00 2001 From: Claude Date: Sat, 17 Oct 2026 01:51:07 +0000 Subject: [PATCH 3/5] fix(rag): keep the noun needs as a relevance term Requirement verbs are now removed by whole word before plural stripping, so "support needs" still requires a passage to cover needs. Co-Authored-By: Claude Opus 5.5 (1M context) Claude-Session: https://claude.ai/code/session_01VkKjYakMrMXofU8foQCgbq --- src/lib/evidence-relevance.ts | 16 ++++++++-------- tests/evidence-relevance.test.ts | 13 +++++++++++++ 2 files changed, 21 insertions(+), 8 deletions(-) diff --git a/src/lib/evidence-relevance.ts b/src/lib/evidence-relevance.ts index fe0b8ae830..4a475f540f 100644 --- a/src/lib/evidence-relevance.ts +++ b/src/lib/evidence-relevance.ts @@ -40,19 +40,12 @@ const genericQueryTerms = new Set([ "item", "list", "management", - // Requirement verbs frame the question like "should" does. Kept as terms, - // they let any passage that happens to say "required" outrank the guideline - // on the topic. "requirement" stays: it names a subject. - "need", - "needed", "overview", "passage", "patient", "policy", "question", "recommendation", - "require", - "required", "review", "reviewed", "reviewing", @@ -116,6 +109,12 @@ function normalizeTerm(term: string) { return cleaned; } +// Requirement verbs frame the question like "should" does. Kept as terms, they +// let any passage that happens to say "required" outrank the guideline on the +// topic. They are removed before plural stripping, so the nouns "needs" and +// "requirements" still name a subject. +const requirementVerbs = /\b(?:need|needed|require|required|requires)\b/gi; + // Only this interrogative scaffold becomes a semantic cadence requirement. const monitoringFrequencyQuestion = /\bHow often (?:is|are) ([^?.]+?) (?:checked|monitored|measured|reviewed)(?=[?.]|$|\s+(?:in|for|with|without|among)\b)/i; @@ -139,7 +138,8 @@ export function queryCoreTerms(query: string) { const semanticQuery = request .replace(/\bFocus on the requested\s+/g, "") .replace(/\b(What|Which)\b([^?.]*?)\bappl(?:y|ies) to\b/gi, "$1$2for") - .replace(monitoringFrequencyQuestion, "$1 frequency"); + .replace(monitoringFrequencyQuestion, "$1 frequency") + .replace(requirementVerbs, " "); const normalized = normalizedClinicalSearchTokens(semanticQuery).map(normalizeTerm).filter(Boolean); const raw = (semanticQuery.toLowerCase().match(/[a-z0-9]+/g) ?? []).map(normalizeTerm).filter(Boolean); const candidates = uniq([...normalized, ...raw], 14); diff --git a/tests/evidence-relevance.test.ts b/tests/evidence-relevance.test.ts index 6234786987..681c758232 100644 --- a/tests/evidence-relevance.test.ts +++ b/tests/evidence-relevance.test.ts @@ -45,6 +45,19 @@ describe("evidence relevance", () => { expect(queryCoreTerms("What monitoring is needed for valproate?")).not.toContain("needed"); expect(queryCoreTerms("What does the infant need?")).not.toContain("need"); expect(queryCoreTerms("What documentation requirements apply?")).toContain("requirement"); + expect(queryCoreTerms("What are the support needs in autism?")).toContain("need"); + }); + + it("keeps the noun needs as evidence a passage must cover", () => { + const [source] = annotateSearchResults("What are the support needs in autism?", [ + result({ + title: "Autism overview", + content: "Autism is a neurodevelopmental condition with onset in early childhood and a range of presentations.", + }), + ]); + + expect(source?.relevance?.verdict).not.toBe("direct"); + expect(source?.relevance?.missingTerms).toContain("need"); }); it("gives the on-topic guideline the same match level as a passage that merely says required", () => { From 83a4c93e3268698a1def6fd44a6bcae75b8a466f Mon Sep 17 00:00:00 2001 From: Claude Date: Sat, 17 Oct 2026 01:59:33 +0000 Subject: [PATCH 4/5] fix(relevance): drop "needs to" as wording, keep the noun "the need for" "needs to" is the requirement verb, so it no longer becomes a core term. "need" after a determiner or modifier ("the clinical need for lithium") names the subject, so it stays a term a passage must cover. RAG impact: behaviour change, covered by the PR 3591 canary pair. Co-Authored-By: Claude Opus 5.5 (1M context) Claude-Session: https://claude.ai/code/session_01VkKjYakMrMXofU8foQCgbq --- src/lib/evidence-relevance.ts | 6 ++++-- tests/evidence-relevance.test.ts | 17 +++++++++++++++++ 2 files changed, 21 insertions(+), 2 deletions(-) diff --git a/src/lib/evidence-relevance.ts b/src/lib/evidence-relevance.ts index 4a475f540f..fe3aa3ef6d 100644 --- a/src/lib/evidence-relevance.ts +++ b/src/lib/evidence-relevance.ts @@ -112,8 +112,10 @@ function normalizeTerm(term: string) { // Requirement verbs frame the question like "should" does. Kept as terms, they // let any passage that happens to say "required" outrank the guideline on the // topic. They are removed before plural stripping, so the nouns "needs" and -// "requirements" still name a subject. -const requirementVerbs = /\b(?:need|needed|require|required|requires)\b/gi; +// "requirements" still name a subject. "needs" goes only as the verb in "needs to", +// and "need" stays after a determiner or modifier, as in "the clinical need for". +const requirementVerbs = + /\b(? { expect(queryCoreTerms("What are the carers' support needs?")).toContain("need"); expect(queryCoreTerms("What documentation requirements apply?")).toContain("requirement"); expect(queryCoreTerms("What are the support needs in autism?")).toContain("need"); + expect(queryCoreTerms("What assessment documentation needs to be completed?")).toEqual([ + "assessment", + "documentation", + "completed", + ]); + expect(queryCoreTerms("What is the clinical need for lithium?")).toContain("need"); + expect(queryCoreTerms("Is there a need for clozapine monitoring?")).toContain("need"); + }); + + it("does not treat a lithium-only passage as direct evidence for the clinical need", () => { + const relevance = buildSourceRelevance( + "What is the clinical need for lithium?", + result({ content: "Lithium is available as 250 mg and 450 mg tablets." }), + ); + + expect(relevance.missingTerms).toContain("need"); + expect(relevance.verdict).not.toBe("direct"); }); it("keeps the noun needs as evidence a passage must cover", () => { From c37ca23688df085d688c731485a1f99509aa02c4 Mon Sep 17 00:00:00 2001 From: Claude Date: Sat, 17 Oct 2026 02:06:20 +0000 Subject: [PATCH 5/5] fix(relevance): keep "need" in "an ongoing need for" and required-versus-optional questions "need" followed by "for" with an article two words back is a noun, so it stays a term. A question that contrasts required with optional keeps the requirement words, since they name one side of the comparison. RAG impact: behaviour change, covered by the PR 3591 canary pair. Co-Authored-By: Claude Opus 5.5 (1M context) Claude-Session: https://claude.ai/code/session_01VkKjYakMrMXofU8foQCgbq --- src/lib/evidence-relevance.ts | 7 ++++--- tests/evidence-relevance.test.ts | 3 +++ 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/src/lib/evidence-relevance.ts b/src/lib/evidence-relevance.ts index fe3aa3ef6d..fee77fa676 100644 --- a/src/lib/evidence-relevance.ts +++ b/src/lib/evidence-relevance.ts @@ -113,9 +113,10 @@ function normalizeTerm(term: string) { // let any passage that happens to say "required" outrank the guideline on the // topic. They are removed before plural stripping, so the nouns "needs" and // "requirements" still name a subject. "needs" goes only as the verb in "needs to", -// and "need" stays after a determiner or modifier, as in "the clinical need for". +// and "need" stays after a determiner or modifier, as in "the clinical need for" +// or "an ongoing need for". A required-versus-optional question keeps them all. const requirementVerbs = - /\b(? (/\boptional\b/i.test(request) ? word : " ")); const normalized = normalizedClinicalSearchTokens(semanticQuery).map(normalizeTerm).filter(Boolean); const raw = (semanticQuery.toLowerCase().match(/[a-z0-9]+/g) ?? []).map(normalizeTerm).filter(Boolean); const candidates = uniq([...normalized, ...raw], 14); diff --git a/tests/evidence-relevance.test.ts b/tests/evidence-relevance.test.ts index ddc7daaba2..ddf1429e3e 100644 --- a/tests/evidence-relevance.test.ts +++ b/tests/evidence-relevance.test.ts @@ -54,6 +54,9 @@ describe("evidence relevance", () => { ]); expect(queryCoreTerms("What is the clinical need for lithium?")).toContain("need"); expect(queryCoreTerms("Is there a need for clozapine monitoring?")).toContain("need"); + expect(queryCoreTerms("Is there an ongoing need for clozapine?")).toContain("need"); + expect(queryCoreTerms("Does the patient need clozapine monitoring?")).not.toContain("need"); + expect(queryCoreTerms("Which clozapine monitoring is required versus optional?")).toContain("required"); }); it("does not treat a lithium-only passage as direct evidence for the clinical need", () => {