From 98e7a98439a6123fc2dc4514333d7cc8e4d04653 Mon Sep 17 00:00:00 2001 From: Adam Date: Wed, 28 Oct 2026 20:05:47 +0000 Subject: [PATCH 1/3] Fix case generator issues found in case-quality batch 1 - Spoken text never mentions the research or missing data; gaps go to sourceQuery with needsSource. - Generate research prefers Hungarian or Spanish sources for cases in those languages, with international sources as the fallback. - R2 asks for 10-12 full "If they say" blocks; reinforce paragraphs are 100-120 words in both the role text and the schema; the R2 counter target is 10/12 and a reply up to three short is kept. - Each role states its word budget at 130 spoken words a minute. - The no-conceding, no-study-method and no-jargon rules apply to every speaker role. Co-Authored-By: Claude Opus 5.5 --- .../features/case-generation.md | 2 +- .../recipes/case-generation.spec.ts | 8 +- e2e/full-stack/fake-openrouter.mjs | 6 +- .../_shared/control-enforcement.test.ts | 36 +++++++- .../functions/_shared/control-enforcement.ts | 7 +- .../functions/_shared/prompts/generate.ts | 76 ++++++++++++----- .../_shared/prompts/prompt-examples.test.ts | 2 +- supabase/functions/_shared/research.test.ts | 50 +++++++++++ supabase/functions/_shared/research.ts | 25 ++++-- supabase/functions/_shared/source-quality.ts | 85 ++++++++++++++++++- supabase/functions/ai/control-matrix.test.ts | 4 +- supabase/functions/ai/features/generate.ts | 5 ++ supabase/functions/ai/prompt-builder.test.ts | 9 +- supabase/functions/ai/source-quality.test.ts | 39 ++++++++- 14 files changed, 306 insertions(+), 48 deletions(-) diff --git a/.claude/skills/verify-debatelab/features/case-generation.md b/.claude/skills/verify-debatelab/features/case-generation.md index 3ba4b2aa5..54083f3a6 100644 --- a/.claude/skills/verify-debatelab/features/case-generation.md +++ b/.claude/skills/verify-debatelab/features/case-generation.md @@ -10,7 +10,7 @@ A signed-in user enters a motion on Generate and receives a structured debate ca - `gen-quota` shows the dialog `Daily AI uses used up` when the account has no quota, without calling the provider. - `gen-save` saves a one-off case (the motion dialog answered with `Not now, use it once`) under a title into `cases` with `debate_id: null`. It appears in History under `One-off and unsorted work`, including from a fresh browser context. - `gen-regenerate` regenerates one section. Only that section shows the overlay `Regenerating
…`; the rest of the case and the result toolbar stay on screen, and every regenerate button and the form's submit button are disabled until it finishes. -- `gen-roles` shows the role's shape: on `/app/generate` pick the `Speaker Role` option `Introductor (4 min)` and the Opening card holds a `What we mean:` line; pick `R2 - Rebuttal/Build (6 min)` and there is no `Arguments` section, `Backing up our case` holds the team's earlier arguments, and `If they say…` holds six full answers covering their main arguments, supporting points and answers to our case (standard length); pick `Conclusor (3 min)` and `Why we win` replaces `Arguments`. +- `gen-roles` shows the role's shape: on `/app/generate` pick the `Speaker Role` option `Introductor (4 min)` and the Opening card holds a `What we mean:` line; pick `R2 - Rebuttal/Build (6 min)` and there is no `Arguments` section, `Backing up our case` holds the team's earlier arguments, and `If they say…` holds twelve full answers covering their main arguments, supporting points and answers to our case (standard length); pick `Conclusor (3 min)` and `Why we win` replaces `Arguments`. - `gen-export` downloads a `.pdf` (starting with `%PDF`) and a `.docx` (starting with `PK`). - `gen-flowchart` builds a flowchart from the Current Debate's case. `/app/flowchart` opens the template; the button `Build from current case` replaces it with the thesis, one node per argument (title, point and evidence, or a Conclusor's reasons why we win), what the other side will say, and our answers, fitted to the canvas. Node titles the app wrote (`Thesis`, `Argument 1`) follow the UI language. diff --git a/.claude/skills/verify-debatelab/recipes/case-generation.spec.ts b/.claude/skills/verify-debatelab/recipes/case-generation.spec.ts index 5020e69c3..16f74eaf4 100644 --- a/.claude/skills/verify-debatelab/recipes/case-generation.spec.ts +++ b/.claude/skills/verify-debatelab/recipes/case-generation.spec.ts @@ -194,12 +194,12 @@ test("gen-roles: the speaker role shapes the case: Introductor, R2 and Conclusor await expect(page.getByRole("button", { name: "Regenerate If they say…" })).toBeVisible(); await expect(page.getByText("They say", { exact: true })).toHaveCount(r2.counters.length); // One full answer for each thing they are likely to say: standard length - // asks for six, and the Edge Function keeps a live reply with at least four. + // asks for twelve, and the Edge Function keeps a live reply with at least nine. if (LIVE) { - expect(r2.counters.length).toBeGreaterThanOrEqual(4); - expect(r2.counters.length).toBeLessThanOrEqual(6); + expect(r2.counters.length).toBeGreaterThanOrEqual(9); + expect(r2.counters.length).toBeLessThanOrEqual(12); } else { - expect(r2.counters).toHaveLength(6); + expect(r2.counters).toHaveLength(12); } await evidence.shot("02-r2"); await evidence.aria("02-r2"); diff --git a/e2e/full-stack/fake-openrouter.mjs b/e2e/full-stack/fake-openrouter.mjs index 5be51c986..26c155ec6 100644 --- a/e2e/full-stack/fake-openrouter.mjs +++ b/e2e/full-stack/fake-openrouter.mjs @@ -70,6 +70,10 @@ const counter = (index) => ({ "Cities cannot afford the extra buses.", "Fares already have discounts for people who need them.", "Free transport has been tried and dropped elsewhere.", + "Better buses would help more than free buses.", + "Fare-free cities still have empty buses at night.", + "Taxpayers who never ride would pay for everyone else.", + "Free fares crowd out walking and cycling.", ][index], counterpoint: `${MARK} answer ${index + 1}: fare collection itself costs a large share of what it brings in, so the real gap is far smaller.`, ...(index % 2 === 0 ? { source: "UITP, 2024" } : {}), @@ -93,7 +97,7 @@ const generate = (user) => { const conc = role === "conc"; const r2 = role === "r2"; // R2: one full answer for each thing they are likely to say. - const answers = length === "brief" ? 5 : 6; + const answers = length === "brief" ? 10 : 12; return { opening: { hook: `${MARK} opening hook: picture a nurse who skips the night bus because the fare is a tenth of her shift's pay.`, diff --git a/supabase/functions/_shared/control-enforcement.test.ts b/supabase/functions/_shared/control-enforcement.test.ts index 49dc207cb..517735330 100644 --- a/supabase/functions/_shared/control-enforcement.test.ts +++ b/supabase/functions/_shared/control-enforcement.test.ts @@ -239,10 +239,11 @@ Deno.test("a Conclusor case without at least two reasons we win is rejected, not }); Deno.test("an R2 case must back up the team's arguments and answer theirs, or it is rejected", () => { + const answers = (count: number) => Array.from({ length: count }, () => ({})); const cases: Array<[string, Record]> = [ - ["no reinforce", { reinforce: [], counters: [{}, {}, {}, {}, {}], contentions: [], metadata: {} }], - ["only blank reinforce", { reinforce: [{ argument: "Our first", text: " " }], counters: [{}, {}, {}, {}, {}], contentions: [], metadata: {} }], - ["too few answers", { reinforce: [{ argument: "Our first", text: "Backed up." }], counters: [{}, {}, {}], contentions: [], metadata: {} }], + ["no reinforce", { reinforce: [], counters: answers(12), contentions: [], metadata: {} }], + ["only blank reinforce", { reinforce: [{ argument: "Our first", text: " " }], counters: answers(12), contentions: [], metadata: {} }], + ["too few answers", { reinforce: [{ argument: "Our first", text: "Backed up." }], counters: answers(8), contentions: [], metadata: {} }], ]; for (const [label, output] of cases) { let rejected = false; @@ -257,7 +258,7 @@ Deno.test("an R2 case must back up the team's arguments and answer theirs, or it // New arguments from the model are dropped: R2 brings none. const output: Record = { reinforce: [{ argument: "Our first", text: "Backed up." }], - counters: [{}, {}, {}, {}, {}], + counters: answers(12), contentions: [{ claim: "A new point", whyItMatters: "x" }], metadata: {}, }; @@ -265,6 +266,33 @@ Deno.test("an R2 case must back up the team's arguments and answer theirs, or it if ((output.contentions as unknown[]).length !== 0) throw new Error("R2 must not keep new arguments"); }); +Deno.test("an R2 case a few answers short of its target is kept, one well short is rejected", () => { + const run = (length: string, count: number) => { + const output: Record = { + reinforce: [{ argument: "Our first", text: "Backed up." }], + counters: Array.from({ length: count }, (_, index) => ({ targetArgument: `claim ${index}` })), + contentions: [], + metadata: {}, + }; + enforceGenerateControls(output, "con", "custom", length, "r2"); + return (output.counters as unknown[]).length; + }; + const rejects = (length: string, count: number) => { + try { + run(length, count); + return false; + } catch (error) { + return error instanceof Error && error.message.includes(`R2 needs ${length === "brief" ? 10 : 12}`); + } + }; + // Standard asks for 12 and keeps a reply of 9; brief asks for 10 and keeps 7. + if (run("standard", 9) !== 9 || run("brief", 7) !== 7) throw new Error("a slightly short R2 reply must be kept as it is"); + if (run("standard", 15) !== 12 || run("brief", 15) !== 10) throw new Error("extra R2 answers must be trimmed to the target"); + if (!rejects("standard", 8) || !rejects("brief", 6) || !rejects("comprehensive", 8)) { + throw new Error("an R2 reply well short of its target must be rejected"); + } +}); + Deno.test("why it matters is filed under the key saved cases already use", () => { const output: Record = { contentions: [{ claim: "a", whyItMatters: "People reach work." }, { claim: "b", impact: "Kept.", whyItMatters: "Ignored." }], diff --git a/supabase/functions/_shared/control-enforcement.ts b/supabase/functions/_shared/control-enforcement.ts index c0380d2df..c34bf4b77 100644 --- a/supabase/functions/_shared/control-enforcement.ts +++ b/supabase/functions/_shared/control-enforcement.ts @@ -192,12 +192,13 @@ export function enforceGenerateControls(output: unknown, side: string, format: s : []; const answers = Array.isArray(generateOutput.counters) ? generateOutput.counters : []; // One full answer for each thing they are likely to say: their main - // arguments, supporting points and answers to our case. - const target = length === "brief" ? 5 : 6; + // arguments, supporting points and answers to our case. A reply a few + // short of the target is still a usable speech, so only a thin one fails. + const target = length === "brief" ? 10 : 12; if (reinforce.length < 1) { throw new Error("Generate response returned no reinforce items, but R2 must back up the team's arguments."); } - if (answers.length < 4) { + if (answers.length < target - 3) { throw new Error(`Generate response returned ${answers.length} counters, but R2 needs ${target}.`); } generateOutput.reinforce = reinforce.slice(0, 3); diff --git a/supabase/functions/_shared/prompts/generate.ts b/supabase/functions/_shared/prompts/generate.ts index 4289c782c..3a2c14fcd 100644 --- a/supabase/functions/_shared/prompts/generate.ts +++ b/supabase/functions/_shared/prompts/generate.ts @@ -5,7 +5,9 @@ import { asString, outputLanguageInstruction } from "./common.ts"; export const GENERATE_PROMPT = `You are an experienced competitive debater and coach with years of tournament experience. Generate a tournament-ready debate case in JSON format. [DATE_CONTEXT] -If WEB RESEARCH RESULTS are attached below, treat them as the primary source for current real-world data, statistics, and sources. Prefer evidence from the last two years before the current date. If no research is attached, do not claim that current sources were checked and mark evidence gaps honestly. +If WEB RESEARCH RESULTS are attached below, treat them as the primary source for current real-world data, statistics, and sources. Prefer sources from the last five years before the current date; use an older one only when nothing recent covers the point. When the case is in Hungarian or Spanish, prefer sources from Hungary or Spain when the research has them (statistics offices, ministries, universities, national press), and use international sources for what they do not cover. If no research is attached, do not claim that current sources were checked: set needsSource to true where evidence is missing. + +The research is for you, the writer, not for the audience. Everything the speaker says out loud must sound like a debater who knows the facts, so no spoken field may mention the research, "our research", "the sources", what the research or a source does or does not cover, missing data, missing evidence, an undated source, or that a number is unknown. When there is no number for a point, make the point with reasoning or a concrete example and say nothing about the missing number. A note on missing evidence belongs only in sourceQuery with needsSource set to true. This overrides any line in the research block that asks you to state that evidence is limited. === THE SPEECH STRUCTURE === @@ -18,24 +20,36 @@ Write the case the way school debaters actually prepare it, in plain words: Do not use debate theory terms anywhere in the output: no technical labels for parts of a case, no talk of "links", "brightlines", "uniqueness" or "turns", and no scoring of size, likelihood or speed. Write everything in the everyday words a 14-year-old uses in class. +=== SPOKEN TEXT RULES (every speaker) === + +These rules hold for every role, in every field the speaker says out loud: the opening, the thesis line, the arguments, the "If they say..." answers, the backing-up paragraphs, the reasons why we win and the closing line. +- Never concede. No "We accept...", "That is a fair worry", "we agree", "we do not deny", "that cost is real", "It is true that..." or any other line that grants the other side's point. Push back straight away and say why their point does not hold. +- Never describe how a study was done. No "cross-sectional", "meta-analysis", "longitudinal", "systematic review", "observational", "randomised", "correlational", "sample size" or "peer-reviewed". Say who found what and how many people it covered: "a study of 10,000 pupils found...", "researchers who followed 2,000 teenagers for five years found...". +- No debate jargon in a sentence: no "contentions", "weighing", "warrant", "framework", "impact calculus", "burden of proof" or "clash". Say "our arguments", "why ours matters more", "the reason", "how we see it". These words may be JSON keys, never words the speaker says. +- Never mention the research or what it does not cover (see above). + +=== SPEECH LENGTH === + +A speaker says about 130 words a minute, so every role has a word budget, given in the Speech Budget line of the request. Keep what the speaker reads out within it. The "If they say..." answers are a list the speaker picks from: only the ones the other side actually used are spoken, so they sit outside the budget unless the Speech Budget line says otherwise. + === THE FOUR SPEAKERS (4-6-6-3) === This is a 4-speaker team format. The role decides what the speech contains: ---- INTRODUCTOR (4 minutes) --- -Opens the team's case. A hook (a vivid scenario, a question, or a fact from the research), then "we will show N arguments that prove..." and the arguments in short form: title, point, and one piece of evidence each. If, and only if, a word in the motion is genuinely unclear (for example "AI tools" or "minors"), add ONE plain line saying how the team understands it in opening.definition. Otherwise leave opening.definition out. Never write a list of definitions. +--- INTRODUCTOR (4 minutes, about 520 words) --- +Opens the team's case, in short form only. A hook (a vivid scenario, a question, or a fact from the research), then "we will show N arguments that prove..." and the arguments in short form: title, a one-sentence point, a short explanation of two or three sentences, and one piece of evidence each. No long walk-throughs: R1 builds the arguments in full. If, and only if, a word in the motion is genuinely unclear (for example "AI tools" or "minors"), add ONE plain line saying how the team understands it in opening.definition. Otherwise leave opening.definition out. Never write a list of definitions. The Introductor's "If they say..." answers are a short prep list outside the four minutes: 3-5 answers of 40-70 words each. ---- R1 - First Speaker (6 minutes) --- -Builds the team's case in full. A short opening line, then the arguments in depth: each explanation walks through why it is true, step by step, and each piece of evidence names its source and year. Then the "If they say..." answers to the strongest opposing claims. +--- R1 - First Speaker (6 minutes, about 780 words) --- +Builds the team's case in full. A short opening line, then the arguments in depth: each explanation walks through why it is true, step by step, and each piece of evidence names its source and year. The opening, the arguments and the closing line together stay within about 600 words, which leaves about a minute and a half for the "If they say..." answers the speaker picks. Then the "If they say..." answers to the strongest opposing claims: 5-8 answers of 60-100 words each. ---- R2 - Second Speaker (6 minutes) --- +--- R2 - Second Speaker (6 minutes, about 780 words) --- R2 brings NO new arguments: leave contentions as an empty array. The speech has four parts, in this order: 1. Opening (a few sentences). A hook that turns the motion around ("Schools present themselves as safe, but if that were true, thousands of children would not be suffering in silence every day."), then who is speaking and what they will prove: "I'm the R2 of my team, and I'm here to show that...". Put the hook in opening.hook and the "I'm the R2..." line in caseStructure.thesis. -2. Backing up our case (about 2 minutes, write it in reinforce). Go back to the arguments the Introductor and R1 already made and make them stronger. Each item names the earlier argument it returns to ("Our second argument was about bullying.") and then, in one spoken paragraph of about 100-120 words (keep to it, even though the answers in part 3 are much longer): the strongest fact for it with its source and year ("In Spain, 1 in 5 pupils has been bullied (Educo, 2022)"), what that means in plain words, one more fact or study if the research has it, and a sentence that ties it back to the motion ("If schools were really safe, these numbers would not look like this."). Never introduce a point the team has not made. If CURRENT DEBATE CONTEXT includes the team's earlier case, back up those arguments by name; otherwise back up the strongest arguments for our side as "our first argument" and "our second argument". -3. If they say... (about 4 minutes, the bulk of the speech, written in counters). Prepare one block for each thing the other side is likely to say: their 2-3 main arguments, then the supporting points they will use to back them up, and the answers they will give to our arguments. The speaker underlines the blocks the other side actually used, then speaks them and builds on them, so each block must be complete enough to say in full. Each item: +2. Backing up our case (about 2 minutes, write it in reinforce). Go back to the arguments the Introductor and R1 already made and make them stronger. Each item names the earlier argument it returns to ("Our second argument was about bullying.") and then, in one spoken paragraph of about 100-120 words (keep to it, even though the answers in part 3 are longer): the strongest fact for it with its source and year ("In Spain, 1 in 5 pupils has been bullied (Educo, 2022)"), what that means in plain words, one more fact or study if the research has it, and a sentence that ties it back to the motion ("If schools were really safe, these numbers would not look like this."). Never introduce a point the team has not made. If CURRENT DEBATE CONTEXT includes the team's earlier case, back up those arguments by name; otherwise back up the strongest arguments for our side as "our first argument" and "our second argument". +3. If they say... (about 4 minutes, the bulk of the speech, written in counters). Prepare 10-12 blocks, one for each thing the other side is likely to say: first their 2-3 main arguments, then the supporting points they will use to back each of them up, then the answers they will give to each of our arguments. Each block covers a different claim. The speaker underlines the blocks the other side actually used, then speaks them and builds on them, so each block must be complete enough to say in full, word for word. Each item: - targetArgument: what they say, in one sentence, the way they would put it ("Teachers and the school solve almost all bullying."). - counterpoint: our full spoken answer, built like this: - a) Push back in the first sentence, never by agreeing or conceding ("We accept...", "That is a fair worry", "we agree" are not allowed): say straight away why their point does not hold ("That is what the adults see, but more than 60% of pupils say their school does nothing."). + a) Push back in the first sentence, never by conceding: say straight away why their point does not hold ("That is what the adults see, but more than 60% of pupils say their school does nothing."). b) Give at least two specific facts against it, each with a number where the research has one, naming the source and year in the sentence ("1 in 3 victims never tells anyone (UNESCO, 2021)"). c) Say in plain words what those facts mean for real people ("There is a gap between what teachers say and what pupils live."). d) Ask a pointed question that turns their point around ("If the rules worked, why does 1 in 5 pupils still suffer?"). @@ -43,9 +57,9 @@ R2 brings NO new arguments: leave contentions as an empty array. The speech has - source: the source of the main fact as "Organisation, year". Every number and fact must come from the attached research or Library. Never invent or estimate one: if the research has no number for a point, make the point with reasoning instead. Every source an answer names must also appear in the sources list. Answer the argument, not the person. 4. Closing line: one sentence that sums up and ends the speech, like "That is why it is clear that schools are NOT truly safe. Thank you." -Keep the sentences short and direct, the way a confident student speaks in class: say "we", never hedge ("This is a fair question", "we do not pretend"), and never describe a study's method. Not every "If they say..." block is spoken, so the written answers can run longer than the four minutes. +Keep the sentences short and direct, the way a confident student speaks in class: say "we", never hedge ("This is a fair question", "we do not pretend"). The opening, the backing-up paragraphs and the closing line together stay within about 300 words. Not every "If they say..." block is spoken, so all the blocks together can run longer than the four minutes. ---- CONCLUSOR (3 minutes) --- +--- CONCLUSOR (3 minutes, about 390 words) --- Closes the debate. NO new arguments: leave contentions and counters as empty arrays. Instead write whyWeWin: 2-3 short paragraphs, each summarising one reason our side won the debate, drawing on what the team already argued. Then the closing line. The opening is one sentence that sets up the summary. === WRITING STYLE, Sound like a REAL debater === @@ -97,7 +111,7 @@ DO NOT: } ], "reinforce": [ - { "argument": "R2 only: the team's earlier argument this returns to, e.g. 'Our second argument: bullying'", "text": "R2 only: one spoken paragraph of about 120-150 words backing it up. Omit for other roles." } + { "argument": "R2 only: the team's earlier argument this returns to, e.g. 'Our second argument: bullying'", "text": "R2 only: one spoken paragraph of about 100-120 words backing it up. Omit for other roles." } ], "whyWeWin": ["Conclusor only: 2-3 short summaries of why our side won. Omit for other roles."], "closingLine": "One sentence that ties back to the opening.", @@ -116,7 +130,7 @@ CONTROL PRECEDENCE: The Length Instruction sets the EXACT number of items it nam SPEECH-READY ARGUMENTS: Each contention must be something the requested speaker can deliver in round: - claim: one sentence a judge can write down. - reasoning: spell out every step from how things are now to the point, so a listener can see why it is true. Never state a conclusion as its own reason. -- evidence: say how a specific research or Library source proves the point, naming the source and year in the sentence. When no attached source supports it, describe the evidence still needed and set needsSource to true. +- evidence: say how a specific research or Library source proves the point, naming the source and year in the sentence. When no attached source supports it, write the strongest real-world example or reasoning the speaker can say instead, set needsSource to true, and put the evidence still needed in sourceQuery. The evidence field is spoken, so it never says that evidence is missing. - whyItMatters: why it matters to real people, compared with what the other side is likely to say, in plain spoken sentences. SOURCES LIST: List each source the case actually cites once. Copy the link exactly as it appears in the attached research or Library; never invent or guess a link, and leave "url" out when you have none. When no research is attached, the list may be empty. @@ -126,12 +140,35 @@ TEAM CONSISTENCY: If CURRENT DEBATE CONTEXT includes an earlier case or brainsto REQUIREMENTS: 1. Side must be exactly "pro" or "con". 2. Generate contentions appropriate to the role (see above). -3. Introductor and R1: include 5-8 counters, one per likely opposing claim. R2: the exact number of counters the Length Instruction gives, covering their main arguments, supporting points and answers to our case. Conclusor: none. +3. Introductor: include 3-5 short counters. R1: include 5-8 counters, one per likely opposing claim. R2: the exact number of counters the Length Instruction gives, covering their main arguments, supporting points and answers to our case. Conclusor: none. 4. Use attached WEB RESEARCH when present. Cite only sources that appear in it. 5. Set needsSource to false only when the attached research supports the contention. Set it to true when the contention still needs evidence. 6. Write the ENTIRE output in the language specified by the user. All arguments, evidence, descriptions, and analysis must be in that language. 7. Every field should read like something a debater can actually say in a speech. Do not output table-like notes or field labels inside prose.`; +/** Speaking time per role in the 4-6-6-3 format. */ +export const SPEECH_MINUTES: Record = { intro: 4, r1: 6, r2: 6, conc: 3 }; +/** A student speaks about this many words a minute in a round. */ +export const SPOKEN_WORDS_PER_MINUTE = 130; + +/** The word budget the speaker reads out, stated in the request so speeches fit their time. */ +export function speechBudgetInstruction(role: string): string | undefined { + const minutes = SPEECH_MINUTES[role]; + if (!minutes) return undefined; + const words = minutes * SPOKEN_WORDS_PER_MINUTE; + const head = `${minutes} minutes, about ${words} spoken words at ${SPOKEN_WORDS_PER_MINUTE} words a minute.`; + switch (role) { + case "intro": + return `${head} The opening, the "we will show" line, the arguments in short form and the closing line together stay within ${words} words. The "If they say..." answers are a separate short prep list.`; + case "r1": + return `${head} The opening, the arguments and the closing line together stay within about ${words - 180} words, leaving time for the "If they say..." answers the speaker picks.`; + case "r2": + return `${head} The opening, the backing-up paragraphs and the closing line together stay within about 300 words (about 2 minutes); the rest of the time is for the "If they say..." blocks the other side actually used.`; + default: + return `${head} Everything in the case is spoken, so the whole case stays within ${words} words.`; + } +} + export function buildGenerateUserPrompt(input: Record, searchContext: string): string { const language = asString(input.language, "en"); const length = asString(input.length, "standard"); @@ -140,10 +177,10 @@ export function buildGenerateUserPrompt(input: Record, searchCo const role = asString(input.role, "intro"); const lengthInstruction = role === "r2" ? ({ - brief: "Return no contentions. Return exactly 2 reinforce items and exactly 5 counters, each answer about 140-170 words.", - standard: "Return no contentions. Return exactly 2 reinforce items and exactly 6 counters, each answer about 170-220 words.", - comprehensive: "Return no contentions. Return exactly 2 reinforce items and exactly 6 counters, each answer about 220-270 words.", - }[length] || "Return no contentions. Return exactly 2 reinforce items and exactly 6 counters, each answer about 170-220 words.") + brief: "Return no contentions. Return exactly 2 reinforce items and exactly 10 counters, each answer about 100-130 words.", + standard: "Return no contentions. Return exactly 2 reinforce items and exactly 12 counters, each answer about 120-150 words.", + comprehensive: "Return no contentions. Return exactly 2 reinforce items and exactly 12 counters, each answer about 150-180 words.", + }[length] || "Return no contentions. Return exactly 2 reinforce items and exactly 12 counters, each answer about 120-150 words.") : role === "conc" ? ({ brief: "Return no contentions and no counters. Return exactly 2 whyWeWin items, each two or three sentences.", @@ -165,13 +202,14 @@ export function buildGenerateUserPrompt(input: Record, searchCo conversational: "Use natural spoken-round phrasing without slang or filler.", aggressive: "Use sharper strategic phrasing and clearer attack language while staying professional.", }[tone] || "Use polished, judge-facing language with controlled phrasing."; + const speechBudget = speechBudgetInstruction(role); return `Motion: ${asString(input.motion)} Side: ${asString(input.side, "pro")} Format: ${asString(input.format, "custom")} Role: ${role} Length: ${length} Length Instruction: ${lengthInstruction} -Audience: ${audience} +${speechBudget ? `Speech Budget: ${speechBudget}\n` : ""}Audience: ${audience} Audience Instruction: ${audienceInstruction} Tone: ${tone} Tone Instruction: ${toneInstruction} diff --git a/supabase/functions/_shared/prompts/prompt-examples.test.ts b/supabase/functions/_shared/prompts/prompt-examples.test.ts index b72e694e5..722aecb4a 100644 --- a/supabase/functions/_shared/prompts/prompt-examples.test.ts +++ b/supabase/functions/_shared/prompts/prompt-examples.test.ts @@ -10,7 +10,7 @@ function assert(condition: unknown, message: string): asserts condition { // The feature-specific quality rules each prompt must carry. const QUALITY_RULES: Record = { - generate: ["SPEECH-READY ARGUMENTS", "Do not use debate theory terms", "never invent or guess a link", "TEAM CONSISTENCY"], + generate: ["SPEECH-READY ARGUMENTS", "Do not use debate theory terms", "never invent or guess a link", "TEAM CONSISTENCY", "SPOKEN TEXT RULES (every speaker)", "no spoken field may mention the research", "about 130 words a minute"], attack: ["TARGET, FLAW, RESPONSE", "exact words from CASE TEXT", "word for word"], brainstorm: ["GENUINELY DIFFERENT DIRECTIONS", "same response are one idea"], factoids: ["CUTTABLE AND HONEST", "limitations: never empty", "Never round, combine, or extrapolate"], diff --git a/supabase/functions/_shared/research.test.ts b/supabase/functions/_shared/research.test.ts index 71dea5a48..fd43b2db4 100644 --- a/supabase/functions/_shared/research.test.ts +++ b/supabase/functions/_shared/research.test.ts @@ -11,6 +11,7 @@ import { MAX_PROVIDER_ERROR_BODY_CHARS, MAX_PROVIDER_RESPONSE_BYTES, type ResearchContext, + type ResearchOptions, } from "./research.ts"; import type { NormalizedUsage } from "./cost.ts"; import { MAX_FETCHED_PAGE_CHARS } from "./prompt-budget.ts"; @@ -629,3 +630,52 @@ Deno.test("research preserves queries for search while keeping them out of logs" if (originalKey === undefined) Deno.env.delete("EXA_API_KEY"); else Deno.env.set("EXA_API_KEY", originalKey); } }); + +Deno.test("research in Hungarian searches Hungarian sources first, asks for them in follow-ups and reads them first", async () => { + const originalFetch = globalThis.fetch; + const originalKey = Deno.env.get("EXA_API_KEY"); + Deno.env.set("EXA_API_KEY", "test-key"); + const content = "Official statistics report on homework time in primary schools, with survey data from 2025 and detailed methodology for each school year."; + const run = async (options: Partial) => { + const searches: Array> = []; + const reads: string[] = []; + const prompts: string[] = []; + globalThis.fetch = (async (input: Request | URL | string, init?: RequestInit) => { + const body = JSON.parse(String(init?.body)) as Record; + if (String(input).endsWith("/contents")) { + reads.push((body.urls as string[])[0]); + return Response.json({ results: [{ text: "page" }] }); + } + searches.push(body); + return Response.json({ + results: [ + { title: "Homework report", url: "https://example.org/homework-report", text: content, publishedDate: "2025-03-01" }, + { title: "Homework report", url: "https://www.ksh.hu/homework-report", text: content, publishedDate: "2025-03-01" }, + ], + }); + }) as typeof fetch; + await deepResearch("homework primary schools", "", { + provider: "openrouter", model: "test-model", reasoningEffort: "low", + callModel: ({ messages }) => { + prompts.push(messages[0].content); + return Promise.resolve({ output: { queries: [] } }); + }, + }, { maxSeedQueries: 4, maxFollowUpQueries: 1, maxUrlsToRead: 1, overallTimeoutMs: 1000, ...options }); + return { searches, reads, prompts }; + }; + try { + const hungarian = await run({ language: "hu", country: "HU" }); + if (!(hungarian.searches[0].includeDomains as string[] | undefined)?.includes("ksh.hu")) throw new Error("the first search must target Hungarian sources"); + if (!hungarian.prompts[0]?.includes("KSH")) throw new Error("follow-up queries must be asked to find Hungarian sources"); + if (hungarian.reads[0] !== "https://www.ksh.hu/homework-report") throw new Error(`the Hungarian page must be read first, got ${hungarian.reads[0]}`); + + // The old call shape and an English request search and read exactly as before. + const before = await run({}); + const english = await run({ language: "en" }); + if (JSON.stringify(english) !== JSON.stringify(before)) throw new Error("an English request must not change research"); + if (before.prompts[0]?.includes("KSH")) throw new Error("no national hint without a locale"); + } finally { + globalThis.fetch = originalFetch; + if (originalKey === undefined) Deno.env.delete("EXA_API_KEY"); else Deno.env.set("EXA_API_KEY", originalKey); + } +}); diff --git a/supabase/functions/_shared/research.ts b/supabase/functions/_shared/research.ts index a2dc1e6dc..78259bc55 100644 --- a/supabase/functions/_shared/research.ts +++ b/supabase/functions/_shared/research.ts @@ -7,11 +7,13 @@ import { buildFollowUpSearchRequest, buildSearchRequests, formatRankedSearchResults, + nationalSourceHint, rankSearchResults, sourceTitleFromUrl, type ExaResult, type ExaSearchRequest, type RankedSearchResult, + type ResearchLocale, type ResearchSource, } from "./source-quality.ts"; @@ -50,6 +52,10 @@ export interface ResearchOptions { overallTimeoutMs: number; // Hard timeout for entire research preferredSourceType?: string; timeframe?: string; + /** Output language code: a Hungarian or Spanish request prefers that country's sources. */ + language?: string; + /** ISO country code whose sources to prefer; derived from language when absent. */ + country?: string; } export const DEEP_RESEARCH: ResearchOptions = { @@ -487,10 +493,12 @@ export async function decideAskSearch( async function generateFollowUpQueries( claim: string, currentResults: string, - deps: ResearchDeps + deps: ResearchDeps, + locale?: ResearchLocale, ): Promise { throwIfAborted(deps.signal); - const prompt = `Claim: "${claim}"\n\nCurrent search results:\n${currentResults.slice(0, 3000)}\n\nGenerate 2-3 specific follow-up queries to find more evidence.`; + const hint = nationalSourceHint(locale); + const prompt = `Claim: "${claim}"\n\nCurrent search results:\n${currentResults.slice(0, 3000)}\n\nGenerate 2-3 specific follow-up queries to find more evidence.${hint ? ` ${hint}` : ""}`; try { const result = await deps.callModel({ provider: deps.provider, @@ -525,7 +533,8 @@ async function deepResearchInner( console.log(`[RESEARCH] Starting (mode: ${options.maxFollowUpQueries > 0 ? "deep" : "light"}) claimLength=${claim.length}`); const preferredSourceType = options.preferredSourceType || "any"; - const seedRequests = buildSearchRequests(claim, region, preferredSourceType, options.timeframe).slice(0, options.maxSeedQueries); + const locale: ResearchLocale = { language: options.language, country: options.country }; + const seedRequests = buildSearchRequests(claim, region, preferredSourceType, options.timeframe, locale).slice(0, options.maxSeedQueries); const allResults: Array = []; const matchesPreferredSourceType = (result: RankedSearchResult) => ( preferredSourceType === "any" || @@ -547,7 +556,7 @@ async function deepResearchInner( const { query } = request; const results = await searchWithFallback(request); throwIfAborted(signal); - const ranked = rankSearchResults(results.map((result) => ({ ...result, query })), claim, preferredSourceType); + const ranked = rankSearchResults(results.map((result) => ({ ...result, query })), claim, preferredSourceType, locale); const visibleRanked = preferredSourceType === "any" ? ranked : ranked.filter(matchesPreferredSourceType); context.searches.push({ query, results: visibleRanked }); console.log(`[RESEARCH] Seed queryLength=${query.length}: ${results.length} results`); @@ -555,19 +564,19 @@ async function deepResearchInner( })); seedResults.forEach((results) => allResults.push(...results)); - const initialRanked = rankSearchResults(allResults, claim, preferredSourceType); + const initialRanked = rankSearchResults(allResults, claim, preferredSourceType, locale); const initialText = formatRankedSearchResults(initialRanked.slice(0, 12)); // Round 2: AI generates follow-up queries (skipped in light mode) if (options.maxFollowUpQueries > 0) { - const followUpQueries = await generateFollowUpQueries(claim, initialText, deps); + const followUpQueries = await generateFollowUpQueries(claim, initialText, deps, locale); throwIfAborted(signal); console.log(`[RESEARCH] Follow-up query count: ${followUpQueries.length}`); const followUpPromises = followUpQueries.slice(0, options.maxFollowUpQueries).map(async (query) => { const results = await searchWithFallback(buildFollowUpSearchRequest(query, preferredSourceType, options.timeframe)); throwIfAborted(signal); - const ranked = rankSearchResults(results.map((result) => ({ ...result, query })), claim, preferredSourceType); + const ranked = rankSearchResults(results.map((result) => ({ ...result, query })), claim, preferredSourceType, locale); const visibleRanked = preferredSourceType === "any" ? ranked : ranked.filter(matchesPreferredSourceType); context.searches.push({ query, results: visibleRanked }); allResults.push(...results.map((result) => ({ ...result, query }))); @@ -577,7 +586,7 @@ async function deepResearchInner( console.log(`[RESEARCH] Round 2 complete, ${context.searches.length} total searches`); } - context.rankedResults = rankSearchResults(allResults, claim, preferredSourceType); + context.rankedResults = rankSearchResults(allResults, claim, preferredSourceType, locale); const readCandidates = preferredSourceType === "any" ? context.rankedResults : context.rankedResults.filter(matchesPreferredSourceType); diff --git a/supabase/functions/_shared/source-quality.ts b/supabase/functions/_shared/source-quality.ts index 603e0c50d..2e65cb8fc 100644 --- a/supabase/functions/_shared/source-quality.ts +++ b/supabase/functions/_shared/source-quality.ts @@ -93,6 +93,68 @@ const LEGAL_PRIMARY_DOMAINS = [ "curia.europa.eu", ]; +/** The output language and country a research request should favour. */ +export type ResearchLocale = { + /** Output language code, e.g. "hu". */ + language?: string; + /** ISO 3166 country code, e.g. "HU". Derived from the language when absent. */ + country?: string; +}; + +type NationalSources = { + /** Words added to the national search, in the country's language. */ + searchTerms: string; + /** Statistics offices, ministries, universities and national press. */ + domains: string[]; + /** Country-code domain: any site under it counts as national when ranking. */ + tld: string; + /** Named in the follow-up query prompt. */ + description: string; +}; + +// Sources a debater in that country would quote. Research tries these first +// and falls back to international sources when they have nothing. +const NATIONAL_SOURCES: Record = { + HU: { + searchTerms: "Magyarország kutatás adatok felmérés", + domains: [ + "ksh.hu", "kormany.hu", "oktatas.hu", "parlament.hu", "njt.hu", "mnb.hu", "ajbh.hu", + "mta.hu", "mtak.hu", "elte.hu", "u-szeged.hu", "unideb.hu", "pte.hu", "uni-corvinus.hu", "bme.hu", + "hvg.hu", "telex.hu", "index.hu", "444.hu", "portfolio.hu", "eduline.hu", "magyarnemzet.hu", "hirado.hu", + ], + tld: "hu", + description: "Hungarian sources: KSH, Hungarian ministries and the Oktatási Hivatal, Hungarian universities and the Hungarian press", + }, + ES: { + searchTerms: "España estudio datos informe", + domains: [ + "ine.es", "gob.es", "boe.es", "cis.es", "bde.es", "csic.es", "defensordelpueblo.es", + "ucm.es", "uam.es", "uc3m.es", "ub.edu", "uv.es", "us.es", "unav.edu", + "elpais.com", "elmundo.es", "rtve.es", "abc.es", "lavanguardia.com", "eldiario.es", "europapress.es", "magisnet.com", + ], + tld: "es", + description: "Spanish sources: the INE, Spanish ministries, the BOE, Spanish universities and the Spanish press", + }, +}; + +const COUNTRY_FOR_LANGUAGE: Record = { hu: "HU", es: "ES" }; + +/** The country whose sources a case in this language should prefer, if any. */ +export function researchCountryForLanguage(language?: string): string | undefined { + return language ? COUNTRY_FOR_LANGUAGE[language.toLowerCase()] : undefined; +} + +function nationalSources(locale?: ResearchLocale): NationalSources | undefined { + const country = locale?.country?.toUpperCase() || researchCountryForLanguage(locale?.language); + return country ? NATIONAL_SOURCES[country] : undefined; +} + +/** A line for the follow-up query prompt naming the sources to look for, if any. */ +export function nationalSourceHint(locale?: ResearchLocale): string | undefined { + const national = nationalSources(locale); + return national && `Write at least one query in the language of the claim that finds ${national.description}. International sources are the fallback.`; +} + const RESULTS_PATTERNS = [ /\btab\b/i, /\btabs\b/i, @@ -198,11 +260,14 @@ function specificityScore(result: ExaResult, claim: string): number { return Math.min(22, Math.round((matches / Math.min(claimTokens.length, 8)) * 22)); } -function domainBonus(result: ExaResult, claim: string): number { +function domainBonus(result: ExaResult, claim: string, locale?: ResearchLocale): number { const url = result.url || ""; const host = hostFromUrl(url); const haystack = `${result.title || ""} ${url} ${result.content || ""}`; let score = 0; + const national = nationalSources(locale); + + if (national && (host.endsWith(`.${national.tld}`) || national.domains.some((domain) => hostMatchesDomain(host, domain)))) score += 12; if (WEAK_DOMAINS.some((domain) => hostMatchesDomain(host, domain))) score -= 30; if (/\b(tabbycat|calicotab|tabroom|debatekeeper)\b/i.test(haystack)) score += 18; @@ -219,6 +284,7 @@ export function rankSearchResults( results: Array, claim: string, preferredSourceType = "any", + locale?: ResearchLocale, ): RankedSearchResult[] { const seen = new Set(); @@ -234,7 +300,7 @@ export function rankSearchResults( typeScore(sourceType, preferredSourceType) + specificity + recency + - domainBonus(result, claim); + domainBonus(result, claim, locale); if (["official", "results", "government", "academic"].includes(sourceType)) qualityReasons.push(`${sourceType} source`); if (["encyclopedia", "video", "social"].includes(sourceType)) qualityReasons.push(`${sourceType} fallback`); @@ -308,7 +374,13 @@ export function buildFollowUpSearchRequest( return { query, ...filtersForSourceType(preferredSourceType, timeframe) }; } -export function buildSearchRequests(claim: string, region = "", preferredSourceType = "any", timeframe = ""): ExaSearchRequest[] { +export function buildSearchRequests( + claim: string, + region = "", + preferredSourceType = "any", + timeframe = "", + locale?: ResearchLocale, +): ExaSearchRequest[] { const base = `${claim} ${region}`.trim().replace(/\s+/g, " "); const years = timeframe || (/\b202[0-9]\b/.test(base) ? "" : "2025 2026"); const requests: ExaSearchRequest[] = []; @@ -317,6 +389,13 @@ export function buildSearchRequests(claim: string, region = "", preferredSourceT }; const publishedDateRange = publishedDateRangeFromTimeframe(timeframe); + // National sources first, so the seed-query limit never drops them; the + // queries below stay as the international fallback. + const national = nationalSources(locale); + if (national) { + add(`${base} ${national.searchTerms} ${years}`, { includeDomains: national.domains, ...publishedDateRange }); + } + // Topic-specific queries first (highest priority) if (OFFICIAL_DEBATE_PATTERNS.some((pattern) => pattern.test(base)) || /\bdebate|debating\b/i.test(base)) { add(`${base} official tournament results`, { includeDomains: DEBATE_RESULT_DOMAINS }); diff --git a/supabase/functions/ai/control-matrix.test.ts b/supabase/functions/ai/control-matrix.test.ts index 5bbc5aff4..1e60128e3 100644 --- a/supabase/functions/ai/control-matrix.test.ts +++ b/supabase/functions/ai/control-matrix.test.ts @@ -29,7 +29,7 @@ Deno.test("generate length and role controls preserve the output contract", () = exordio: { hook: "hook", transition: "transition" }, opening: { hook: "hook", definition: "definition" }, contentions: [{}, {}, {}, {}], - counters: Array.from({ length: 9 }, () => ({})), + counters: Array.from({ length: 14 }, () => ({})), reinforce: [{ argument: "Our first argument", text: "backed up" }, { argument: "", text: " " }], whyWeWin: ["a", "b", "c", "d"], metadata: {}, @@ -39,7 +39,7 @@ Deno.test("generate length and role controls preserve the output contract", () = // R2 brings no new arguments: it backs up the team's and answers theirs. assert((output.contentions as unknown[]).length === 0, `generate.${length}.r2 has no contentions`); assert((output.reinforce as unknown[]).length === 1, `generate.${length}.r2 keeps only reinforce items with text`); - assert((output.counters as unknown[]).length === (length === "brief" ? 5 : 6), `generate.${length}.r2 answer count`); + assert((output.counters as unknown[]).length === (length === "brief" ? 10 : 12), `generate.${length}.r2 answer count`); assert(!("whyWeWin" in output), "generate.r2 has no whyWeWin"); } else if (role === "conc") { // The Conclusor brings 2-3 reasons we win and no new arguments. diff --git a/supabase/functions/ai/features/generate.ts b/supabase/functions/ai/features/generate.ts index 19cd0a0d6..8a73614ef 100644 --- a/supabase/functions/ai/features/generate.ts +++ b/supabase/functions/ai/features/generate.ts @@ -12,6 +12,7 @@ import { getResearchSources, withResearchPreferences, } from "../../_shared/research.ts"; +import { researchCountryForLanguage } from "../../_shared/source-quality.ts"; import { enforceGenerateControls, enforceGenerateSources } from "../../_shared/control-enforcement.ts"; import { boundPromptSources } from "../../_shared/prompts.ts"; import { JEV_MODEL } from "../../_shared/jev.ts"; @@ -32,12 +33,16 @@ export async function handleGenerate(ctx: FeatureContext): Promise { const format = String(sanitizedInput.format || "custom"); const length = String(sanitizedInput.length || "standard"); const role = String(sanitizedInput.role || "intro"); + const language = String(sanitizedInput.language || "en"); // Perform deep research to find supporting evidence console.log("[GENERATE] Starting deep research for motion:", motion); ctx.stream?.phase("research"); + // A Hungarian or Spanish case looks for that country's sources first. const research = await deepResearch(motion, "", researchDeps, withResearchPreferences(DEEP_RESEARCH, { preferredSourceType: "any", + language, + country: researchCountryForLanguage(language), })); const discoveredSources = buildDiscoveredSources(research); const discoveredCandidates: SourceDecisionCandidate[] = discoveredSources.map((source) => ({ diff --git a/supabase/functions/ai/prompt-builder.test.ts b/supabase/functions/ai/prompt-builder.test.ts index 941b01e5d..f4fafc0cf 100644 --- a/supabase/functions/ai/prompt-builder.test.ts +++ b/supabase/functions/ai/prompt-builder.test.ts @@ -70,6 +70,8 @@ Deno.test("generate prompt matrix preserves every school role and length rule", ["comprehensive", "4"], ] as const; const roles = ["intro", "r1", "r2", "conc"] as const; + // 4-6-6-3 minutes at about 130 spoken words a minute. + const budgets = { intro: "4 minutes, about 520", r1: "6 minutes, about 780", r2: "6 minutes, about 780", conc: "3 minutes, about 390" }; for (const role of roles) { for (const [length, contentionCount] of lengths) { @@ -90,12 +92,17 @@ Deno.test("generate prompt matrix preserves every school role and length rule", ? `Length Instruction: Return no contentions and no counters. Return exactly ${length === "brief" ? 2 : 3} whyWeWin items` : role === "r2" // R2 brings no new arguments: one full answer for each thing they are likely to say. - ? `Length Instruction: Return no contentions. Return exactly 2 reinforce items and exactly ${length === "brief" ? 5 : 6} counters` + ? `Length Instruction: Return no contentions. Return exactly 2 reinforce items and exactly ${length === "brief" ? 10 : 12} counters` : `Length Instruction: Return exactly ${contentionCount} contentions`); + // Each speech states its word budget so it fits the speaker's time. + assertIncludes(prompt, `Speech Budget: ${budgets[role]} spoken words at 130 words a minute.`); assertIncludes(prompt, "Language: Hungarian (hu)"); assertIncludes(prompt, "RESEARCH CONTEXT"); } } + + const general = buildUserPrompt("generate", { motion: "MATRIX MOTION", role: "general" }, ""); + if (general.includes("Speech Budget:")) throw new Error("a case with no speaker role has no speech budget"); }); Deno.test("attack prompt includes intensity, paradigm, and toggle states", () => { diff --git a/supabase/functions/ai/source-quality.test.ts b/supabase/functions/ai/source-quality.test.ts index 50f0aae04..56d5ca93f 100644 --- a/supabase/functions/ai/source-quality.test.ts +++ b/supabase/functions/ai/source-quality.test.ts @@ -1,4 +1,4 @@ -import { buildSearchRequests, rankSearchResults } from "../_shared/source-quality.ts"; +import { buildSearchRequests, rankSearchResults, researchCountryForLanguage } from "../_shared/source-quality.ts"; function assert(condition: boolean, message: string) { if (!condition) throw new Error(message); @@ -102,3 +102,40 @@ Deno.test("intergovernmental domains satisfy the government source preference", }], "GDP statistics", "government"); assert(ranked[0]?.sourceType === "government", `Expected World Bank to classify as government, got ${ranked[0]?.sourceType}`); }); + +Deno.test("a Hungarian or Spanish request searches that country's sources first and keeps the international ones", () => { + const motion = "Ez a ház eltörölné a házi feladatot az általános iskolákban"; + const international = buildSearchRequests(motion); + const hungarian = buildSearchRequests(motion, "", "any", "", { language: "hu", country: "HU" }); + // First, so the seed-query limit (four) can never drop it. + assert(hungarian[0].includeDomains?.includes("ksh.hu") === true, "the first Hungarian search must target Hungarian sources"); + assert(hungarian[0].query.includes("Magyarország"), `expected a Hungarian search, got ${hungarian[0].query}`); + assert(JSON.stringify(hungarian.slice(1, 1 + 3)) === JSON.stringify(international.slice(0, 3)), "international fallback searches must follow unchanged"); + + const spanish = buildSearchRequests("Esta casa prohibiría los deberes", "", "any", "", { language: "es" }); + assert(spanish[0].includeDomains?.includes("ine.es") === true, "the language alone must pick Spain's sources"); + assert(researchCountryForLanguage("hu") === "HU" && researchCountryForLanguage("es") === "ES", "language to country mapping changed"); + + // English, an unknown country, or no locale at all: the requests are exactly as before. + const before = JSON.stringify(buildSearchRequests("school phone ban", "United States", "government", "2024-2026")); + for (const locale of [undefined, {}, { language: "en" }, { country: "ZZ" }]) { + assert(JSON.stringify(buildSearchRequests("school phone ban", "United States", "government", "2024-2026", locale)) === before, + `locale ${JSON.stringify(locale)} must not change the requests`); + } +}); + +Deno.test("a Hungarian request ranks Hungarian pages above equally good foreign ones", () => { + const content = "Official statistics report on homework time in primary schools, with survey data from 2025 and detailed methodology for each school year."; + const results = [ + { title: "Homework report", url: "https://example.org/homework-report", content, publishedDate: "2025-03-01" }, + { title: "Homework report", url: "https://www.ksh.hu/homework-report", content, publishedDate: "2025-03-01" }, + { title: "Homework report", url: "https://news.example.hu/homework-report", content, publishedDate: "2025-03-01" }, + ]; + const plain = rankSearchResults(results, "homework primary schools"); + const hungarian = rankSearchResults(results, "homework primary schools", "any", { language: "hu" }); + const score = (ranked: typeof plain, host: string) => ranked.find((item) => item.url?.includes(host))!.qualityScore; + assert(score(plain, "ksh.hu") === score(plain, "example.org"), "without a locale the pages must score the same"); + assert(score(hungarian, "ksh.hu") > score(hungarian, "example.org"), "a Hungarian statistics page must outrank a foreign one"); + assert(score(hungarian, "example.hu") > score(hungarian, "example.org"), "any .hu page must outrank a foreign one"); + assert(score(hungarian, "example.org") === score(plain, "example.org"), "foreign pages keep their score"); +}); From 83b5729d8cdde31a38c2512cb5dfca30021d4d07 Mon Sep 17 00:00:00 2001 From: Adam Date: Wed, 28 Oct 2026 20:26:38 +0000 Subject: [PATCH 2/3] Case prompt: keep the two-year recency preference, cover every evidence-limited rule in the no-research-talk override, give general-role cases R1 counters --- supabase/functions/_shared/prompts/generate.ts | 6 +++--- supabase/functions/_shared/prompts/prompt-examples.test.ts | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/supabase/functions/_shared/prompts/generate.ts b/supabase/functions/_shared/prompts/generate.ts index 3a2c14fcd..c815092a1 100644 --- a/supabase/functions/_shared/prompts/generate.ts +++ b/supabase/functions/_shared/prompts/generate.ts @@ -5,9 +5,9 @@ import { asString, outputLanguageInstruction } from "./common.ts"; export const GENERATE_PROMPT = `You are an experienced competitive debater and coach with years of tournament experience. Generate a tournament-ready debate case in JSON format. [DATE_CONTEXT] -If WEB RESEARCH RESULTS are attached below, treat them as the primary source for current real-world data, statistics, and sources. Prefer sources from the last five years before the current date; use an older one only when nothing recent covers the point. When the case is in Hungarian or Spanish, prefer sources from Hungary or Spain when the research has them (statistics offices, ministries, universities, national press), and use international sources for what they do not cover. If no research is attached, do not claim that current sources were checked: set needsSource to true where evidence is missing. +If WEB RESEARCH RESULTS are attached below, treat them as the primary source for current real-world data, statistics, and sources. Prefer sources from the last two years before the current date; use an older one only when nothing recent covers the point, and then the newest one that does. When the case is in Hungarian or Spanish, prefer sources from Hungary or Spain when the research has them (statistics offices, ministries, universities, national press), and use international sources for what they do not cover. If no research is attached, do not claim that current sources were checked: set needsSource to true where evidence is missing. -The research is for you, the writer, not for the audience. Everything the speaker says out loud must sound like a debater who knows the facts, so no spoken field may mention the research, "our research", "the sources", what the research or a source does or does not cover, missing data, missing evidence, an undated source, or that a number is unknown. When there is no number for a point, make the point with reasoning or a concrete example and say nothing about the missing number. A note on missing evidence belongs only in sourceQuery with needsSource set to true. This overrides any line in the research block that asks you to state that evidence is limited. +The research is for you, the writer, not for the audience. Everything the speaker says out loud must sound like a debater who knows the facts, so no spoken field may mention the research, "our research", "the sources", what the research or a source does or does not cover, missing data, missing evidence, an undated source, or that a number is unknown. When there is no number for a point, make the point with reasoning or a concrete example and say nothing about the missing number. A note on missing evidence belongs only in sourceQuery with needsSource set to true. This overrides every other line in this prompt that asks you to say evidence is unavailable, weak or limited, including the product identity, the research block and the research citation rules: in a case, that note goes only in sourceQuery. === THE SPEECH STRUCTURE === @@ -140,7 +140,7 @@ TEAM CONSISTENCY: If CURRENT DEBATE CONTEXT includes an earlier case or brainsto REQUIREMENTS: 1. Side must be exactly "pro" or "con". 2. Generate contentions appropriate to the role (see above). -3. Introductor: include 3-5 short counters. R1: include 5-8 counters, one per likely opposing claim. R2: the exact number of counters the Length Instruction gives, covering their main arguments, supporting points and answers to our case. Conclusor: none. +3. Introductor: include 3-5 short counters. R1, or a case with no speaker role: include 5-8 counters, one per likely opposing claim. R2: the exact number of counters the Length Instruction gives, covering their main arguments, supporting points and answers to our case. Conclusor: none. 4. Use attached WEB RESEARCH when present. Cite only sources that appear in it. 5. Set needsSource to false only when the attached research supports the contention. Set it to true when the contention still needs evidence. 6. Write the ENTIRE output in the language specified by the user. All arguments, evidence, descriptions, and analysis must be in that language. diff --git a/supabase/functions/_shared/prompts/prompt-examples.test.ts b/supabase/functions/_shared/prompts/prompt-examples.test.ts index 722aecb4a..9cdf1f4f7 100644 --- a/supabase/functions/_shared/prompts/prompt-examples.test.ts +++ b/supabase/functions/_shared/prompts/prompt-examples.test.ts @@ -10,7 +10,7 @@ function assert(condition: unknown, message: string): asserts condition { // The feature-specific quality rules each prompt must carry. const QUALITY_RULES: Record = { - generate: ["SPEECH-READY ARGUMENTS", "Do not use debate theory terms", "never invent or guess a link", "TEAM CONSISTENCY", "SPOKEN TEXT RULES (every speaker)", "no spoken field may mention the research", "about 130 words a minute"], + generate: ["SPEECH-READY ARGUMENTS", "Do not use debate theory terms", "never invent or guess a link", "TEAM CONSISTENCY", "SPOKEN TEXT RULES (every speaker)", "no spoken field may mention the research", "about 130 words a minute", "in a case, that note goes only in sourceQuery"], attack: ["TARGET, FLAW, RESPONSE", "exact words from CASE TEXT", "word for word"], brainstorm: ["GENUINELY DIFFERENT DIRECTIONS", "same response are one idea"], factoids: ["CUTTABLE AND HONEST", "limitations: never empty", "Never round, combine, or extrapolate"], From a771f86b09ecc687cd73cdb2f0e22ffdc986aaf1 Mon Sep 17 00:00:00 2001 From: Adam Date: Thu, 29 Oct 2026 19:35:02 +0000 Subject: [PATCH 3/3] Write R2 cases in parallel parts, and give each argument field a word target R2 cases came within 6 s of the 130 s request budget (124 s for a Hungarian case at medium effort). Both Supabase projects are on the Free plan, which stops a worker at 150 s, so the budget cannot grow. An R2 case is now planned at low effort (our two arguments and the claims to answer), then written in four parts at the same time at the student's model and effort. Part 1 writes the speech and streams to the browser. Each part has its own smaller cap, which keeps its reasoning budget at 8,000 tokens at medium. Measured: 68 s, 58 s and 53 s, down from 124, 109 and 105. A failed part stops the others and every call is billed. R1 speeches ran to 752 and 823 words against a 600-word budget. The Speech Budget line now gives each argument field a word target derived from the speaker's time and the number of arguments. Measured: 533 and 531. Co-Authored-By: Claude Opus 5.5 --- .../features/case-generation.md | 2 +- docs/architecture.md | 2 +- e2e/full-stack/fake-openrouter.mjs | 24 ++- supabase/functions/_shared/output-caps.ts | 13 ++ .../functions/_shared/prompts/generate.ts | 79 ++++++++- supabase/functions/ai/features/generate.ts | 7 +- supabase/functions/ai/r2-case.ts | 154 ++++++++++++++++++ supabase/functions/ai/r2-case_test.ts | 74 +++++++++ 8 files changed, 346 insertions(+), 9 deletions(-) create mode 100644 supabase/functions/ai/r2-case.ts create mode 100644 supabase/functions/ai/r2-case_test.ts diff --git a/.claude/skills/verify-debatelab/features/case-generation.md b/.claude/skills/verify-debatelab/features/case-generation.md index 54083f3a6..47317d94a 100644 --- a/.claude/skills/verify-debatelab/features/case-generation.md +++ b/.claude/skills/verify-debatelab/features/case-generation.md @@ -1,6 +1,6 @@ # Case generation -A signed-in user enters a motion on Generate and receives a structured debate case: an opening (hook and "we will show N arguments" line), arguments (title, point, explanation, evidence, why it matters), an "If they say…" list, a closing line and a sources list. The role shapes it: the Introductor may add one definition line; R2 brings no new arguments, only `Backing up our case` (the team's earlier arguments) and "If they say…" with one full answer for each thing the other side is likely to say; the Conclusor has 2-3 "Why we win" points instead of new arguments. Saved cases from before still show their old sections under plain names (`How to judge this debate`, `What we have to show`, `They may also say`) when they have content. +A signed-in user enters a motion on Generate and receives a structured debate case: an opening (hook and "we will show N arguments" line), arguments (title, point, explanation, evidence, why it matters), an "If they say…" list, a closing line and a sources list. The role shapes it: the Introductor may add one definition line; R2 brings no new arguments, only `Backing up our case` (the team's earlier arguments) and "If they say…" with one full answer for each thing the other side is likely to say; the Conclusor has 2-3 "Why we win" points instead of new arguments. An R2 case makes five provider calls, all `generate` in `providerCalls()`: a plan, then four parts written at the same time; the last one in the list still carries `Role: r2`. Saved cases from before still show their old sections under plain names (`How to judge this debate`, `What we have to show`, `They may also say`) when they have content. ## Sub-features diff --git a/docs/architecture.md b/docs/architecture.md index ac1691b8f..ff82628d8 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -82,7 +82,7 @@ Every structured answer is shown while the model writes it. A page passes a `liv A streamed run keeps its worker alive with `EdgeRuntime.waitUntil` ([`_shared/background.ts`](../supabase/functions/_shared/background.ts)) from the first event until the HTTP layer has read the last one. Supabase otherwise counts the response as sent once its headers are out and may retire the worker mid-run (`EarlyDrop`). Streamed Ask (`ai-stream`) is held the same way until the browser has read its final event. A run still unanswered about 142 s after its request arrived is aborted and gets a retryable timeout `result` instead of a connection cut at the 150 s wall clock; when the aborted run answers within 2 s, that answer is sent instead, so it says `charged: false` when the units went back. Storing the result and waiting for the browser to read it are cut short to finish before the wall clock. The request sends its id as `resultKey`. The function then writes a row to `ai_run_results` when the run starts, and stores the `result` there before sending it ([`ai/run-results.ts`](../supabase/functions/ai/run-results.ts); one row per user and feature, readable only by its owner). When the stream drops, stalls (no keepalive for 45 s), or delivers a result it cannot read, [`src/lib/ai/run-recovery.ts`](../src/lib/ai/run-recovery.ts) polls that row until the run's result appears, then resolves or fails exactly as if the event had arrived. A row still without a status long after it started is a run its worker never finished. A browser Stop, reload, or closed tab still cancels the run. The server cannot tell those from a lost connection, so it stops spending as soon as the connection goes. -A reply that hits its token cap is written again once with twice the cap, after a `reset`. The Generate cap (12,000 tokens) leaves room for a full comprehensive case on Claude Haiku, so a long case finishes on its first attempt. Claude rejects an assistant prefill, so a cut-off reply cannot be continued where it stopped. Sparring calls send a strict `json_schema` (`sparringOutputSchema` in `ai/sparring-persistence.ts`). Anthropic's structured outputs take no numeric or array-size limits, so the 0-25 range, the four categories, and the 12-item lists are enforced by the parser. When a reply parses but has the wrong shape, the log records only the failing field path, never the model's text. The call is then retried once with a repair prompt that names the field. A second failure answers `invalid_output` (502, logged as `invalid_output`), which the browser shows as a malformed answer. The judge also retries once when Claude is briefly unavailable or its stream breaks. The round has already paid, so retries cost the student nothing. +A reply that hits its token cap is written again once with twice the cap, after a `reset`. The Generate cap (12,000 tokens) leaves room for a full comprehensive case on Claude Haiku, so a long case finishes on its first attempt. Claude rejects an assistant prefill, so a cut-off reply cannot be continued where it stopped. An R2 case is the longest reply (the speech and 10-12 full answers), and one call wrote it in up to 124 s of the 130 s budget, which cannot grow because both Supabase projects are on the Free plan and stop a worker at 150 s. So [`ai/r2-case.ts`](../supabase/functions/ai/r2-case.ts) asks for a short plan first (our two arguments and the claims to answer, at low effort), then writes four parts at the same time at the student's model and effort: part 1 the speech and its share of the answers, streamed to the browser, the others only answers. Each part has its own smaller cap (`caseWriterPart`), which also keeps its reasoning budget small enough to finish well inside the budget. One failed part fails the case, stops the others, and bills every call. Sparring calls send a strict `json_schema` (`sparringOutputSchema` in `ai/sparring-persistence.ts`). Anthropic's structured outputs take no numeric or array-size limits, so the 0-25 range, the four categories, and the 12-item lists are enforced by the parser. When a reply parses but has the wrong shape, the log records only the failing field path, never the model's text. The call is then retried once with a repair prompt that names the field. A second failure answers `invalid_output` (502, logged as `invalid_output`), which the browser shows as a malformed answer. The judge also retries once when Claude is briefly unavailable or its stream breaks. The round has already paid, so retries cost the student nothing. For a validated Crossfire next turn, the server returns the reply first with a short-lived signed sidecar token. The client then requests a Jev classification of only the newest completed question-answer exchange. The token is atomically claimed through its own one-use counter, separate from the product's daily AI quota. The optional directness, concession, and follow-up signal is attached by turn ID to the answer; a missing, slow, replayed, forged, or failed sidecar never delays or invalidates the main reply. Draft version 1 remains compatible because turn signals are optional. diff --git a/e2e/full-stack/fake-openrouter.mjs b/e2e/full-stack/fake-openrouter.mjs index 26c155ec6..aae62c349 100644 --- a/e2e/full-stack/fake-openrouter.mjs +++ b/e2e/full-stack/fake-openrouter.mjs @@ -91,13 +91,33 @@ const r2Answer = (index) => ({ // theirs, and the Conclusor brings reasons why we win. Neither R2 nor the // Conclusor brings new arguments. const generate = (user) => { + // R2 is planned, then written in parts (supabase/functions/ai/r2-case.ts). + const planCount = /PLANNING STEP[\s\S]*?theirClaims: exactly (\d+)/.exec(user)?.[1]; + if (planCount) { + return { + ourArguments: [ + { title: "Fares shut people out", claim: "Fares keep the riders who need transit most off the bus." }, + { title: "More people ride", claim: "Free fares bring more passengers onto public transport." }, + ], + theirClaims: Array.from({ length: Number(planCount) }, (_, index) => `Free transit claim ${index + 1}: it costs the city too much.`), + }; + } + const part = /=== WRITING PART (\d+) OF \d+: claims (\d+)-(\d+) ===/.exec(user); + if (part && part[1] !== "1") { + const [first, last] = [Number(part[2]), Number(part[3])]; + return { + counters: Array.from({ length: last - first + 1 }, (_, index) => r2Answer(first - 1 + index)), + sources: [{ org: "UITP", year: "2024", claim: "Ridership rose after fares were removed", url: "https://www.uitp.org/free-fares-2024" }], + }; + } const length = /^Length: (\w+)/m.exec(user)?.[1] ?? "standard"; const role = /^Role: (\w+)/m.exec(user)?.[1] ?? "r1"; const count = { brief: 2, standard: 3, comprehensive: 4 }[length] ?? 3; const conc = role === "conc"; const r2 = role === "r2"; - // R2: one full answer for each thing they are likely to say. - const answers = length === "brief" ? 10 : 12; + // R2: one full answer for each thing they are likely to say; part 1 of a + // split R2 case writes only its own share of them. + const answers = part ? Number(part[3]) - Number(part[2]) + 1 : length === "brief" ? 10 : 12; return { opening: { hook: `${MARK} opening hook: picture a nurse who skips the night bus because the fare is a tenth of her shift's pay.`, diff --git a/supabase/functions/_shared/output-caps.ts b/supabase/functions/_shared/output-caps.ts index 93327f2a9..37f1ba79b 100644 --- a/supabase/functions/_shared/output-caps.ts +++ b/supabase/functions/_shared/output-caps.ts @@ -26,6 +26,18 @@ export const REPLY_TOKEN_CAPS = { * limit. The cap only bounds the reply, so room costs nothing unless used. */ generate: 12_000, + /** + * The R2 plan: our two arguments and the claims the other side will make, + * one line each. Written at low effort before the parts below. + */ + casePlan: 1_500, + /** + * One of the parts an R2 case is written in at the same time: the speech + * and three answers, or three answers. A part is a quarter of the case, so + * its smaller cap also keeps its reasoning budget (a share of max_tokens) + * small enough that the slowest part finishes well inside the 130 s budget. + */ + caseWriterPart: 6_000, attack: 4_000, brainstorm: 4_000, factoids: 4_000, @@ -66,6 +78,7 @@ const REASONING_HEADROOM: Record = { low: 2_048, medium */ export const REASONING_FLOOR_TOKENS: Partial> = { generate: 10_000, + caseWriterPart: 10_000, }; /** diff --git a/supabase/functions/_shared/prompts/generate.ts b/supabase/functions/_shared/prompts/generate.ts index c815092a1..37cd53941 100644 --- a/supabase/functions/_shared/prompts/generate.ts +++ b/supabase/functions/_shared/prompts/generate.ts @@ -151,17 +151,38 @@ export const SPEECH_MINUTES: Record = { intro: 4, r1: 6, r2: 6, /** A student speaks about this many words a minute in a round. */ export const SPOKEN_WORDS_PER_MINUTE = 130; +/** Arguments per length setting, as the Length Instruction asks for them. */ +const CONTENTION_COUNT: Record = { brief: 2, standard: 3, comprehensive: 4 }; +/** Words the opening hook, the thesis line and the closing line take together. */ +const FRAME_WORDS = 90; +const roundTo5 = (words: number) => Math.max(5, Math.round(words / 5) * 5); + +/** + * Word targets for each spoken field of one argument, so the arguments add up + * to the speech's budget. A single total is not enough: a model asked for + * "about 600 words" wrote 750-820, while per-field targets it can follow. + */ +function argumentWordTargets(spokenWords: number, length: string): string { + const count = CONTENTION_COUNT[length] ?? 3; + const perArgument = Math.floor((spokenWords - FRAME_WORDS) / count); + const claim = Math.min(20, roundTo5(perArgument * 0.12)); + const reasoning = roundTo5(perArgument * 0.45); + const evidence = roundTo5(perArgument * 0.28); + const whyItMatters = roundTo5(perArgument * 0.15); + return `With ${count} arguments, each argument is about ${perArgument} spoken words: claim one sentence of at most ${claim} words, reasoning about ${reasoning} words, evidence about ${evidence} words, whyItMatters about ${whyItMatters} words. opening.hook and caseStructure.thesis at most 35 words each, closingLine one sentence of at most 25 words. Aim for these numbers, not under them: a shorter field leaves the speaker's time unused, a longer one runs past it.`; +} + /** The word budget the speaker reads out, stated in the request so speeches fit their time. */ -export function speechBudgetInstruction(role: string): string | undefined { +export function speechBudgetInstruction(role: string, length = "standard"): string | undefined { const minutes = SPEECH_MINUTES[role]; if (!minutes) return undefined; const words = minutes * SPOKEN_WORDS_PER_MINUTE; const head = `${minutes} minutes, about ${words} spoken words at ${SPOKEN_WORDS_PER_MINUTE} words a minute.`; switch (role) { case "intro": - return `${head} The opening, the "we will show" line, the arguments in short form and the closing line together stay within ${words} words. The "If they say..." answers are a separate short prep list.`; + return `${head} The opening, the "we will show" line, the arguments in short form and the closing line together stay within ${words} words. ${argumentWordTargets(words, length)} The "If they say..." answers are a separate short prep list.`; case "r1": - return `${head} The opening, the arguments and the closing line together stay within about ${words - 180} words, leaving time for the "If they say..." answers the speaker picks.`; + return `${head} The opening, the arguments and the closing line together stay within about ${words - 180} words, leaving time for the "If they say..." answers the speaker picks. ${argumentWordTargets(words - 180, length)}`; case "r2": return `${head} The opening, the backing-up paragraphs and the closing line together stay within about 300 words (about 2 minutes); the rest of the time is for the "If they say..." blocks the other side actually used.`; default: @@ -202,7 +223,7 @@ export function buildGenerateUserPrompt(input: Record, searchCo conversational: "Use natural spoken-round phrasing without slang or filler.", aggressive: "Use sharper strategic phrasing and clearer attack language while staying professional.", }[tone] || "Use polished, judge-facing language with controlled phrasing."; - const speechBudget = speechBudgetInstruction(role); + const speechBudget = speechBudgetInstruction(role, length); return `Motion: ${asString(input.motion)} Side: ${asString(input.side, "pro")} Format: ${asString(input.format, "custom")} @@ -217,3 +238,53 @@ ${outputLanguageInstruction(language)} ${searchContext}`; } + +// === R2 written in parts === +// One R2 case is about 12 full answers plus the speech, which one call wrote +// in up to 124 s of the 130 s budget. A short plan call picks our arguments +// and the claims to answer; the parts then write at the same time from it. + +/** "If they say..." answers in an R2 case, by length. */ +export function r2CounterCount(length: string): number { + return length === "brief" ? 10 : 12; +} + +/** Words per R2 answer, by length (the Length Instruction's range). */ +export function r2AnswerWords(length: string): string { + return ({ brief: "100-130", standard: "120-150", comprehensive: "150-180" } as Record)[length] ?? "120-150"; +} + +export interface R2Plan { + ourArguments: Array<{ title: string; claim: string }>; + theirClaims: string[]; +} + +export function r2PlanInstruction(counterCount: number): string { + return `=== PLANNING STEP: do not write the case yet === +This R2 case is written in parts at the same time, all from the plan you write now. This step overrides the Length Instruction and the output format: return ONLY this JSON object, in the output language: +{"ourArguments": [{"title": "a few words", "claim": "one sentence"}, {"title": "a few words", "claim": "one sentence"}], "theirClaims": ["one sentence", "..."]} +- ourArguments: exactly 2. The team's earlier arguments when CURRENT DEBATE CONTEXT has them, by their own names; otherwise the two strongest arguments for our side that the research backs. +- theirClaims: exactly ${counterCount} different things the other side is likely to say, each one sentence the way they would put it: first their 2-3 main arguments, then the supporting points they will use to back each of them up, then the answers they will give to each of our two arguments. No two claims make the same point. Prefer claims the attached research lets us answer with facts.`; +} + +/** + * The instruction for one part. Part 1 writes the speech (opening, backing up + * our case, closing) and its share of the answers; every other part writes + * only answers. Answers are numbered across the whole case. + */ +export function r2PartInstruction(plan: R2Plan, claims: string[], firstNumber: number, part: number, parts: number, length: string): string { + const ours = plan.ourArguments.map((argument, index) => `${index + 1}. ${argument.title}: ${argument.claim}`).join("\n"); + const list = claims.map((claim, index) => `${firstNumber + index}) ${claim}`).join("\n"); + const answers = `exactly ${claims.length} counters, one for each of these claims, in this order, each using the claim as its targetArgument:\n${list}\nEach answer about ${r2AnswerWords(length)} words, built as the R2 rules ask, and tied to our arguments by name.`; + const head = `=== WRITING PART ${part} OF ${parts}: claims ${firstNumber}-${firstNumber + claims.length - 1} === +This R2 case is written in ${parts} parts at the same time, from one plan. This part overrides the Length Instruction's counts. +Our arguments, in this order (call them "our first argument" and "our second argument"): +${ours}`; + return part === 1 + ? `${head} +Write the opening, caseStructure, exactly ${plan.ourArguments.length} reinforce items (one for each of our arguments, in order), the closingLine, and ${answers} +contentions stays an empty array. sources lists every source this part names.` + : `${head} +Write only "If they say..." answers: return a JSON object with only "counters" and "sources". Write ${answers} +No opening, reinforce, whyWeWin or closingLine: another part writes them. sources lists every source this part names.`; +} diff --git a/supabase/functions/ai/features/generate.ts b/supabase/functions/ai/features/generate.ts index 8a73614ef..0c913a68a 100644 --- a/supabase/functions/ai/features/generate.ts +++ b/supabase/functions/ai/features/generate.ts @@ -1,6 +1,7 @@ import { logUsage } from "../../_shared/entitlements.ts"; import { mergeNormalizedUsage } from "../../_shared/cost.ts"; import { callAI } from "../provider.ts"; +import { writeR2Case } from "../r2-case.ts"; import { isRecord } from "../request.ts"; import { cleanGeneratedText } from "../text.ts"; import { buildUserPrompt } from "../prompt-builder.ts"; @@ -87,7 +88,11 @@ export async function handleGenerate(ctx: FeatureContext): Promise { const systemPrompt = await buildSystemPromptFor(feature, research); const userPrompt = buildUserPrompt(feature, sanitizedInput, searchContext); const messages = [{ role: "user", content: userPrompt }]; - const aiResult = recordMainResult(await callAI(provider, model, systemPrompt, messages, reasoningEffort, "generate", workSignal, ctx.stream)); + // R2 is the longest reply by far, so it is planned and written in parts at + // the same time to finish well inside the request budget (see r2-case.ts). + const aiResult = recordMainResult(role === "r2" + ? await writeR2Case({ callAI, provider, model, reasoningEffort, systemPrompt, userPrompt, length, signal: workSignal, stream: ctx.stream }) + : await callAI(provider, model, systemPrompt, messages, reasoningEffort, "generate", workSignal, ctx.stream)); const output = cleanGeneratedText(aiResult.output); // Preserve model evidence warnings when research exists. If research diff --git a/supabase/functions/ai/r2-case.ts b/supabase/functions/ai/r2-case.ts new file mode 100644 index 000000000..6b9b738fd --- /dev/null +++ b/supabase/functions/ai/r2-case.ts @@ -0,0 +1,154 @@ +// An R2 case written in parts at the same time. +// +// One call wrote the speech and all 12 answers in up to 124 s of the 130 s +// request budget, and the Free plan stops a worker at 150 s, so the budget +// cannot grow. Instead a short plan call (low effort) picks our two arguments +// and the claims to answer, then four parts write from it in parallel at the +// student's model and effort: part 1 the speech and three answers (streamed to +// the browser), parts 2-4 three answers each. The slowest part is about a +// quarter of the old reply. + +import { mergeNormalizedUsage, type NormalizedUsage } from "../_shared/cost.ts"; +import type { ReasoningEffort } from "../_shared/model-catalog.ts"; +import type { ReplyCap } from "../_shared/output-caps.ts"; +import { type R2Plan, r2CounterCount, r2PartInstruction, r2PlanInstruction } from "../_shared/prompts/generate.ts"; +import type { Provider } from "../_shared/research.ts"; +import { type AIResult, AIResponseValidationError, type GenerationStream } from "./provider.ts"; + +export const R2_PARTS = 4; + +type CallAI = ( + provider: Provider, + model: string, + systemPrompt: string, + messages: Array<{ role: string; content: string }>, + reasoningEffort: ReasoningEffort, + cap: ReplyCap, + signal?: AbortSignal, + stream?: GenerationStream, +) => Promise; + +export interface R2CaseRequest { + callAI: CallAI; + provider: Provider; + model: string; + reasoningEffort: ReasoningEffort; + systemPrompt: string; + /** The ordinary generate user prompt; each step appends its own instruction. */ + userPrompt: string; + length: string; + signal: AbortSignal; + stream?: GenerationStream; +} + +const record = (value: unknown): Record | undefined => + value && typeof value === "object" && !Array.isArray(value) ? value as Record : undefined; +const line = (value: unknown, max: number): string => typeof value === "string" ? value.replace(/\s+/g, " ").trim().slice(0, max) : ""; + +/** + * Reads the plan the model returned. A plan short of claims still makes a + * usable case down to the R2 floor (three short of the target), as + * enforceGenerateControls allows; anything thinner fails the request. + */ +export function parseR2Plan(output: unknown, counterCount: number): R2Plan | string { + const plan = record(output); + if (!plan) return "plan"; + const ourArguments = (Array.isArray(plan.ourArguments) ? plan.ourArguments : []) + .map((item) => ({ title: line(record(item)?.title, 120), claim: line(record(item)?.claim, 400) })) + .filter((item) => item.title && item.claim) + .slice(0, 2); + if (ourArguments.length === 0) return "plan.ourArguments"; + const seen = new Set(); + const theirClaims = (Array.isArray(plan.theirClaims) ? plan.theirClaims : []) + .map((claim) => line(claim, 400)) + .filter((claim) => claim && !seen.has(claim.toLowerCase()) && seen.add(claim.toLowerCase())) + .slice(0, counterCount); + if (theirClaims.length < counterCount - 3) return "plan.theirClaims"; + return { ourArguments, theirClaims }; +} + +/** Part 1 also writes the speech, so it takes the smallest share of claims; the rest split evenly. */ +export function splitR2Claims(claims: string[], parts = R2_PARTS): string[][] { + const first = Math.floor(claims.length / parts); + const rest = claims.slice(first); + const chunks = [claims.slice(0, first)]; + for (let index = 0, start = 0; index < parts - 1; index++) { + const size = Math.ceil((rest.length - start) / (parts - 1 - index)); + chunks.push(rest.slice(start, start + size)); + start += size; + } + return chunks.filter((chunk, index) => index === 0 || chunk.length > 0); +} + +/** Part 1's case with every part's answers in plan order and each source once. */ +export function mergeR2Parts(outputs: unknown[]): Record { + const parts = outputs.map((output) => record(output) ?? {}); + const seen = new Set(); + const sources = parts.flatMap((part) => Array.isArray(part.sources) ? part.sources : []).filter((source) => { + const item = record(source); + if (!item) return false; + const key = (typeof item.url === "string" && item.url) || `${String(item.org ?? "").toLowerCase()}|${String(item.year ?? "")}`; + if (seen.has(key)) return false; + seen.add(key); + return true; + }); + return { + ...parts[0], + contentions: [], + counters: parts.flatMap((part) => Array.isArray(part.counters) ? part.counters : []), + sources, + }; +} + +/** Bills every call the failed request made, then hands back the first real failure. */ +function withUsage(error: unknown, usage: NormalizedUsage | undefined, completedUsage: NormalizedUsage | undefined): unknown { + const failure = (error && typeof error === "object" ? error : new Error(String(error))) as { usage?: NormalizedUsage; completedUsage?: NormalizedUsage }; + if (usage) failure.usage = usage; + if (completedUsage) failure.completedUsage = completedUsage; + return failure; +} + +export async function writeR2Case(request: R2CaseRequest): Promise { + const { callAI, provider, model, reasoningEffort, systemPrompt, userPrompt, length, signal, stream } = request; + const counterCount = r2CounterCount(length); + const ask = (instruction: string) => [{ role: "user", content: `${userPrompt}\n\n${instruction}` }]; + + // The plan is short and decides what each part writes, so it runs at low + // effort; the parts keep the student's effort for the writing itself. + const planResult = await callAI(provider, model, systemPrompt, ask(r2PlanInstruction(counterCount)), "low", "casePlan", signal); + const plan = parseR2Plan(planResult.output, counterCount); + if (typeof plan === "string") throw new AIResponseValidationError(plan, planResult.usage); + + // One failed part fails the case, so stop the others at once. + const stopParts = new AbortController(); + const partSignal = AbortSignal.any([signal, stopParts.signal]); + let firstFailure = undefined as { error: unknown } | undefined; + // Part 1 streams the draft; the answer is finished only once every part is. + const partStream = stream && { ...stream, finished: undefined }; + const chunks = splitR2Claims(plan.theirClaims); + let firstNumber = 1; + const settled = await Promise.allSettled(chunks.map((claims, index) => { + const instruction = r2PartInstruction(plan, claims, firstNumber, index + 1, chunks.length, length); + firstNumber += claims.length; + return callAI(provider, model, systemPrompt, ask(instruction), reasoningEffort, "caseWriterPart", partSignal, index === 0 ? partStream : undefined) + .catch((error) => { + firstFailure ??= { error }; + stopParts.abort(); + throw error; + }); + })); + + const partUsage = settled.map((result) => result.status === "fulfilled" + ? result.value.usage + : record(result.reason)?.usage as NormalizedUsage | undefined); + const usage = mergeNormalizedUsage([planResult.usage, ...partUsage]); + if (firstFailure) { + const completed = mergeNormalizedUsage([planResult.usage, ...settled.map((result) => result.status === "fulfilled" ? result.value.usage : undefined)]); + throw withUsage(firstFailure.error, usage, completed); + } + stream?.finished?.(); + return { + output: mergeR2Parts(settled.map((result) => (result as PromiseFulfilledResult).value.output)), + usage, + }; +} diff --git a/supabase/functions/ai/r2-case_test.ts b/supabase/functions/ai/r2-case_test.ts new file mode 100644 index 000000000..27bacb384 --- /dev/null +++ b/supabase/functions/ai/r2-case_test.ts @@ -0,0 +1,74 @@ +import { assertEquals, assertRejects } from "jsr:@std/assert@1.0.19"; +import type { NormalizedUsage } from "../_shared/cost.ts"; +import type { AIResult, GenerationStream } from "./provider.ts"; +import { parseR2Plan, writeR2Case } from "./r2-case.ts"; + +const usage = (cost: number): NormalizedUsage => ({ inputTokens: 100, outputTokens: 10, reportedCostUsd: cost, capCostUsd: cost }); +const plan = { + ourArguments: [{ title: "Bullying", claim: "Bullying follows pupils home." }, { title: "Unequal care", claim: "Help depends on background." }], + theirClaims: Array.from({ length: 12 }, (_, index) => `claim ${index + 1}`), +}; +const cents = (usd?: number) => Math.round((usd ?? NaN) * 100); +const partClaims = (prompt: string) => [...prompt.matchAll(/^(\d+)\) (claim \d+)$/gm)].map((match) => match[2]); + +function request(callAI: (prompt: string, cap: string, signal?: AbortSignal, stream?: GenerationStream) => Promise, stream?: GenerationStream) { + return { + callAI: (_provider: string, _model: string, _system: string, messages: Array<{ content: string }>, _effort: string, cap: string, signal?: AbortSignal, partStream?: GenerationStream) => + callAI(messages[0].content, cap, signal, partStream), + provider: "openrouter" as const, + model: "anthropic/claude-sonnet-5.5", + reasoningEffort: "medium" as const, + systemPrompt: "SYSTEM", + userPrompt: "Role: r2", + length: "standard", + signal: new AbortController().signal, + stream, + }; +} + +Deno.test("an R2 case keeps every planned answer in plan order, streams only part 1, and finishes once", async () => { + const streamed: number[] = []; + let finished = 0; + const stream = { delta: () => {}, reset: () => {}, phase: () => {}, finished: () => finished++ } as GenerationStream; + const result = await writeR2Case(request(async (prompt, cap, _signal, partStream) => { + if (cap === "casePlan") return { output: plan, usage: usage(0.01) }; + const part = Number(/WRITING PART (\d+)/.exec(prompt)![1]); + if (partStream) streamed.push(part); + // Part 1 must not mark the answer finished while the others still write. + partStream?.finished?.(); + return { + output: { + ...(part === 1 ? { opening: { hook: "hook" }, reinforce: [{ argument: "a", text: "t" }], closingLine: "close" } : {}), + counters: partClaims(prompt).map((targetArgument) => ({ targetArgument, counterpoint: "answer" })), + sources: [{ org: "KSH", year: "2025", url: "https://ksh.hu/a" }, { org: `Part ${part}`, year: "2024" }], + }, + usage: usage(0.1), + }; + }, stream)); + + const output = result.output as { counters: Array<{ targetArgument: string }>; sources: unknown[]; closingLine: string }; + assertEquals(output.counters.map((counter) => counter.targetArgument), plan.theirClaims); + assertEquals(output.sources.length, 5); + assertEquals(output.closingLine, "close"); + assertEquals(streamed, [1]); + assertEquals(finished, 1); + assertEquals(cents(result.usage?.reportedCostUsd), 41); +}); + +Deno.test("a failed R2 part stops the other parts and bills every call", async () => { + const error = await assertRejects(() => writeR2Case(request(async (prompt, cap, signal) => { + if (cap === "casePlan") return { output: plan, usage: usage(0.01) }; + if (prompt.includes("WRITING PART 2")) throw Object.assign(new Error("provider down"), { usage: usage(0.02) }); + // The other parts run until they are stopped, then report what they used. + await new Promise((_, reject) => signal!.addEventListener("abort", () => reject(Object.assign(new Error("aborted"), { usage: usage(0.03) })))); + throw new Error("unreachable"); + })), Error, "provider down") as Error & { usage?: NormalizedUsage; completedUsage?: NormalizedUsage }; + assertEquals(cents(error.usage?.capCostUsd), 12); + assertEquals(cents(error.completedUsage?.capCostUsd), 1); +}); + +Deno.test("an R2 plan too thin for the case is refused", () => { + assertEquals(parseR2Plan({ ourArguments: plan.ourArguments, theirClaims: plan.theirClaims.slice(0, 8) }, 12), "plan.theirClaims"); + assertEquals(parseR2Plan({ ourArguments: [], theirClaims: plan.theirClaims }, 12), "plan.ourArguments"); + assertEquals((parseR2Plan({ ...plan, theirClaims: [...plan.theirClaims, "claim 1"] }, 12) as { theirClaims: string[] }).theirClaims.length, 12); +});