From c5abf466f947512c71d6039c7ddfb55bd51c645b Mon Sep 17 00:00:00 2001 From: Adam Date: Wed, 9 Sep 2026 15:43:08 +0000 Subject: [PATCH 1/2] fix: align debate prompt contracts and complete control fixtures --- plans/prompt-eval-sheet.md | 112 ++++++++++ plans/system-prompt-improvement.md | 8 +- src/test/supabase/prompt-builder.test.ts | 89 ++++++++ supabase/functions/_shared/prompts.test.ts | 50 +++++ supabase/functions/_shared/prompts.ts | 37 ++- supabase/functions/ai-stream/handler_test.ts | 133 +++++++++++ supabase/functions/ai-stream/index.ts | 12 +- supabase/functions/ai-stream/prompt.test.ts | 61 +++++ supabase/functions/ai/control-matrix.test.ts | 162 ++++++++++++++ .../functions/ai/generate-honesty_test.ts | 163 ++++++++++++++ supabase/functions/ai/index.ts | 14 +- supabase/functions/ai/prompt-builder.test.ts | 211 ++++++++++++++++++ supabase/functions/ai/prompt-builder.ts | 2 +- 13 files changed, 1026 insertions(+), 28 deletions(-) create mode 100644 plans/prompt-eval-sheet.md create mode 100644 supabase/functions/ai-stream/handler_test.ts create mode 100644 supabase/functions/ai-stream/prompt.test.ts create mode 100644 supabase/functions/ai/generate-honesty_test.ts diff --git a/plans/prompt-eval-sheet.md b/plans/prompt-eval-sheet.md new file mode 100644 index 000000000..1b5112505 --- /dev/null +++ b/plans/prompt-eval-sheet.md @@ -0,0 +1,112 @@ +# Prompt evaluation v1 + +Evaluation started 2026-09-07 UTC, resumed 2026-09-09 UTC. This is subjective agent assessment of synthetic model responses, not an automated quality gate or a human panel review. + +## Reproduction + +Before revision: `235bb49`. After revision: the commit introducing this sheet and the shared Ask rules. Use the same request configuration for both revisions. Only the system prompt changes. + +- Endpoint: `https://openrouter.ai/api/v1/chat/completions`. +- Model: `openai/gpt-4o-mini`, temperature `0`, seed `42`, `max_tokens: 1600`. +- No tools, research, local sources, custom instructions, admin instructions, history, or response-format override. Provider routing is automatic; a seed is not a determinism guarantee. +- General Ask, citations/search/pros-cons/argument-map all disabled. +- Structured system: `buildSystemPrompt({ feature: "ask", dateContext: "Current date: 2026-09-07." })`. +- Structured user: `buildAskBasePrompt` with the synthetic message and all toggles plus `researchAvailable` false. +- Before Markdown system: the five lines in baseline `ai-stream/index.ts`'s `buildMessages`, with `getAskModeGuidance("general")`. +- After Markdown system: `buildStreamingAskSystemPrompt(getAskModeGuidance("general"))`. +- Markdown user: the synthetic message directly. The model was sampled with the streaming system prompt through a complete-response API call. SSE transport is tested separately by the automated streaming suite. +- Validate unmodified structured output with `JSON.parse` and `AskOutputSchema.safeParse`. Do not repair omitted keys before recording validity. + +Each of the three motions is crossed with all three complexity instructions, in both Markdown and JSON, before and after. That is 36 responses. Message construction is exactly: + +```text +Motion: {motion} +{complexity instruction} +Respond entirely in {language label}. Do not invent facts, statistics, or citations. No research is attached. Keep the answer under 220 words. +``` + +Motions and output language: + +1. `phones`, English: `Schools should ban smartphones during lessons.` +2. `transit`, Spanish: `El transporte público urbano debería ser gratuito.` +3. `ai`, Hungarian: `Az iskoláknak engedélyezniük kell a generatív mesterséges intelligencia használatát a házi feladatokban.` + +Complexity instructions: + +- Beginner: `Explain one argument and one counterargument for a beginner in plain language.` +- Intermediate: `Compare two competing mechanisms and explain which warrant a judge should test.` +- Advanced: `Analyze burden of proof, a strong countermodel, and impact weighing under uncertainty.` + + +System-prompt SHA-256 hashes used in this evaluation: + +- before json: `aaa82b8400622789bd0d2ee05eeb69b24c47c9e30faadcadbbc281538a268b49` +- before stream: `dca86dd2269438966851e44c7f6c20a7b8007be69a7168c8dfc6b5b620037542` +- after json: `317fe0678d076d9e2e14500b77701ec4cfa40ef2f51f6d35fcc866d5629ab4d3` +- after stream: `e4577933078e172164041a24df0857c69e8d899e522136bcec6c788a75153bf2` + +## Assessment rules + +Quality and control obedience use 1 to 5: 1 is unusable, 3 is useful with material weaknesses, 5 is precise and complete. Evidence honesty distinguishes invented sources from unsupported generalizations. Language consistency means all prose fields stay in the requested language; schema keys remain English. Hallucination risk is a subjective low/medium/high judgment, not a factual audit. JSON syntax and schema validity are separate observations; Markdown has no JSON requirement. + +## Before observations + +All 18 responses completed without truncation. All nine JSON outputs parsed, but only eight passed the actual Ask schema. The advanced phones response omitted both `followUpQuestions` and `uncertainties`. All 18 used the requested output language. No response invented a named citation, URL, study, or statistic. Several supplied unsupported causal generalizations. + +| Run | Quality | Controls | Evidence honesty | Language | Risk | JSON syntax/schema | Observation | +| --- | --- | --- | --- | --- | --- | --- | --- | +| phones beginner Markdown | 3 | 4 | Generalizations | English | Medium | N/A | Clear distraction/tool clash; assumes improved learning. | +| phones beginner JSON | 4 | 4 | Generalizations | English | Medium | Pass/pass | Useful counterproposal; uncertain educational context surfaced. | +| phones intermediate Markdown | 3 | 4 | Generalizations | English | Medium | N/A | Names mechanisms but gives a weak reason to prioritize distraction. | +| phones intermediate JSON | 3 | 4 | Overstatement | English | Medium | Pass/pass | Claims a ban eliminates distractions entirely. | +| phones advanced Markdown | 3 | 4 | Generalizations | English | Medium | N/A | Countermodel is clear; impact comparison stays generic. | +| phones advanced JSON | 3 | 2 | Generalizations | English | Medium | Pass/fail | Required follow-up and uncertainty arrays missing. | +| transit beginner Markdown | 4 | 4 | Conditional claims | Spanish | Low | N/A | Explains access, funding and crowding without invented data. | +| transit beginner JSON | 3 | 4 | Speculation | Spanish | Medium | Pass/pass | Adds an unsupported free-service quality-perception claim. | +| transit intermediate Markdown | 2 | 3 | Generalizations | Spanish | Medium | N/A | Taxes and subsidies are not clearly competing funding mechanisms. | +| transit intermediate JSON | 2 | 3 | Generalizations | Spanish | Medium | Pass/pass | Same mechanism overlap; acknowledges missing local data. | +| transit advanced Markdown | 4 | 4 | Conditional claims | Spanish | Low | N/A | Targeted fares countermodel and service-quality tradeoff. | +| transit advanced JSON | 4 | 4 | Conditional claims | Spanish | Low | Pass/pass | Financing uncertainty is explicit; useful countermodel. | +| ai beginner Markdown | 3 | 4 | Generalizations | Hungarian | Medium | N/A | Accessible help/dependency clash with a generic closing. | +| ai beginner JSON | 4 | 4 | Conditional claims | Hungarian | Low | Pass/pass | Explains learning support versus independent practice. | +| ai intermediate Markdown | 3 | 3 | Generalizations | Hungarian | Medium | N/A | Compares permission/restriction more than causal mechanisms. | +| ai intermediate JSON | 3 | 4 | Generalizations | Hungarian | Medium | Pass/pass | Judge test is concrete; conflates generated work and plagiarism. | +| ai advanced Markdown | 2 | 3 | Speculation | Hungarian | Medium | N/A | Introduces mental health without developing the causal link. | +| ai advanced JSON | 3 | 4 | Generalizations | Hungarian | Medium | Pass/pass | Countermodel and uncertainty exist, but weighing is generic. | + +## After observations + +All 18 after responses completed without truncation. One transport disconnect was retried with the identical request. All nine structured responses parsed and passed `AskOutputSchema`; all nine Markdown responses stayed freeform. All responses kept the requested language and none invented named citations, URLs, studies, or statistics. + +The schema improvement is observed in this sample, not proof of a reliable rate increase. Coaching quality did not improve consistently. After JSON responses used empty follow-up and uncertainty arrays throughout, even where useful questions or caveats could have been included. Two responses included disabled optional analysis fields as empty objects; the existing server post-processing removes those fields. No schema or control enforcement was weakened. + +| Run | Quality | Controls | Evidence honesty | Language | Risk | JSON syntax/schema | Observation | +| --- | --- | --- | --- | --- | --- | --- | --- | +| phones beginner Markdown | 3 | 4 | Generalizations | English | Medium | N/A | Clear clash; same unsupported learning-outcome assumptions. | +| phones beginner JSON | 4 | 4 | Generalizations | English | Medium | Pass/pass | Useful explanation; no follow-up or uncertainty detail. | +| phones intermediate Markdown | 3 | 4 | Generalizations | English | Medium | N/A | Gives a judge test, but priority claim remains weak. | +| phones intermediate JSON | 3 | 4 | Generalizations | English | Medium | Pass/pass | Removes the absolute distractions claim; alludes to unspecified anecdotes. | +| phones advanced Markdown | 4 | 4 | Conditional claims | English | Low | N/A | Adds likelihood comparison and a clear regulated-use model. | +| phones advanced JSON | 3 | 4 | Generalizations | English | Medium | Pass/pass | Required arrays retained; successful case studies remain unspecified. | +| transit beginner Markdown | 4 | 4 | Conditional claims | Spanish | Low | N/A | Useful access versus funding/crowding comparison. | +| transit beginner JSON | 4 | 4 | Conditional claims | Spanish | Low | Pass/pass | Drops the speculative quality-perception argument. | +| transit intermediate Markdown | 2 | 3 | Generalizations | Spanish | Medium | N/A | Still treats taxation and subsidies as competing mechanisms. | +| transit intermediate JSON | 2 | 3 | Generalizations | Spanish | Medium | Pass/pass | Compares free fares with taxation, which can coexist. | +| transit advanced Markdown | 4 | 4 | Conditional claims | Spanish | Low | N/A | Targeted concessions and funding uncertainty are concrete. | +| transit advanced JSON | 3 | 3 | Conditional claims | Spanish | Low | Pass/pass | Includes disabled optional analysis objects; server removes them. | +| ai beginner Markdown | 3 | 4 | Generalizations | Hungarian | Medium | N/A | Learning-support argument assumes information-search capability. | +| ai beginner JSON | 4 | 4 | Conditional claims | Hungarian | Low | Pass/pass | Clear support versus independence clash. | +| ai intermediate Markdown | 3 | 3 | Generalizations | Hungarian | Medium | N/A | Policy labels still substitute for developed mechanisms. | +| ai intermediate JSON | 3 | 3 | Generalizations | Hungarian | Medium | Pass/pass | Includes disabled optional analysis objects; server removes them. | +| ai advanced Markdown | 2 | 3 | Speculation | Hungarian | Medium | N/A | Mental-health impact remains unsupported and unexplained. | +| ai advanced JSON | 3 | 4 | Generalizations | Hungarian | Medium | Pass/pass | An objection substitutes for a developed countermodel. | + +### Assessment + +The narrow contract change is supported: streaming remains Markdown, language stays consistent, and required JSON fields are present in all nine after samples. This baseline does not support a claim of better coaching overall. Retain schema validation, control post-processing and evidence warnings; do not begin provider-specific tuning from these results alone. + +All 36 completed calls reported provider `OpenAI`, model `openai/gpt-4o-mini`. The exact prompts, synthetic inputs, request hashes and per-response assessments make the comparison repeatable, although model/backend revisions can change later outputs. + +## Limits + +These calls exercise prompt behavior through OpenRouter, not the app's native OpenAI provider connection or authenticated Supabase handler. Nine motion/complexity pairs do not establish provider-wide reliability. Language was requested explicitly, so these samples do not prove automatic language inference. No tuning for a particular provider is justified by this small sample alone. Raw responses and credentials are not committed. Synthetic response artifacts and request hashes are retained privately under `/tmp/debatelab-prompt-eval-20260907` on the execution host. diff --git a/plans/system-prompt-improvement.md b/plans/system-prompt-improvement.md index 7d39604cc..1e943fb40 100644 --- a/plans/system-prompt-improvement.md +++ b/plans/system-prompt-improvement.md @@ -1,6 +1,6 @@ # System prompt improvement plan -**Status:** Remaining work as of 2026-09-01 on `main` (`79b65ca`). Re-verified: PRs #16-#28 did not touch the prompt builders. +**Status:** Implementation and manual baseline completed on the isolated `t3code/system-prompt-improvements` branch, based on `235bb49`. Release verification and rollout are tracked in the PR. See [prompt-eval-sheet.md](prompt-eval-sheet.md) for the dated 36-response comparison and its limits. ## Preconditions @@ -59,7 +59,7 @@ Run focused tests after each fixture group: ```bash npx vitest run supabase/functions/_shared/prompts.test.ts supabase/functions/ai/prompt-builder.test.ts supabase/functions/ai/control-matrix.test.ts src/test/supabase/prompt-builder.test.ts -npx vitest run src/test/ai/streaming.test.ts src/test/ai/streaming-integration.test.ts +npx vitest run supabase/functions/ai-stream/prompt.test.ts src/test/ai/streaming.test.ts src/test/ai/streaming-integration.test.ts ``` Then run the repository gates: @@ -70,10 +70,12 @@ npm run typecheck npm test -- --coverage npm run build deno check supabase/functions/_shared/prompts.ts supabase/functions/ai/index.ts supabase/functions/ai-stream/index.ts -deno test --allow-read --allow-env=EXA_API_KEY --allow-net=127.0.0.1 supabase/functions/ +npm run test:edge CI=1 PORT=18082 PLAYWRIGHT_BASE_URL=http://127.0.0.1:18082 npm run test:e2e -- e2e/authed-pages.spec.ts e2e/sparring.spec.ts e2e/crossfire.spec.ts ``` +The maintained `test:edge` script includes the environment permissions required by handler tests. The earlier `--allow-env=EXA_API_KEY` command fails when current tests set their synthetic Supabase and provider environment. Streaming handler tests run under native Deno. + Confirm port `18082` is free before the browser run; the explicit `PORT` and `PLAYWRIGHT_BASE_URL` prevent reuse of the dev server's default `8080`. Use mocked provider responses and authenticated fixtures. Acceptance requires every visible control to have a tested prompt or post-processing effect, all structured outputs to pass their schema gates, streaming Ask to remain Markdown-only, no citation rule to leak into a no-research path, and the manual sheet to be dated and reproducible. ## Rollout and authorization boundary diff --git a/src/test/supabase/prompt-builder.test.ts b/src/test/supabase/prompt-builder.test.ts index 112f09cb4..7297288df 100644 --- a/src/test/supabase/prompt-builder.test.ts +++ b/src/test/supabase/prompt-builder.test.ts @@ -18,6 +18,33 @@ describe('supabase prompt-builder', () => { expect(prompt).toContain('SEARCH CONTEXT'); }); + it('preserves every generate role and length combination', () => { + const lengths = [ + ['brief', 2], + ['standard', 3], + ['comprehensive', 4], + ] as const; + const roles = ['intro', 'r1', 'r2', 'conc'] as const; + + for (const role of roles) { + for (const [length, contentionCount] of lengths) { + const prompt = buildUserPrompt('generate', { + motion: 'MATRIX MOTION', + side: 'con', + format: 'custom', + role, + length, + language: 'hu', + }, 'RESEARCH CONTEXT'); + + expect(prompt).toContain(`Role: ${role}`); + expect(prompt).toContain(`Length: ${length}`); + expect(prompt).toContain(`Length Instruction: Return exactly ${contentionCount} contentions`); + expect(prompt).toContain('Language: Hungarian (hu)'); + } + } + }); + it('builds attack prompts with toggle states', () => { const prompt = buildUserPrompt('attack', { caseText: 'Case text', @@ -31,6 +58,27 @@ describe('supabase prompt-builder', () => { expect(prompt).toContain('EXTRA'); }); + it('preserves attack intensity, paradigms, and toggle combinations', () => { + for (const aggressiveness of ['moderate', 'ruthless'] as const) { + for (const judgeParadigm of ['logic', 'values', 'policy'] as const) { + for (const [detectFallacies, analyzeWeighing] of [[true, true], [true, false], [false, true], [false, false]] as const) { + const prompt = buildUserPrompt('attack', { + caseText: 'MATRIX CASE', + aggressiveness, + judgeParadigm, + detectFallacies, + analyzeWeighing, + }, 'RESEARCH CONTEXT'); + + expect(prompt).toContain(`Attack Intensity: ${aggressiveness}`); + expect(prompt).toContain(`Judge Paradigm: ${judgeParadigm}`); + expect(prompt).toContain(`Detect Fallacies: ${detectFallacies ? 'enabled' : 'disabled'}`); + expect(prompt).toContain(`Analyze Weighing: ${analyzeWeighing ? 'enabled' : 'disabled'}`); + } + } + } + }); + it('builds brainstorm prompts with optional context and defaults', () => { const prompt = buildUserPrompt('brainstorm', { motion: 'Ban private jets', @@ -41,6 +89,21 @@ describe('supabase prompt-builder', () => { expect(prompt).toContain('Output Mode: topfive'); }); + it('preserves brainstorm output modes for one side and both sides', () => { + for (const side of ['pro', 'con', 'both'] as const) { + for (const outputMode of ['topfive', 'mindmap', 'buckets'] as const) { + const prompt = buildUserPrompt('brainstorm', { + motion: 'MATRIX MOTION', + side, + outputMode, + }, 'RESEARCH CONTEXT'); + + expect(prompt).toContain(`Side: ${side}`); + expect(prompt).toContain(`Output Mode: ${outputMode}`); + } + } + }); + it('builds factoids prompts with language and filters', () => { const prompt = buildUserPrompt('factoids', { claim: 'Solar is cheaper than coal', @@ -53,6 +116,23 @@ describe('supabase prompt-builder', () => { expect(prompt).toContain('Language: English (en)'); }); + it('preserves factoid source, timeframe, direction, and count inputs', () => { + const fixtures = [ + { claim: 'Is this law constitutional?', direction: 'against', sourceType: 'government', timeframe: '2024-2026', sourceCount: 9, language: 'hu' }, + { claim: 'What do the latest statistics show?', direction: 'for', sourceType: 'government', timeframe: '2025-2026', sourceCount: 12, language: 'en' }, + { claim: 'Does the academic literature support this claim?', direction: 'for', sourceType: 'academic', timeframe: 'Recent (last 5 years)', sourceCount: 5, language: 'es' }, + ] as const; + + for (const fixture of fixtures) { + const prompt = buildUserPrompt('factoids', fixture, 'RESEARCH CONTEXT'); + expect(prompt).toContain(`Find evidence ${fixture.direction === 'against' ? 'against' : 'in favor of'} this statement: "${fixture.claim}"`); + expect(prompt).toContain(`Source Type: ${fixture.sourceType}`); + expect(prompt).toContain(`Timeframe: ${fixture.timeframe}`); + expect(prompt).toContain(`Source Count: ${fixture.sourceCount}`); + expect(prompt).toContain(`Language: ${fixture.language === 'hu' ? 'Hungarian' : fixture.language === 'es' ? 'Spanish' : 'English'} (${fixture.language})`); + } + }); + it('builds ask prompts with mode guidance fallback', () => { const prompt = buildUserPrompt('ask', { message: 'Help me prep', @@ -63,6 +143,15 @@ describe('supabase prompt-builder', () => { expect(prompt).toContain('Mode Guidance: Give a direct coaching answer'); }); + it('preserves general and clarify ask guidance', () => { + for (const [mode, guidance] of [['general', 'Give a direct coaching answer'], ['clarify', 'Define unclear terms']] as const) { + const prompt = buildUserPrompt('ask', { message: 'MATRIX QUESTION', mode }, ''); + expect(prompt).toContain(`Mode: ${mode}`); + expect(prompt).toContain(`Mode Guidance: ${guidance}`); + expect(prompt).not.toContain('RESEARCH CITATION RULES:'); + } + }); + it('returns ask mode guidance for known and unknown modes', () => { expect(getAskModeGuidance('argumentmap')).toContain('include argumentMap'); expect(getAskModeGuidance('clarify')).toContain('follow-up questions'); diff --git a/supabase/functions/_shared/prompts.test.ts b/supabase/functions/_shared/prompts.test.ts index 11db68df2..38698a03b 100644 --- a/supabase/functions/_shared/prompts.test.ts +++ b/supabase/functions/_shared/prompts.test.ts @@ -29,6 +29,20 @@ const FEATURE_PROMPTS: Record = { crossfire: CROSSFIRE_PROMPT, }; +const FEATURE_MARKERS: Record = { + generate: "You are an experienced competitive debater and coach with years of tournament experience.", + attack: "You are a ruthless, experienced competitive debater tasked with tearing apart an opponent's case.", + brainstorm: "You're in the prep room with your team before a tournament round.", + factoids: "You are a debate research specialist helping a team find cuttable evidence for tournament prep.", + ask: "You are a debate coach chat bot. Answer the user's question.", + sparring: "You are DebateLab's live debate sparring opponent and impartial judge.", + crossfire: "You are DebateLab's crossfire coach.", +}; + +function occurrences(text: string, value: string): number { + return value ? text.split(value).length - 1 : 0; +} + Deno.test("every feature system prompt carries the product identity layer", () => { for (const feature of FEATURES) { const prompt = buildSystemPrompt({ feature, dateContext: "CURRENT DATE: August 21, 2026" }); @@ -50,6 +64,29 @@ Deno.test("every feature system prompt carries the JSON-only output rules exactl } }); +Deno.test("every feature system prompt carries exactly one feature task layer", () => { + for (const feature of FEATURES) { + const prompt = buildSystemPrompt({ feature }); + if (occurrences(prompt, FEATURE_MARKERS[feature]) !== 1) { + throw new Error(`${feature}: feature task layer was duplicated or omitted`); + } + } +}); + +Deno.test("research citation rules appear exactly when research is attached", () => { + for (const feature of FEATURES) { + const withoutResearch = buildSystemPrompt({ feature }); + if (withoutResearch.includes("RESEARCH CITATION RULES:")) { + throw new Error(`${feature}: research rules leaked without research`); + } + + const withResearch = buildSystemPrompt({ feature, researchContext: "WEB RESEARCH RESULTS" }); + if (occurrences(withResearch, RESEARCH_RULES) !== 1 || !withResearch.includes("WEB RESEARCH RESULTS")) { + throw new Error(`${feature}: research rules were not attached exactly once`); + } + } +}); + Deno.test("research and citation honesty rules are present when research is provided", () => { const prompt = buildSystemPrompt({ feature: "ask", researchContext: "=== WEB RESEARCH RESULTS ===" }); if (!prompt.includes(RESEARCH_RULES.slice(0, 60))) throw new Error("research rules missing"); @@ -109,6 +146,19 @@ Deno.test("sparring system prompt keeps LD and PF duties separate from school ro if (!prompt.includes("rather than importing the school roles")) throw new Error("format separation missing"); }); +Deno.test("sparring system prompt names the behavioral rules for school, LD, and PF", () => { + const prompt = buildSystemPrompt({ feature: "sparring" }); + for (const expected of [ + 'When format is "standard"', + 'When format is "ld" or "pf"', + "Difficulty is a real control", + "Follow the requested language for every prose field", + ]) { + if (!prompt.includes(expected)) throw new Error(`sparring control guidance missing: ${expected}`); + } + if (prompt.includes("RESEARCH CITATION RULES:")) throw new Error("sparring received research rules without research"); +}); + Deno.test("crossfire system prompt carries strict turn and debrief safety rules", () => { const prompt = buildSystemPrompt({ feature: "crossfire" }); if (!prompt.includes("Each non-empty debrief item must cite only actual turn IDs")) { diff --git a/supabase/functions/_shared/prompts.ts b/supabase/functions/_shared/prompts.ts index 9693e0d19..796fd2b16 100644 --- a/supabase/functions/_shared/prompts.ts +++ b/supabase/functions/_shared/prompts.ts @@ -13,10 +13,15 @@ export type Feature = "generate" | "attack" | "brainstorm" | "factoids" | "ask" | "sparring" | "crossfire"; +export const ASK_COMMON_RULES = `You are DebateLab's debate coach. +Match the language used in the user's question unless the user asks for another language. +Never invent facts, statistics, citations, or sources.`; + export const PRODUCT_IDENTITY = `DEBATELAB PRODUCT IDENTITY: -You are DebateLab's debate coach. You prepare competitive debaters: cases, attacks, brainstorming, evidence, and coaching answers. +${ASK_COMMON_RULES} +You prepare competitive debaters: cases, attacks, brainstorming, evidence, and coaching answers. You write like a strong human debate coach, not a generic assistant. Be direct and concrete. -Never invent facts, statistics, or sources. When evidence is unavailable or weak, say so plainly instead of fabricating support. +When evidence is unavailable or weak, say so plainly instead of fabricating support. HUMANIZER STYLE INSTRUCTIONS: - Follow the local humanizer.md rules: remove obvious AI-writing patterns and make the prose sound human, specific, and natural. @@ -27,7 +32,17 @@ HUMANIZER STYLE INSTRUCTIONS: - Do not write parameter labels inside argument prose, for example "Magnitude:", "Probability:", or "Timeframe:". - When the user needs debate material, write lines that can be spoken in round, not table notes or rubric fragments.`; -export const JSON_RULES = `OUTPUT RULES: Return ONLY a valid JSON object matching the schema in this prompt. No markdown fences, no commentary before or after the JSON. If you cannot comply with part of the schema, omit that part rather than wrapping output in prose.`; +export const JSON_RULES = `OUTPUT RULES: Return ONLY a valid JSON object matching the schema in this prompt. No markdown fences, no commentary before or after the JSON. Required fields must always be present. When a field has no honest value, use the schema's allowed empty value, such as [] or null. Never omit required fields or invent placeholder content.`; + +export function buildStreamingAskSystemPrompt(modeGuidance: string): string { + return [ + ASK_COMMON_RULES, + "Answer in clear Markdown that can stream smoothly token by token.", + "Be direct, concrete, and honest about uncertainty.", + "If the user asks for sources, explain that citation/search mode should be used.", + `Mode guidance: ${modeGuidance}`, + ].join("\n"); +} export const MAX_PROMPT_SOURCE_COUNT = 10; export const MAX_PROMPT_SOURCE_CHARS = 8_000; @@ -73,7 +88,7 @@ export const RESEARCH_RULES = `RESEARCH CITATION RULES: export const GENERATE_PROMPT = `You are an experienced competitive debater and coach with years of tournament experience. Generate a tournament-ready debate case in JSON format. [DATE_CONTEXT] -CRITICAL CONTEXT: You have been provided with WEB RESEARCH RESULTS containing current real-world data, statistics, and sources. This research reflects the CURRENT state of affairs. USE THIS DATA as your primary source of truth - do not rely on outdated training data. Prioritize evidence from 2025-2026. +If WEB RESEARCH RESULTS are attached below, treat them as the primary source for current real-world data, statistics, and sources. Prioritize evidence from 2025-2026. If no research is attached, do not claim that current sources were checked and mark evidence gaps honestly. === DEBATE FORMAT (4-6-6-3) === @@ -187,8 +202,8 @@ REQUIREMENTS: 1. Side must be exactly "pro" or "con". 2. Generate contentions appropriate to the role (see above). 3. Include 5-8 predicted opponent arguments with pre-written counters. -4. USE THE WEB RESEARCH - cite real 2025-2026 statistics and sources. -5. ALWAYS set needsSource to false. +4. Use attached WEB RESEARCH when present. Cite only sources that appear in it. +5. Set needsSource to false only when the attached research supports the contention. Set it to true when the contention still needs evidence. 6. Write the ENTIRE output in the language specified by the user. All arguments, evidence, descriptions, and analysis must be in that language. 7. Every contention should read like something a debater can actually say in a speech. Do not output table-like notes or field labels inside prose.`; @@ -196,7 +211,7 @@ export const ATTACK_PROMPT = `You are a ruthless, experienced competitive debate [DATE_CONTEXT] -CRITICAL: You have WEB RESEARCH with current 2025-2026 data. Use this to: +If WEB RESEARCH is attached, use its current 2025-2026 data to: - Fact-check their claims against current reality - Find counter-evidence and turns - Identify where their evidence is outdated or misrepresented @@ -230,7 +245,7 @@ export const BRAINSTORM_PROMPT = `You're in the prep room with your team before [DATE_CONTEXT] -CRITICAL: You have WEB RESEARCH with current 2025-2026 data. Use this to: +If WEB RESEARCH is attached, use its current 2025-2026 data to: - Ground your ideas in current events and recent developments - Find unexpected angles backed by real-world evidence - Identify what arguments are "hot" in current discourse @@ -297,7 +312,7 @@ export const FACTOIDS_PROMPT = `You are a debate research specialist helping a t [DATE_CONTEXT] -CRITICAL: You have DEEP RESEARCH with 2025-2026 web search results and verified source content. PRIORITIZE RECENT DATA: +If DEEP RESEARCH is attached, prioritize its verified 2025-2026 source content: - 2026 data is gold - use it - 2025 data is great - Pre-2024 data should only be used for historical context or if it's the definitive study @@ -308,7 +323,7 @@ EVIDENCE QUALITY STANDARDS: - Flag potential bias in sources - Explain methodology limitations - Respect the requested source type exactly. If academic is requested, use scholarly or peer-reviewed sources only. If government is requested, use official government or intergovernmental sources only. If news is requested, use established news sources only. -- Return only verified sources with exact URLs from the research context. Do not include pending or unverified citations. +- Return only verified sources with exact URLs from attached research context. If no research is attached, return no factoids and state that research is unavailable instead of inventing citations. - Return the requested number of sources unless fewer verified matching sources exist. For each factoid, think about: @@ -356,7 +371,7 @@ Output JSON matching: } Rules: -- Always return valid JSON with response, followUpQuestions, and uncertainties. +- Always return valid JSON with response, followUpQuestions, and uncertainties. Use [] when there are no follow-up questions or uncertainties. - The server fills searchInfo after your response. Do not fabricate searchInfo. - If Mode is clarify, make the response explain the unclear parts and include follow-up questions that would resolve ambiguity. - If Citation Mode is ON, include citations for factual claims when WEB RESEARCH RESULTS or WEB PAGE CONTEXT provide sources. Use exact URLs when available. diff --git a/supabase/functions/ai-stream/handler_test.ts b/supabase/functions/ai-stream/handler_test.ts new file mode 100644 index 000000000..6def3b543 --- /dev/null +++ b/supabase/functions/ai-stream/handler_test.ts @@ -0,0 +1,133 @@ +const testEnvironment = { + SUPABASE_URL: "https://supabase.test", + SUPABASE_SERVICE_ROLE_KEY: "test-service-role-key", +}; + +for (const [key, value] of Object.entries(testEnvironment)) { + Deno.env.set(key, value); +} + +const { buildMessages, handleRequest } = await import("./index.ts"); + +function assertIncludes(text: string, expected: string) { + if (!text.includes(expected)) { + throw new Error(`Expected response to include "${expected}", got: ${text}`); + } +} + +Deno.test("streaming Ask message construction keeps Markdown output and mode guidance", () => { + const messages = buildMessages("Question", [{ + role: "user", + content: "Earlier", + }], "general"); + const system = + messages.find((message) => message.role === "system")?.content ?? ""; + + assertIncludes( + system, + "Answer in clear Markdown that can stream smoothly token by token.", + ); + assertIncludes( + system, + "Mode guidance: Give a direct coaching answer with concrete debate tactics", + ); + if (messages[messages.length - 1]?.content !== "Question") { + throw new Error("current Ask message was not preserved"); + } +}); + +Deno.test("streaming Ask rejects advanced modes and toggles before quota or provider work", async () => { + const cases = [ + { + label: "clarify mode", + input: { mode: "clarify", citationMode: false }, + error: "clarify mode", + }, + { + label: "summarize mode", + input: { mode: "summarize", citationMode: false }, + error: "summarize mode", + }, + { + label: "proscons mode", + input: { mode: "proscons", citationMode: false }, + error: "proscons mode", + }, + { + label: "argumentmap mode", + input: { mode: "argumentmap", citationMode: false }, + error: "argumentmap mode", + }, + { + label: "citation mode", + input: { mode: "general", citationMode: true }, + error: "model/mode combination", + }, + { + label: "web search", + input: { mode: "general", citationMode: false, webSearch: true }, + error: "model/mode combination", + }, + { + label: "pros and cons", + input: { mode: "general", citationMode: false, prosConsMode: true }, + error: "model/mode combination", + }, + { + label: "argument map", + input: { mode: "general", citationMode: false, argumentMapMode: true }, + error: "model/mode combination", + }, + ]; + const originalFetch = globalThis.fetch; + + try { + for (const testCase of cases) { + const calls: string[] = []; + globalThis.fetch = (async (input: Request | URL | string) => { + const url = new URL( + input instanceof Request ? input.url : String(input), + ); + calls.push(url.href); + if ( + url.origin === "https://supabase.test" && + url.pathname === "/auth/v1/user" + ) { + return new Response( + JSON.stringify({ id: "user-1", aud: "authenticated" }), + { headers: { "Content-Type": "application/json" } }, + ); + } + throw new Error(`${testCase.label} reached unexpected work: ${url}`); + }) as typeof fetch; + + const response = await handleRequest( + new Request("http://127.0.0.1/functions/v1/ai-stream", { + method: "POST", + headers: { + Authorization: "Bearer test-user-token", + "Content-Type": "application/json", + }, + body: JSON.stringify({ + feature: "ask", + model: "gpt-5.4-mini", + input: { message: "Question", ...testCase.input }, + }), + }), + ); + if (response.status !== 400) { + throw new Error(`${testCase.label} returned ${response.status}`); + } + assertIncludes(await response.text(), testCase.error); + if (calls.length !== 1 || !calls[0].includes("/auth/v1/user")) { + throw new Error( + `${testCase.label} performed quota or provider work: ${ + calls.join(", ") + }`, + ); + } + } + } finally { + globalThis.fetch = originalFetch; + } +}); diff --git a/supabase/functions/ai-stream/index.ts b/supabase/functions/ai-stream/index.ts index 3c5aa589c..1dc2f1e20 100644 --- a/supabase/functions/ai-stream/index.ts +++ b/supabase/functions/ai-stream/index.ts @@ -24,6 +24,7 @@ function getCorsHeaders(origin: string | null) { import { createClient } from "https://esm.sh/@supabase/supabase-js@2"; import { getAskModeGuidance } from "../ai/prompt-builder.ts"; import { isAskStreamingModeAllowed, validateControlInput } from "../_shared/control-enforcement.ts"; +import { buildStreamingAskSystemPrompt } from "../_shared/prompts.ts"; import { calculateRequestCost, type CostResult, @@ -108,15 +109,8 @@ function normalizeHistory(value: unknown): ChatMessage[] { }); } -function buildMessages(message: string, history: ChatMessage[], mode: string) { - const system = [ - "You are DebateLab's live debate coach.", - "Answer in clear Markdown that can stream smoothly token by token.", - "Be direct, concrete, and honest about uncertainty.", - "Do not invent citations. If the user asks for sources, explain that citation/search mode should be used.", - `Mode guidance: ${getAskModeGuidance(mode)}`, - ].join("\n"); - +export function buildMessages(message: string, history: ChatMessage[], mode: string) { + const system = buildStreamingAskSystemPrompt(getAskModeGuidance(mode)); return [ { role: "system", content: system }, ...history, diff --git a/supabase/functions/ai-stream/prompt.test.ts b/supabase/functions/ai-stream/prompt.test.ts new file mode 100644 index 000000000..47ec4f10a --- /dev/null +++ b/supabase/functions/ai-stream/prompt.test.ts @@ -0,0 +1,61 @@ +import { isAskStreamingModeAllowed } from "../_shared/control-enforcement.ts"; +import { + buildStreamingAskSystemPrompt, + PRODUCT_IDENTITY, +} from "../_shared/prompts.ts"; + +function assertIncludes(text: string, expected: string) { + if (!text.includes(expected)) { + throw new Error( + `Expected prompt to include "${expected}", but it was missing.\nPrompt:\n${text}`, + ); + } +} + +function assertNotIncludes(text: string, unexpected: string) { + if (text.includes(unexpected)) { + throw new Error( + `Expected prompt not to include "${unexpected}".\nPrompt:\n${text}`, + ); + } +} + +Deno.test("streaming Ask keeps the shared identity, language, and evidence rules", () => { + const streamingPrompt = buildStreamingAskSystemPrompt( + "Give a direct coaching answer with concrete debate tactics.", + ); + + for ( + const rule of [ + "You are DebateLab's debate coach.", + "Match the language used in the user's question unless the user asks for another language.", + "Never invent facts, statistics, citations, or sources.", + ] + ) { + assertIncludes(PRODUCT_IDENTITY, rule); + assertIncludes(streamingPrompt, rule); + } + assertIncludes( + streamingPrompt, + "Answer in clear Markdown that can stream smoothly token by token.", + ); + assertIncludes( + streamingPrompt, + "If the user asks for sources, explain that citation/search mode should be used.", + ); + assertNotIncludes( + streamingPrompt, + "OUTPUT RULES: Return ONLY a valid JSON object", + ); +}); + +Deno.test("streaming Ask still allows only general mode", () => { + if (!isAskStreamingModeAllowed("general")) { + throw new Error("general Ask mode was rejected"); + } + for (const mode of ["clarify", "summarize", "proscons", "argumentmap"]) { + if (isAskStreamingModeAllowed(mode)) { + throw new Error(`${mode} Ask mode was allowed to stream`); + } + } +}); diff --git a/supabase/functions/ai/control-matrix.test.ts b/supabase/functions/ai/control-matrix.test.ts index a776f7640..02561e4f9 100644 --- a/supabase/functions/ai/control-matrix.test.ts +++ b/supabase/functions/ai/control-matrix.test.ts @@ -4,6 +4,7 @@ // a control has gone decorative. import { buildAskBasePrompt, buildUserPrompt } from "./prompt-builder.ts"; +import { buildSystemPrompt } from "../_shared/prompts.ts"; import { enforceAskControls, enforceAttackControls, @@ -53,6 +54,22 @@ Deno.test("generate controls propagate to prompt and post-processing", () => { assertIncludes(buildUserPrompt("generate", { motion: "M", side: "pro", role: "conc" }, ""), "Role: conc", "generate.role"); }); +Deno.test("generate length and role controls preserve the output contract", () => { + const targets = { brief: 2, standard: 3, comprehensive: 4 } as const; + for (const length of ["brief", "standard", "comprehensive"] as const) { + for (const role of ["intro", "r1", "r2", "conc"] as const) { + const output: Record = { + exordio: { hook: "hook", transition: "transition" }, + contentions: [{}, {}, {}, {}], + metadata: {}, + }; + enforceGenerateControls(output, "pro", "custom", length, role); + assert((output.contentions as unknown[]).length === targets[length], `generate.${length} output count`); + assert((role === "intro") === ("exordio" in output), `generate.${role} exordio shape`); + } + } +}); + Deno.test("attack controls propagate to prompt and post-processing", () => { const attackOff = buildUserPrompt("attack", { caseText: "C", detectFallacies: false, analyzeWeighing: false }, ""); assertIncludes(attackOff, "Detect Fallacies: disabled", "attack.fallacies payload"); @@ -76,6 +93,32 @@ Deno.test("attack controls propagate to prompt and post-processing", () => { ); }); +Deno.test("attack toggle controls retain or clear only their requested sections", () => { + for (const detectFallacies of [true, false]) { + for (const analyzeWeighing of [true, false]) { + const output: Record = { + fallaciesDetected: [{}], + weighingAnalysis: { + currentWeighing: "x", + missingComparisons: ["y"], + improvementSuggestions: ["z"], + }, + }; + enforceAttackControls(output, detectFallacies, analyzeWeighing); + assert( + detectFallacies === ((output.fallaciesDetected as unknown[]).length > 0), + `attack.fallacies ${detectFallacies ? "enabled" : "disabled"}`, + ); + if (!analyzeWeighing) { + const weighing = output.weighingAnalysis as { currentWeighing?: string }; + assert(weighing.currentWeighing === "Weighing analysis was not requested.", "attack.weighing disabled"); + } else { + assert((output.weighingAnalysis as { currentWeighing?: string }).currentWeighing === "x", "attack.weighing enabled"); + } + } + } +}); + Deno.test("brainstorm controls propagate to prompt and post-processing", () => { assertIncludes( buildUserPrompt("brainstorm", { motion: "M", side: "pro", outputMode: "buckets" }, ""), @@ -88,6 +131,20 @@ Deno.test("brainstorm controls propagate to prompt and post-processing", () => { assertIncludes(buildUserPrompt("brainstorm", { motion: "M", side: "both", outputMode: "topfive" }, ""), "Side: both", "brainstorm.side"); }); +Deno.test("brainstorm output mode controls keep only the requested branch", () => { + for (const side of ["pro", "con", "both"] as const) { + for (const outputMode of ["topfive", "mindmap", "buckets"] as const) { + const output: Record = { topFive: [], mindmap: [], buckets: [] }; + enforceBrainstormControls(output, side, outputMode); + const expectedKey = outputMode === "topfive" ? "topFive" : outputMode; + assert(expectedKey in output, `brainstorm.${side}.${outputMode} selected output`); + for (const key of ["topFive", "mindmap", "buckets"] as const) { + if (key !== expectedKey) assert(!(key in output), `brainstorm.${side}.${outputMode} removed ${key}`); + } + } + } +}); + Deno.test("factoids controls propagate to prompt and query construction", () => { assertIncludes( buildUserPrompt("factoids", { claim: "C", region: "EU", sourceType: "government", timeframe: "2024-2026" }, ""), @@ -105,6 +162,34 @@ Deno.test("factoids controls propagate to prompt and query construction", () => assert(govRequests.some((request) => request.endPublishedDate?.startsWith("2026")), "factoids.timeframe upper bound"); }); +Deno.test("factoids matrix preserves legal, statistics, academic, recent, direction, and count effects", () => { + const legalRequests = buildSearchRequests("Is this law constitutional?", "EU", "government", "2024-2026"); + assert(legalRequests.some((request) => request.query.includes("official law court ruling")), "factoids.legal query"); + assert(legalRequests.some((request) => request.includeDomains?.includes("curia.europa.eu")), "factoids.legal domains"); + + const statisticsRequests = buildSearchRequests("What do the latest statistics show?", "Global", "government", "2025-2026"); + assert(statisticsRequests.some((request) => request.query.includes("government statistics report")), "factoids.statistics query"); + assert(statisticsRequests.some((request) => request.includeDomains?.includes("census.gov")), "factoids.statistics domains"); + + const academicRequests = buildSearchRequests("Does the literature support this?", "United States", "academic", "Recent (last 5 years)"); + assert(academicRequests.some((request) => request.category === "publication"), "factoids.academic source filter"); + + const recentRequests = buildSearchRequests("Recent evidence", "Global", "any", "2025-2026"); + assert(recentRequests.some((request) => request.startPublishedDate?.startsWith("2025")), "factoids.recent lower bound"); + assert(recentRequests.some((request) => request.endPublishedDate?.startsWith("2026")), "factoids.recent upper bound"); + + const againstPrompt = buildUserPrompt("factoids", { + claim: "MATRIX CLAIM", + direction: "against", + sourceType: "academic", + timeframe: "2025-2026", + sourceCount: 12, + }, ""); + assertIncludes(againstPrompt, "Find evidence against this statement", "factoids.direction"); + assertIncludes(againstPrompt, "Source Type: academic", "factoids.academic prompt"); + assertIncludes(againstPrompt, "Source Count: 12", "factoids.sourceCount prompt"); +}); + Deno.test("ask toggles and modes appear verbatim in the request prompt", () => { assertIncludes(buildUserPrompt("ask", { message: "Hi", mode: "clarify" }, ""), "Mode Guidance:", "ask.mode payload"); @@ -154,6 +239,83 @@ Deno.test("ask toggles and modes appear verbatim in the request prompt", () => { assert(Array.isArray(stripped.citations) && stripped.citations.length === 0, "ask citations off clears list"); }); +Deno.test("ask matrix preserves every mode, toggle, research, and output effect", () => { + const commonLanguageRule = "Match the language used in the user's question unless the user asks for another language."; + + for (const mode of ["general", "clarify"] as const) { + for (const citationMode of [false, true]) { + for (const webSearchEnabled of [false, true]) { + for (const prosConsMode of [false, true]) { + for (const argumentMapMode of [false, true]) { + for (const researchAvailable of [false, true]) { + const prompt = buildAskBasePrompt({ + message: "MATRIX QUESTION", + mode, + citationMode, + webSearchEnabled, + prosConsMode, + argumentMapMode, + researchAvailable, + }); + assertIncludes(prompt, "Message: MATRIX QUESTION", "ask.message"); + assertIncludes(prompt, `Mode: ${mode}`, `ask.${mode}.mode`); + assertIncludes(prompt, "Mode Guidance:", `ask.${mode}.guidance`); + assertIncludes(prompt, `Citation Mode: ${citationMode ? "ON" : "OFF"}`, "ask.citationMode"); + assertIncludes(prompt, `Web Search: ${webSearchEnabled ? "ENABLED" : "DISABLED"}`, "ask.webSearch"); + assertIncludes(prompt, `Pros/Cons: ${prosConsMode ? "ON" : "OFF"}`, "ask.prosCons"); + assertIncludes(prompt, `Argument Map: ${argumentMapMode ? "ON" : "OFF"}`, "ask.argumentMap"); + assertIncludes(prompt, `Research Context: ${researchAvailable ? "AVAILABLE" : "NONE"}`, "ask.research"); + + const systemPrompt = buildSystemPrompt({ + feature: "ask", + ...(researchAvailable ? { researchContext: "MATRIX RESEARCH" } : {}), + }); + assert(systemPrompt.includes(commonLanguageRule), "ask.system language rule"); + assert( + systemPrompt.includes("RESEARCH CITATION RULES:") === researchAvailable, + `ask.${researchAvailable ? "research" : "no-research"} citation rule gating`, + ); + if (researchAvailable) assert(systemPrompt.includes("MATRIX RESEARCH"), "ask.research context"); + + const output: Record = { + response: "r", + followUpQuestions: [], + uncertainties: [], + citations: [{}], + prosConsAnalysis: { pros: ["p"], cons: ["c"] }, + argumentMap: { mainClaim: "m", supportingPoints: [], counterpoints: [], assumptions: [] }, + }; + enforceAskControls(output, { mode, citationMode, prosConsMode, argumentMapMode }); + + assert( + citationMode + ? Array.isArray(output.citations) && output.citations.length === 1 + : Array.isArray(output.citations) && output.citations.length === 0, + `ask.${mode}.${citationMode ? "citation-on" : "citation-off"}`, + ); + assert( + prosConsMode === ("prosConsAnalysis" in output), + `ask.${mode}.${prosConsMode ? "proscons-on" : "proscons-off"}`, + ); + assert( + argumentMapMode === ("argumentMap" in output), + `ask.${mode}.${argumentMapMode ? "argumentmap-on" : "argumentmap-off"}`, + ); + if (mode === "clarify") { + const uncertainties = output.uncertainties as string[]; + assert( + uncertainties.includes("Clarify mode was requested, but no clarifying questions were returned."), + "ask.clarify missing follow-ups adds uncertainty", + ); + } + } + } + } + } + } + } +}); + Deno.test("only plain general ask mode is eligible for streaming", () => { for (const mode of ["clarify", "summarize", "proscons", "argumentmap"]) { assert(!isAskStreamingModeAllowed(mode), `ask.${mode} streaming bypass`); diff --git a/supabase/functions/ai/generate-honesty_test.ts b/supabase/functions/ai/generate-honesty_test.ts new file mode 100644 index 000000000..de2140032 --- /dev/null +++ b/supabase/functions/ai/generate-honesty_test.ts @@ -0,0 +1,163 @@ +const environment = { + SUPABASE_URL: "https://supabase.test", + SUPABASE_SERVICE_ROLE_KEY: "test-service-role-key", + EXA_API_KEY: "test-exa-key", + OPENAI_API_KEY: "test-openai-key", +}; + +for (const [key, value] of Object.entries(environment)) { + Deno.env.set(key, value); +} + +const { handleRequest } = await import("./index.ts"); + +function jsonResponse(body: unknown, status = 200): Response { + return new Response(JSON.stringify(body), { + status, + headers: { "Content-Type": "application/json" }, + }); +} + +const generatedOutput = { + exordio: { hook: "Hook", transition: "Transition" }, + caseStructure: { + thesis: "Thesis", + framework: "Pragmatic", + keyDefinitions: [], + burdenOfProof: "Burden", + }, + contentions: [1, 2, 3].map((id) => ({ + id: String(id), + claim: `Claim ${id}`, + reasoning: "Reasoning", + evidence: "", + needsSource: false, + sourceQuery: "", + impact: "Impact", + })), + predictedOpponentCase: { + likelyArguments: ["Argument"], + strategicWeaknesses: ["Weakness"], + keyClashes: ["Clash"], + }, + counters: [{ + targetArgument: "Argument", + counterpoint: "Response", + rebuttal: "Rebuttal", + }], + metadata: { + side: "pro", + format: "custom", + generatedAt: "CURRENT_ISO_TIMESTAMP", + }, +}; + +Deno.test("generate marks every contention when research has no citable sources", async () => { + const originalFetch = globalThis.fetch; + let providerCalls = 0; + + globalThis.fetch = + (async (input: Request | URL | string, init?: RequestInit) => { + const url = new URL(input instanceof Request ? input.url : String(input)); + + if (url.origin === "https://supabase.test") { + if (url.pathname === "/auth/v1/user") { + return jsonResponse({ id: "user-1", aud: "authenticated" }); + } + if (url.pathname.includes("/rest/v1/user_roles")) { + return jsonResponse({ role: "user" }); + } + if (url.pathname.includes("/rest/v1/user_settings")) { + return jsonResponse({ + subscription_tier: "trial", + daily_limit_override: 50, + trial_ends_at: "2099-01-01T00:00:00.000Z", + grace_period_ends_at: null, + }); + } + if (url.pathname.includes("/rest/v1/rate_limits")) { + return jsonResponse({ limit_value: 50 }); + } + if (url.pathname.includes("/rest/v1/rpc/consume_daily_quota")) { + return jsonResponse([{ allowed: true, current_count: 1 }]); + } + if (url.pathname.includes("/rest/v1/global_prompts")) { + return jsonResponse({ prompt_text: null }); + } + if (url.pathname.includes("/rest/v1/ai_requests")) { + return new Response(null, { status: 201 }); + } + throw new Error(`Unhandled Supabase request: ${url}`); + } + + if (url.href === "https://api.exa.ai/search") { + return jsonResponse({ results: [] }); + } + + if (url.href === "https://api.openai.com/v1/chat/completions") { + providerCalls += 1; + const content = providerCalls === 1 ? { queries: [] } : generatedOutput; + const requestBody = init?.body + ? JSON.parse(String(init.body)) as Record + : {}; + if ( + providerCalls === 1 && + !JSON.stringify(requestBody.messages).includes("follow-up") + ) { + throw new Error("unexpected provider request order"); + } + return jsonResponse({ + choices: [{ message: { content: JSON.stringify(content) } }], + usage: { prompt_tokens: 10, completion_tokens: 5, total_tokens: 15 }, + }); + } + + throw new Error(`Unhandled request: ${url}`); + }) as typeof fetch; + + try { + const response = await handleRequest( + new Request("http://127.0.0.1/functions/v1/ai", { + method: "POST", + headers: { + Authorization: "Bearer test-user-token", + "Content-Type": "application/json", + }, + body: JSON.stringify({ + feature: "generate", + model: "gpt-5.4-mini", + input: { + motion: "Should schools teach media literacy?", + side: "pro", + format: "custom", + role: "intro", + length: "standard", + }, + }), + }), + ); + if (response.status !== 200) { + throw new Error( + `generate handler returned ${response.status}: ${await response + .text()}`, + ); + } + + const body = await response.json() as { + output?: { contentions?: Array<{ needsSource?: boolean }> }; + }; + const contentions = body.output?.contentions ?? []; + if ( + contentions.length !== 3 || + contentions.some((contention) => contention.needsSource !== true) + ) { + throw new Error( + `missing-source honesty was not enforced: ${ + JSON.stringify(contentions) + }`, + ); + } + } finally { + globalThis.fetch = originalFetch; + } +}); diff --git a/supabase/functions/ai/index.ts b/supabase/functions/ai/index.ts index 227c27a2b..3358c9774 100644 --- a/supabase/functions/ai/index.ts +++ b/supabase/functions/ai/index.ts @@ -1588,11 +1588,17 @@ export async function handleRequest(req: Request): Promise { const aiResult = recordMainResult(await callAI(provider, model, systemPrompt, messages, temperature, req.signal)); const output = cleanGeneratedText(aiResult.output); - // Post-process: force needsSource to false since we provided research + // Preserve model evidence warnings when research exists. If research + // returned no usable material, every contention still needs a source. if (output && typeof output === "object" && "contentions" in output) { - const contentions = (output as { contentions: Array<{ needsSource?: boolean }> }).contentions; - for (const c of contentions) { - c.needsSource = false; + const contentions = (output as { contentions?: unknown }).contentions; + const hasUsableSources = getResearchSources(research).length > 0 || boundPromptSources(sources).length > 0; + if (Array.isArray(contentions) && !hasUsableSources) { + for (const contention of contentions) { + if (contention && typeof contention === "object") { + Object.assign(contention, { needsSource: true }); + } + } } } diff --git a/supabase/functions/ai/prompt-builder.test.ts b/supabase/functions/ai/prompt-builder.test.ts index 31c4fcd35..23309b15e 100644 --- a/supabase/functions/ai/prompt-builder.test.ts +++ b/supabase/functions/ai/prompt-builder.test.ts @@ -12,6 +12,31 @@ function assertIncludes(text: string, expected: string) { } } +function assertNotIncludes(text: string, unexpected: string) { + if (text.includes(unexpected)) { + throw new Error(`Expected prompt not to include "${unexpected}".\nPrompt:\n${text}`); + } +} + +const crossfireConfig = { + resolution: "Cities should ban private cars downtown.", + argument: "Private cars create congestion and unsafe streets.", + mode: "question-ai" as const, + style: "cooperative" as const, + durationSeconds: 180 as const, + useSourceLibrary: false, + seedQuestions: ["What is the mechanism?"], +}; + +const crossfireTranscript = [{ + id: "T1", + sequence: 1, + speaker: "user" as const, + kind: "question" as const, + text: "What is the mechanism?", + createdAt: 1, +}]; + Deno.test("generate prompt includes language and generation controls", () => { const prompt = buildUserPrompt("generate", { motion: "This house would regulate social media algorithms", @@ -38,6 +63,36 @@ Deno.test("generate prompt includes language and generation controls", () => { assertIncludes(prompt, "SEARCH CONTEXT"); }); +Deno.test("generate prompt matrix preserves every school role and length rule", () => { + const lengths = [ + ["brief", "2"], + ["standard", "3"], + ["comprehensive", "4"], + ] as const; + const roles = ["intro", "r1", "r2", "conc"] as const; + + for (const role of roles) { + for (const [length, contentionCount] of lengths) { + const prompt = buildUserPrompt("generate", { + motion: "MATRIX MOTION", + side: "con", + format: "custom", + role, + length, + audience: "advanced", + tone: "conversational", + language: "hu", + }, "RESEARCH CONTEXT"); + + assertIncludes(prompt, `Role: ${role}`); + assertIncludes(prompt, `Length: ${length}`); + assertIncludes(prompt, `Length Instruction: Return exactly ${contentionCount} contentions`); + assertIncludes(prompt, "Language: Hungarian (hu)"); + assertIncludes(prompt, "RESEARCH CONTEXT"); + } + } +}); + Deno.test("attack prompt includes intensity, paradigm, and toggle states", () => { const prompt = buildUserPrompt("attack", { caseText: "Nuclear energy should replace fossil fuels.", @@ -55,6 +110,37 @@ Deno.test("attack prompt includes intensity, paradigm, and toggle states", () => assertIncludes(prompt, "SEARCH CONTEXT"); }); +Deno.test("attack prompt matrix preserves intensity, every judge paradigm, and toggles", () => { + const intensities = ["moderate", "ruthless"] as const; + const paradigms = ["logic", "values", "policy"] as const; + const toggleStates = [ + [true, true], + [true, false], + [false, true], + [false, false], + ] as const; + + for (const aggressiveness of intensities) { + for (const judgeParadigm of paradigms) { + for (const [detectFallacies, analyzeWeighing] of toggleStates) { + const prompt = buildUserPrompt("attack", { + caseText: "MATRIX CASE", + aggressiveness, + judgeParadigm, + detectFallacies, + analyzeWeighing, + }, "RESEARCH CONTEXT"); + + assertIncludes(prompt, `Attack Intensity: ${aggressiveness}`); + assertIncludes(prompt, `Judge Paradigm: ${judgeParadigm}`); + assertIncludes(prompt, `Detect Fallacies: ${detectFallacies ? "enabled" : "disabled"}`); + assertIncludes(prompt, `Analyze Weighing: ${analyzeWeighing ? "enabled" : "disabled"}`); + assertIncludes(prompt, 'CASE TEXT:\n"MATRIX CASE"'); + } + } + } +}); + Deno.test("brainstorm prompt includes side, output mode, and context", () => { const prompt = buildUserPrompt("brainstorm", { motion: "Ban private cars in city centers", @@ -70,6 +156,22 @@ Deno.test("brainstorm prompt includes side, output mode, and context", () => { assertIncludes(prompt, "SEARCH CONTEXT"); }); +Deno.test("brainstorm prompt matrix preserves both one-side and comparative inputs", () => { + for (const side of ["pro", "con", "both"] as const) { + for (const outputMode of ["topfive", "mindmap", "buckets"] as const) { + const prompt = buildUserPrompt("brainstorm", { + motion: "MATRIX MOTION", + side, + outputMode, + }, "RESEARCH CONTEXT"); + + assertIncludes(prompt, `Side: ${side}`); + assertIncludes(prompt, `Output Mode: ${outputMode}`); + assertIncludes(prompt, "RESEARCH CONTEXT"); + } + } +}); + Deno.test("factoids prompt includes source filters and language", () => { const prompt = buildUserPrompt("factoids", { claim: "Solar is cheaper than coal", @@ -86,6 +188,48 @@ Deno.test("factoids prompt includes source filters and language", () => { assertIncludes(prompt, "Language: English (en)"); }); +Deno.test("factoids prompt matrix preserves legal, statistics, academic, recent, direction, and count inputs", () => { + const fixtures = [ + { + claim: "Is this law constitutional?", + direction: "against", + region: "EU", + sourceType: "government", + timeframe: "2024-2026", + sourceCount: 9, + language: "hu", + }, + { + claim: "What do the latest statistics show?", + direction: "for", + region: "Global", + sourceType: "government", + timeframe: "2025-2026", + sourceCount: 12, + language: "en", + }, + { + claim: "Does the academic literature support this claim?", + direction: "for", + region: "United States", + sourceType: "academic", + timeframe: "Recent (last 5 years)", + sourceCount: 5, + language: "es", + }, + ] as const; + + for (const fixture of fixtures) { + const prompt = buildUserPrompt("factoids", fixture, "RESEARCH CONTEXT"); + assertIncludes(prompt, `Find evidence ${fixture.direction === "against" ? "against" : "in favor of"} this statement: "${fixture.claim}"`); + assertIncludes(prompt, `Region: ${fixture.region}`); + assertIncludes(prompt, `Source Type: ${fixture.sourceType}`); + assertIncludes(prompt, `Timeframe: ${fixture.timeframe}`); + assertIncludes(prompt, `Source Count: ${fixture.sourceCount}`); + assertIncludes(prompt, `Language: ${fixture.language === "hu" ? "Hungarian" : fixture.language === "es" ? "Spanish" : "English"} (${fixture.language})`); + } +}); + Deno.test("ask prompt includes mode guidance", () => { const prompt = buildUserPrompt("ask", { message: "Help me prep cross-ex questions.", @@ -96,6 +240,23 @@ Deno.test("ask prompt includes mode guidance", () => { assertIncludes(prompt, "Mode Guidance: Map the claim, supports, counters, and assumptions explicitly and include argumentMap."); }); +Deno.test("ask prompt matrix preserves general and clarify modes", () => { + const fixtures = [ + ["general", "Give a direct coaching answer", "en"], + ["clarify", "Define unclear terms", "es"], + ] as const; + for (const [mode, guidance, language] of fixtures) { + const prompt = buildUserPrompt("ask", { + message: "MATRIX QUESTION", + mode, + language, + }, ""); + assertIncludes(prompt, `Mode: ${mode}`); + assertIncludes(prompt, `Mode Guidance: ${guidance}`); + assertNotIncludes(prompt, "RESEARCH CITATION RULES:"); + } +}); + Deno.test("ask mode guidance falls back to general", () => { const guidance = getAskModeGuidance("unknown-mode"); if (!guidance.includes("direct coaching answer")) { @@ -124,6 +285,35 @@ Deno.test("sparring prompt carries the session controls and bounded transcript", assertIncludes(prompt, "The user argument"); }); +Deno.test("sparring prompt matrix preserves school, LD, and PF controls", () => { + const fixtures = [ + ["standard", "introductor", "Pro Introductor"], + ["ld", "constructive", "Affirmative constructive"], + ["pf", "final", "Pro final focus"], + ] as const; + + for (const [format, kind, label] of fixtures) { + const prompt = buildUserPrompt("sparring", { + action: "respond", + sessionId: "00000000-0000-4000-8000-000000000001", + motion: "MATRIX MOTION", + userSide: "con", + format, + difficulty: "hard", + language: "hu", + currentTurn: { id: `${format}-turn`, label, side: "pro", kind, timeLimitSeconds: 180 }, + speeches: [], + }, ""); + + assertIncludes(prompt, `Format: ${format}`); + assertIncludes(prompt, "Difficulty: hard"); + assertIncludes(prompt, "Language: Hungarian (hu)"); + assertIncludes(prompt, `"label":"${label}"`); + assertIncludes(prompt, `"kind":"${kind}"`); + assertNotIncludes(prompt, "RESEARCH CONTEXT"); + } +}); + Deno.test("sparring prompt drops the oldest speeches when the transcript exceeds its bound", () => { const speeches = Array.from({ length: 12 }, (_, index) => ({ content: `${index === 0 ? "OLDEST_MARKER" : `speech-${index + 1}`} ${"x".repeat(5_980)}`, @@ -189,6 +379,27 @@ Deno.test("crossfire next-turn prompt carries the expected reply and real turn I assertIncludes(prompt, "same language as the user's resolution"); }); +Deno.test("crossfire next-turn builder preserves all three style controls", () => { + const styleGuidance = { + cooperative: "Do not force a concession", + evasive: "Answer narrowly", + aggressive: "Do not insult, threaten, harass", + } as const; + + for (const style of ["cooperative", "evasive", "aggressive"] as const) { + const prompt = buildCrossfireNextTurnPrompt({ + config: { ...crossfireConfig, style }, + transcript: crossfireTranscript, + expectedReply: "answer", + }); + assertIncludes(prompt, `Style: ${style}`); + assertIncludes(prompt, styleGuidance[style]); + assertIncludes(prompt, 'The next model reply must be type "answer"'); + assertIncludes(prompt, "turn-id=T1"); + assertNotIncludes(prompt, "RESEARCH CITATION RULES:"); + } +}); + Deno.test("crossfire debrief prompt requires cited IDs and omits typed-session scoring", () => { const prompt = buildCrossfireDebriefPrompt({ config: { diff --git a/supabase/functions/ai/prompt-builder.ts b/supabase/functions/ai/prompt-builder.ts index b29a3abc9..6a2c0ff31 100644 --- a/supabase/functions/ai/prompt-builder.ts +++ b/supabase/functions/ai/prompt-builder.ts @@ -258,7 +258,7 @@ ${searchContext} Region: ${asString(input.region, "Global")} Source Type: ${asString(input.sourceType, "any")} Timeframe: ${asString(input.timeframe, "Recent (last 5 years)")} -Source Count: ${asString(input.sourceCount, "5")} +Source Count: ${typeof input.sourceCount === "number" && Number.isFinite(input.sourceCount) ? input.sourceCount : 5} Language: ${languageLabel(asString(input.language, "en"))} (${asString(input.language, "en")})`; } From a1f4fdd31772cb5746ef015f3e6a0715b9d48059 Mon Sep 17 00:00:00 2001 From: Adam Date: Wed, 9 Sep 2026 16:25:39 +0000 Subject: [PATCH 2/2] fix: close prompt review gaps and verify protected staging --- e2e/staging/auth.setup.ts | 1 - e2e/staging/authed-flows.spec.ts | 4 +- package.json | 2 +- plans/prompt-eval-sheet.md | 52 ++++---- plans/system-prompt-improvement.md | 2 +- src/lib/ai/schemas/factoids.ts | 2 +- src/test/schemas.test.ts | 4 + .../_shared/control-enforcement.test.ts | 1 + .../functions/_shared/control-enforcement.ts | 4 +- supabase/functions/_shared/prompts.test.ts | 7 ++ supabase/functions/_shared/prompts.ts | 2 +- supabase/functions/_shared/research.test.ts | 8 ++ supabase/functions/_shared/research.ts | 2 + supabase/functions/ai-stream/handler_test.ts | 106 ++++++++++++++++ supabase/functions/ai-stream/prompt.test.ts | 2 +- supabase/functions/ai/control-matrix.test.ts | 2 +- .../functions/ai/generate-honesty_test.ts | 46 ++++++- .../functions/ai/handler-research.test.ts | 113 ++++++++++++++++++ supabase/functions/ai/index.ts | 32 +++-- 19 files changed, 345 insertions(+), 47 deletions(-) diff --git a/e2e/staging/auth.setup.ts b/e2e/staging/auth.setup.ts index a0b125623..83577245c 100644 --- a/e2e/staging/auth.setup.ts +++ b/e2e/staging/auth.setup.ts @@ -118,7 +118,6 @@ setup('authenticate with the protected staging demo code', async ({ page }) => { auth: { persistSession: false, autoRefreshToken: false, detectSessionInUrl: false }, }); const { data, error } = await supabase.auth.verifyOtp({ - email: handoff.email, token_hash: handoff.tokenHash, type: 'magiclink', }); diff --git a/e2e/staging/authed-flows.spec.ts b/e2e/staging/authed-flows.spec.ts index 8a72806bc..897ba58ea 100644 --- a/e2e/staging/authed-flows.spec.ts +++ b/e2e/staging/authed-flows.spec.ts @@ -58,7 +58,7 @@ test('real staging session persists, writes and removes a source, and completes const saveResponseButton = page.getByRole('button', { name: 'Save message' }).last(); await expect(saveResponseButton).toBeAttached({ timeout: 90_000 }); const assistantBubble = saveResponseButton.locator('..'); - await expect(assistantBubble.locator(':scope > .text-sm')).toHaveText(/\S/, { timeout: 90_000 }); + await expect(assistantBubble.locator(':scope > div.text-sm')).toHaveText(/\S/, { timeout: 90_000 }); } finally { if (sourceCreated) { await page.goto('/app/library'); @@ -66,7 +66,7 @@ test('real staging session persists, writes and removes a source, and completes const sourceDelete = page.waitForResponse((response) => response.request().method() === 'DELETE' && response.url().includes('/rest/v1/sources?') ); - await page.getByRole('button', { name: `Delete ${sourceTitle}` }).click(); + await page.getByRole('button', { name: `Delete ${sourceTitle}`, exact: true }).click(); expect((await sourceDelete).ok()).toBe(true); await page.evaluate(() => localStorage.removeItem('debate-trainer-storage')); await page.reload(); diff --git a/package.json b/package.json index 6954aa357..d0f739477 100644 --- a/package.json +++ b/package.json @@ -18,7 +18,7 @@ "preview": "vite preview", "test": "vitest run", "test:watch": "vitest", - "test:edge": "deno test --allow-read --allow-env=DEMO_CODE_PEPPER,EXA_API_KEY,SUPABASE_URL,SUPABASE_SERVICE_ROLE_KEY,SPARRING_INTEGRITY_SECRET,OPENAI_API_KEY --allow-net=127.0.0.1 supabase/functions/", + "test:edge": "deno test --allow-read --allow-env=DEMO_CODE_PEPPER,EXA_API_KEY,SUPABASE_URL,SUPABASE_SERVICE_ROLE_KEY,SPARRING_INTEGRITY_SECRET,OPENAI_API_KEY,GOOGLE_API_KEY,ANTHROPIC_API_KEY --allow-net=127.0.0.1 supabase/functions/", "test:db": "supabase test db", "test:e2e": "playwright test --project=desktop-chromium --project=mobile-chromium", "test:e2e:webkit": "playwright test --project=webkit", diff --git a/plans/prompt-eval-sheet.md b/plans/prompt-eval-sheet.md index 1b5112505..882f31fe6 100644 --- a/plans/prompt-eval-sheet.md +++ b/plans/prompt-eval-sheet.md @@ -1,10 +1,10 @@ -# Prompt evaluation v1 +# Prompt evaluation v2 Evaluation started 2026-09-07 UTC, resumed 2026-09-09 UTC. This is subjective agent assessment of synthetic model responses, not an automated quality gate or a human panel review. ## Reproduction -Before revision: `235bb49`. After revision: the commit introducing this sheet and the shared Ask rules. Use the same request configuration for both revisions. Only the system prompt changes. +Before revision: `235bb49`. After revision: the final shared prompt version committed with this v2 sheet, identified by the system-prompt hashes below. Use the same request configuration for both revisions. Only the system prompt changes. - Endpoint: `https://openrouter.ai/api/v1/chat/completions`. - Model: `openai/gpt-4o-mini`, temperature `0`, seed `42`, `max_tokens: 1600`. @@ -17,7 +17,7 @@ Before revision: `235bb49`. After revision: the commit introducing this sheet an - Markdown user: the synthetic message directly. The model was sampled with the streaming system prompt through a complete-response API call. SSE transport is tested separately by the automated streaming suite. - Validate unmodified structured output with `JSON.parse` and `AskOutputSchema.safeParse`. Do not repair omitted keys before recording validity. -Each of the three motions is crossed with all three complexity instructions, in both Markdown and JSON, before and after. That is 36 responses. Message construction is exactly: +Each of the three motions is crossed with all three complexity instructions, in both Markdown and JSON, before and after. That is 36 responses in the final comparison. An earlier 18-response after batch preceded the review fixes, for 54 completed calls overall; the after observations below use only the final batch. Message construction is exactly: ```text Motion: {motion} @@ -42,8 +42,8 @@ System-prompt SHA-256 hashes used in this evaluation: - before json: `aaa82b8400622789bd0d2ee05eeb69b24c47c9e30faadcadbbc281538a268b49` - before stream: `dca86dd2269438966851e44c7f6c20a7b8007be69a7168c8dfc6b5b620037542` -- after json: `317fe0678d076d9e2e14500b77701ec4cfa40ef2f51f6d35fcc866d5629ab4d3` -- after stream: `e4577933078e172164041a24df0857c69e8d899e522136bcec6c788a75153bf2` +- after json: `3ef82d881b727f3f1d894dd182efdea4d38e02414ecc7f4d2cbe8c94c100b325` +- after stream: `07f5c00df06c542c29729918374fde57c518a2006d9170d2a0f280d99a3fd3d4` ## Assessment rules @@ -76,37 +76,37 @@ All 18 responses completed without truncation. All nine JSON outputs parsed, but ## After observations -All 18 after responses completed without truncation. One transport disconnect was retried with the identical request. All nine structured responses parsed and passed `AskOutputSchema`; all nine Markdown responses stayed freeform. All responses kept the requested language and none invented named citations, URLs, studies, or statistics. +All 18 final after responses completed without truncation. All nine structured responses parsed and passed `AskOutputSchema`; all nine Markdown responses stayed freeform. All responses kept the requested language and none invented named citations, URLs, studies, or statistics. -The schema improvement is observed in this sample, not proof of a reliable rate increase. Coaching quality did not improve consistently. After JSON responses used empty follow-up and uncertainty arrays throughout, even where useful questions or caveats could have been included. Two responses included disabled optional analysis fields as empty objects; the existing server post-processing removes those fields. No schema or control enforcement was weakened. +The schema improvement is observed in this sample, not proof of a reliable rate increase. Coaching quality did not improve consistently. All after JSON responses used empty uncertainty arrays; only the beginner phones response supplied follow-up questions. Four responses included disabled optional analysis fields, including a populated argument map in the beginner phones response. Existing server post-processing removes these fields. No schema or control enforcement was weakened. | Run | Quality | Controls | Evidence honesty | Language | Risk | JSON syntax/schema | Observation | | --- | --- | --- | --- | --- | --- | --- | --- | -| phones beginner Markdown | 3 | 4 | Generalizations | English | Medium | N/A | Clear clash; same unsupported learning-outcome assumptions. | -| phones beginner JSON | 4 | 4 | Generalizations | English | Medium | Pass/pass | Useful explanation; no follow-up or uncertainty detail. | -| phones intermediate Markdown | 3 | 4 | Generalizations | English | Medium | N/A | Gives a judge test, but priority claim remains weak. | -| phones intermediate JSON | 3 | 4 | Generalizations | English | Medium | Pass/pass | Removes the absolute distractions claim; alludes to unspecified anecdotes. | -| phones advanced Markdown | 4 | 4 | Conditional claims | English | Low | N/A | Adds likelihood comparison and a clear regulated-use model. | -| phones advanced JSON | 3 | 4 | Generalizations | English | Medium | Pass/pass | Required arrays retained; successful case studies remain unspecified. | -| transit beginner Markdown | 4 | 4 | Conditional claims | Spanish | Low | N/A | Useful access versus funding/crowding comparison. | -| transit beginner JSON | 4 | 4 | Conditional claims | Spanish | Low | Pass/pass | Drops the speculative quality-perception argument. | -| transit intermediate Markdown | 2 | 3 | Generalizations | Spanish | Medium | N/A | Still treats taxation and subsidies as competing mechanisms. | -| transit intermediate JSON | 2 | 3 | Generalizations | Spanish | Medium | Pass/pass | Compares free fares with taxation, which can coexist. | -| transit advanced Markdown | 4 | 4 | Conditional claims | Spanish | Low | N/A | Targeted concessions and funding uncertainty are concrete. | -| transit advanced JSON | 3 | 3 | Conditional claims | Spanish | Low | Pass/pass | Includes disabled optional analysis objects; server removes them. | -| ai beginner Markdown | 3 | 4 | Generalizations | Hungarian | Medium | N/A | Learning-support argument assumes information-search capability. | -| ai beginner JSON | 4 | 4 | Conditional claims | Hungarian | Low | Pass/pass | Clear support versus independence clash. | +| phones beginner Markdown | 3 | 4 | Generalizations | English | Medium | N/A | Clear clash; unsupported learning-outcome assumptions remain. | +| phones beginner JSON | 3 | 2 | Generalizations | English | Medium | Pass/pass | Emits disabled argument map and pros/cons; server removes them. | +| phones intermediate Markdown | 3 | 4 | Generalizations | English | Medium | N/A | Gives a judge test, but prioritization remains weak. | +| phones intermediate JSON | 3 | 4 | Overstatement | English | Medium | Pass/pass | Again claims a ban eliminates distractions entirely. | +| phones advanced Markdown | 3 | 4 | Generalizations | English | Medium | N/A | Clear regulated-use model; impact comparison stays broad. | +| phones advanced JSON | 3 | 3 | Generalizations | English | Medium | Pass/pass | Required arrays retained; emits disabled empty analysis fields. | +| transit beginner Markdown | 4 | 4 | Conditional claims | Spanish | Low | N/A | Useful access versus financing comparison. | +| transit beginner JSON | 4 | 4 | Conditional claims | Spanish | Low | Pass/pass | Explains mobility benefits and budget displacement. | +| transit intermediate Markdown | 2 | 3 | Generalizations | Spanish | Medium | N/A | Taxes and subsidies remain overlapping mechanisms. | +| transit intermediate JSON | 3 | 4 | Speculation | Spanish | Medium | Pass/pass | Distinct route-optimization mechanism; savings are unproven. | +| transit advanced Markdown | 4 | 4 | Conditional claims | Spanish | Low | N/A | Targeted fares countermodel and funding uncertainty. | +| transit advanced JSON | 3 | 3 | Conditional claims | Spanish | Low | Pass/pass | Funding objection substitutes for a policy countermodel; disabled fields emitted. | +| ai beginner Markdown | 3 | 4 | Generalizations | Hungarian | Medium | N/A | Understandable, but translates motion awkwardly as physical movement. | +| ai beginner JSON | 4 | 4 | Conditional claims | Hungarian | Low | Pass/pass | Clear learning support versus independence clash. | | ai intermediate Markdown | 3 | 3 | Generalizations | Hungarian | Medium | N/A | Policy labels still substitute for developed mechanisms. | -| ai intermediate JSON | 3 | 3 | Generalizations | Hungarian | Medium | Pass/pass | Includes disabled optional analysis objects; server removes them. | -| ai advanced Markdown | 2 | 3 | Speculation | Hungarian | Medium | N/A | Mental-health impact remains unsupported and unexplained. | -| ai advanced JSON | 3 | 4 | Generalizations | Hungarian | Medium | Pass/pass | An objection substitutes for a developed countermodel. | +| ai intermediate JSON | 3 | 4 | Generalizations | Hungarian | Medium | Pass/pass | Concrete judge tests; conflates generated work and plagiarism. | +| ai advanced Markdown | 3 | 3 | Speculation | Hungarian | Medium | N/A | Discusses motivation risk without a developed causal link. | +| ai advanced JSON | 2 | 3 | Generalizations | Hungarian | Medium | Pass/pass | Awkward burden-of-proof terminology; disabled empty fields emitted. | ### Assessment The narrow contract change is supported: streaming remains Markdown, language stays consistent, and required JSON fields are present in all nine after samples. This baseline does not support a claim of better coaching overall. Retain schema validation, control post-processing and evidence warnings; do not begin provider-specific tuning from these results alone. -All 36 completed calls reported provider `OpenAI`, model `openai/gpt-4o-mini`. The exact prompts, synthetic inputs, request hashes and per-response assessments make the comparison repeatable, although model/backend revisions can change later outputs. +All 54 completed calls reported provider `OpenAI`, model `openai/gpt-4o-mini`. The exact prompts, synthetic inputs, request hashes and per-response assessments make the comparison repeatable, although model/backend revisions can change later outputs. ## Limits -These calls exercise prompt behavior through OpenRouter, not the app's native OpenAI provider connection or authenticated Supabase handler. Nine motion/complexity pairs do not establish provider-wide reliability. Language was requested explicitly, so these samples do not prove automatic language inference. No tuning for a particular provider is justified by this small sample alone. Raw responses and credentials are not committed. Synthetic response artifacts and request hashes are retained privately under `/tmp/debatelab-prompt-eval-20260907` on the execution host. +These calls exercise prompt behavior through OpenRouter, not the app's native OpenAI provider connection or authenticated Supabase handler. Nine motion/complexity pairs do not establish provider-wide reliability. Language was requested explicitly, so these samples do not prove automatic language inference. No tuning for a particular provider is justified by this small sample alone. Generate’s `needsSource` field is a model-authored warning, not a verified-source badge. Available sources do not establish support for every contention; this work preserves model warnings and forces warnings when no eligible evidence exists, without claiming semantic verification. One transport disconnect in the intermediate batch was retried with the identical request. Raw responses and credentials are not committed. Synthetic response artifacts and request hashes are retained privately under `/tmp/debatelab-prompt-eval-20260907` on the execution host. diff --git a/plans/system-prompt-improvement.md b/plans/system-prompt-improvement.md index 1e943fb40..e992b279b 100644 --- a/plans/system-prompt-improvement.md +++ b/plans/system-prompt-improvement.md @@ -1,6 +1,6 @@ # System prompt improvement plan -**Status:** Implementation and manual baseline completed on the isolated `t3code/system-prompt-improvements` branch, based on `235bb49`. Release verification and rollout are tracked in the PR. See [prompt-eval-sheet.md](prompt-eval-sheet.md) for the dated 36-response comparison and its limits. +**Status:** Implementation and manual baseline completed on the isolated `t3code/system-prompt-improvements` branch, based on `235bb49`. Credential-free and protected staging verification passed. Release verification and rollout are tracked in PR #42. See [prompt-eval-sheet.md](prompt-eval-sheet.md) for the dated 36-response comparison and its limits. ## Preconditions diff --git a/src/lib/ai/schemas/factoids.ts b/src/lib/ai/schemas/factoids.ts index 232d121ad..0004bf524 100644 --- a/src/lib/ai/schemas/factoids.ts +++ b/src/lib/ai/schemas/factoids.ts @@ -8,7 +8,7 @@ export const FactoidsInputSchema = z.object({ region: z.string().optional(), sourceType: z.enum(['academic', 'government', 'news', 'any']), timeframe: z.string().optional(), - sourceCount: z.coerce.number().min(3).max(15).default(5), + sourceCount: z.coerce.number().int().min(3).max(15).default(5), }); export type FactoidsInput = z.infer; diff --git a/src/test/schemas.test.ts b/src/test/schemas.test.ts index e19060b29..39785c83a 100644 --- a/src/test/schemas.test.ts +++ b/src/test/schemas.test.ts @@ -442,6 +442,10 @@ describe('Factoids Schema', () => { expect(FactoidsInputSchema.safeParse({ ...validInput, sourceType: 'blog' }).success).toBe(false); }); + it('rejects fractional source counts', () => { + expect(FactoidsInputSchema.safeParse({ ...validInput, sourceCount: 3.5 }).success).toBe(false); + }); + it('accepts all valid sourceTypes', () => { expect(FactoidsInputSchema.safeParse({ ...validInput, sourceType: 'academic' }).success).toBe(true); expect(FactoidsInputSchema.safeParse({ ...validInput, sourceType: 'government' }).success).toBe(true); diff --git a/supabase/functions/_shared/control-enforcement.test.ts b/supabase/functions/_shared/control-enforcement.test.ts index 8dbb9a620..0bd2f73a9 100644 --- a/supabase/functions/_shared/control-enforcement.test.ts +++ b/supabase/functions/_shared/control-enforcement.test.ts @@ -81,6 +81,7 @@ Deno.test("factoids typed controls enforce their existing boundaries", () => { for (const input of [ { sourceCount: "5" }, + { sourceCount: 3.5 }, { sourceCount: Number.NaN }, { sourceCount: Number.POSITIVE_INFINITY }, { sourceCount: 2 }, diff --git a/supabase/functions/_shared/control-enforcement.ts b/supabase/functions/_shared/control-enforcement.ts index 2a761ac1a..d7be24a0f 100644 --- a/supabase/functions/_shared/control-enforcement.ts +++ b/supabase/functions/_shared/control-enforcement.ts @@ -64,8 +64,8 @@ export function validateControlInput(feature: string, input: Record 15)) { - return "factoids.sourceCount must be a number from 3 to 15"; + (typeof input.sourceCount !== "number" || !Number.isInteger(input.sourceCount) || input.sourceCount < 3 || input.sourceCount > 15)) { + return "factoids.sourceCount must be an integer from 3 to 15"; } } diff --git a/supabase/functions/_shared/prompts.test.ts b/supabase/functions/_shared/prompts.test.ts index 38698a03b..7853864b4 100644 --- a/supabase/functions/_shared/prompts.test.ts +++ b/supabase/functions/_shared/prompts.test.ts @@ -125,6 +125,13 @@ Deno.test("product identity includes the humanizer style rules", () => { if (!PRODUCT_IDENTITY.includes("HUMANIZER STYLE INSTRUCTIONS")) throw new Error("humanizer missing from identity layer"); }); +Deno.test("shared language rules honor explicit output-language instructions", () => { + const prompt = buildSystemPrompt({ feature: "generate" }); + if (!prompt.includes("Follow any explicit output-language instruction. Otherwise match the language used by the user.")) { + throw new Error("explicit output-language rule missing"); + } +}); + Deno.test("sparring system prompt defines difficulty and transcript-only scoring", () => { const prompt = buildSystemPrompt({ feature: "sparring", dateContext: "CURRENT DATE: Testdate" }); if (!prompt.includes("Difficulty is a real control")) throw new Error("difficulty guidance missing"); diff --git a/supabase/functions/_shared/prompts.ts b/supabase/functions/_shared/prompts.ts index 796fd2b16..f6b1b3732 100644 --- a/supabase/functions/_shared/prompts.ts +++ b/supabase/functions/_shared/prompts.ts @@ -14,7 +14,7 @@ export type Feature = "generate" | "attack" | "brainstorm" | "factoids" | "ask" | "sparring" | "crossfire"; export const ASK_COMMON_RULES = `You are DebateLab's debate coach. -Match the language used in the user's question unless the user asks for another language. +Follow any explicit output-language instruction. Otherwise match the language used by the user. Never invent facts, statistics, citations, or sources.`; export const PRODUCT_IDENTITY = `DEBATELAB PRODUCT IDENTITY: diff --git a/supabase/functions/_shared/research.test.ts b/supabase/functions/_shared/research.test.ts index 74ab208ac..aea2f07ad 100644 --- a/supabase/functions/_shared/research.test.ts +++ b/supabase/functions/_shared/research.test.ts @@ -1,6 +1,7 @@ import { applyFactoidsVerification, buildDiscoveredSources, + buildResearchContext, deepResearch, emptyResearchContext, exaGetContents, @@ -16,6 +17,13 @@ function ctxWith(overrides: Partial): ResearchContext { return { ...emptyResearchContext(), ...overrides }; } +Deno.test("empty research does not render a research context block", () => { + const rendered = buildResearchContext(ctxWith({ + searches: [{ query: "empty search", results: "" }], + })); + if (rendered !== "") throw new Error(`empty research rendered unexpected context: ${rendered}`); +}); + Deno.test("factoid citations backed by strong or moderate fetched sources stay verified", () => { const research = ctxWith({ readContent: [ diff --git a/supabase/functions/_shared/research.ts b/supabase/functions/_shared/research.ts index 9a13e3241..09a03e126 100644 --- a/supabase/functions/_shared/research.ts +++ b/supabase/functions/_shared/research.ts @@ -564,6 +564,8 @@ export async function deepResearch( } export function buildResearchContext(research: ResearchContext): string { + if (research.rankedResults.length === 0 && research.readContent.length === 0) return ""; + let ctx = "=== WEB RESEARCH RESULTS ===\n\n"; ctx += "IMPORTANT: Only cite sources marked 'strong' or 'moderate'. Sources marked 'weak' are fallback context only, do NOT cite them as evidence. If only weak sources are available, state that evidence is limited.\n\n"; diff --git a/supabase/functions/ai-stream/handler_test.ts b/supabase/functions/ai-stream/handler_test.ts index 6def3b543..5de1a44d7 100644 --- a/supabase/functions/ai-stream/handler_test.ts +++ b/supabase/functions/ai-stream/handler_test.ts @@ -1,6 +1,7 @@ const testEnvironment = { SUPABASE_URL: "https://supabase.test", SUPABASE_SERVICE_ROLE_KEY: "test-service-role-key", + OPENAI_API_KEY: "test-openai-key", }; for (const [key, value] of Object.entries(testEnvironment)) { @@ -36,6 +37,111 @@ Deno.test("streaming Ask message construction keeps Markdown output and mode gui } }); +Deno.test("streaming Ask forwards the shared prompt and returns provider SSE", async () => { + const originalFetch = globalThis.fetch; + let providerRequest: Record | undefined; + const upstream = [ + 'data: {"choices":[{"delta":{"content":"Szia"}}]}\n\n', + 'data: {"choices":[{"delta":{"content":" Adam"}}]}\n\n', + 'data: {"usage":{"prompt_tokens":4,"completion_tokens":2,"total_tokens":6}}\n\n', + "data: [DONE]\n\n", + ].join(""); + + globalThis.fetch = (async (input: Request | URL | string, init?: RequestInit) => { + const url = new URL(input instanceof Request ? input.url : String(input)); + + if (url.origin === "https://supabase.test") { + if (url.pathname === "/auth/v1/user") { + return new Response(JSON.stringify({ id: "user-1", aud: "authenticated" }), { + headers: { "Content-Type": "application/json" }, + }); + } + if (url.pathname.includes("/rest/v1/user_roles")) { + return new Response(JSON.stringify({ role: "user" }), { + headers: { "Content-Type": "application/json" }, + }); + } + if (url.pathname.includes("/rest/v1/user_settings")) { + return new Response(JSON.stringify({ + subscription_tier: "trial", + daily_limit_override: 50, + trial_ends_at: "2099-01-01T00:00:00.000Z", + grace_period_ends_at: null, + }), { headers: { "Content-Type": "application/json" } }); + } + if (url.pathname.includes("/rest/v1/rate_limits")) { + return new Response(JSON.stringify({ limit_value: 50 }), { + headers: { "Content-Type": "application/json" }, + }); + } + if (url.pathname.includes("/rest/v1/rpc/consume_daily_quota")) { + return new Response(JSON.stringify([{ allowed: true, current_count: 1 }]), { + headers: { "Content-Type": "application/json" }, + }); + } + if (url.pathname.includes("/rest/v1/ai_requests")) { + return new Response(null, { status: 201 }); + } + throw new Error(`Unexpected Supabase request: ${url}`); + } + + if (url.href === "https://api.openai.com/v1/chat/completions") { + providerRequest = JSON.parse(String(init?.body)) as Record; + return new Response(upstream, { + headers: { "Content-Type": "text/event-stream" }, + }); + } + + throw new Error(`Unexpected request: ${url}`); + }) as typeof fetch; + + try { + const response = await handleRequest( + new Request("http://127.0.0.1/functions/v1/ai-stream", { + method: "POST", + headers: { + Authorization: "Bearer test-user-token", + "Content-Type": "application/json", + }, + body: JSON.stringify({ + feature: "ask", + model: "gpt-4o-mini", + input: { + message: "¿Cómo respondo a este argumento?", + history: [], + mode: "general", + }, + }), + }), + ); + + if (response.status !== 200) { + throw new Error(`streaming handler returned ${response.status}: ${await response.text()}`); + } + if (!response.headers.get("Content-Type")?.startsWith("text/event-stream")) { + throw new Error("streaming handler did not return SSE"); + } + + const body = await response.text(); + assertIncludes(body, 'event: delta\ndata: {"text":"Szia"}'); + assertIncludes(body, 'event: delta\ndata: {"text":" Adam"}'); + assertIncludes(body, "event: final"); + assertIncludes(body, '"response":"Szia Adam"'); + + const messages = providerRequest?.messages as Array<{ role: string; content: string }> | undefined; + const system = messages?.find((message) => message.role === "system")?.content ?? ""; + assertIncludes(system, "Answer in clear Markdown that can stream smoothly token by token."); + assertIncludes(system, "Follow any explicit output-language instruction."); + assertIncludes(system, "Otherwise match the language used by the user."); + assertIncludes(system, "Never invent facts, statistics, citations, or sources."); + if (messages?.at(-1)?.content !== "¿Cómo respondo a este argumento?") { + throw new Error("provider did not receive the current Ask message"); + } + } finally { + globalThis.fetch = originalFetch; + } +}); + Deno.test("streaming Ask rejects advanced modes and toggles before quota or provider work", async () => { const cases = [ { diff --git a/supabase/functions/ai-stream/prompt.test.ts b/supabase/functions/ai-stream/prompt.test.ts index 47ec4f10a..ecf75fad7 100644 --- a/supabase/functions/ai-stream/prompt.test.ts +++ b/supabase/functions/ai-stream/prompt.test.ts @@ -28,7 +28,7 @@ Deno.test("streaming Ask keeps the shared identity, language, and evidence rules for ( const rule of [ "You are DebateLab's debate coach.", - "Match the language used in the user's question unless the user asks for another language.", + "Follow any explicit output-language instruction. Otherwise match the language used by the user.", "Never invent facts, statistics, citations, or sources.", ] ) { diff --git a/supabase/functions/ai/control-matrix.test.ts b/supabase/functions/ai/control-matrix.test.ts index 02561e4f9..c0eb44ccd 100644 --- a/supabase/functions/ai/control-matrix.test.ts +++ b/supabase/functions/ai/control-matrix.test.ts @@ -240,7 +240,7 @@ Deno.test("ask toggles and modes appear verbatim in the request prompt", () => { }); Deno.test("ask matrix preserves every mode, toggle, research, and output effect", () => { - const commonLanguageRule = "Match the language used in the user's question unless the user asks for another language."; + const commonLanguageRule = "Follow any explicit output-language instruction. Otherwise match the language used by the user."; for (const mode of ["general", "clarify"] as const) { for (const citationMode of [false, true]) { diff --git a/supabase/functions/ai/generate-honesty_test.ts b/supabase/functions/ai/generate-honesty_test.ts index de2140032..79abdf38e 100644 --- a/supabase/functions/ai/generate-honesty_test.ts +++ b/supabase/functions/ai/generate-honesty_test.ts @@ -55,6 +55,7 @@ const generatedOutput = { Deno.test("generate marks every contention when research has no citable sources", async () => { const originalFetch = globalThis.fetch; let providerCalls = 0; + let generatedRequestBody: Record | undefined; globalThis.fetch = (async (input: Request | URL | string, init?: RequestInit) => { @@ -96,10 +97,11 @@ Deno.test("generate marks every contention when research has no citable sources" if (url.href === "https://api.openai.com/v1/chat/completions") { providerCalls += 1; - const content = providerCalls === 1 ? { queries: [] } : generatedOutput; + const content = providerCalls % 2 === 1 ? { queries: [] } : generatedOutput; const requestBody = init?.body ? JSON.parse(String(init.body)) as Record : {}; + if (providerCalls > 1) generatedRequestBody = requestBody; if ( providerCalls === 1 && !JSON.stringify(requestBody.messages).includes("follow-up") @@ -146,6 +148,12 @@ Deno.test("generate marks every contention when research has no citable sources" const body = await response.json() as { output?: { contentions?: Array<{ needsSource?: boolean }> }; }; + const generatedPrompt = JSON.stringify(generatedRequestBody?.messages ?? []); + for (const marker of ["=== WEB RESEARCH RESULTS ===", "RESEARCH CITATION RULES:"]) { + if (generatedPrompt.includes(marker)) { + throw new Error(`empty research leaked into the Generate provider prompt: ${marker}`); + } + } const contentions = body.output?.contentions ?? []; if ( contentions.length !== 3 || @@ -157,6 +165,42 @@ Deno.test("generate marks every contention when research has no citable sources" }`, ); } + + for (const contention of generatedOutput.contentions) contention.needsSource = true; + const localSourceResponse = await handleRequest( + new Request("http://127.0.0.1/functions/v1/ai", { + method: "POST", + headers: { + Authorization: "Bearer test-user-token", + "Content-Type": "application/json", + }, + body: JSON.stringify({ + feature: "generate", + model: "gpt-5.4-mini", + input: { + motion: "Should schools teach media literacy?", + side: "pro", + format: "custom", + role: "intro", + length: "standard", + }, + sources: [{ title: "Local source", content: "Source content", type: "text" }], + }), + }), + ); + if (localSourceResponse.status !== 200) { + throw new Error(`generate with a local source returned ${localSourceResponse.status}`); + } + const localSourceBody = await localSourceResponse.json() as { + output?: { contentions?: Array<{ needsSource?: boolean }> }; + }; + const localContentions = localSourceBody.output?.contentions ?? []; + if ( + localContentions.length !== 3 || + localContentions.some((contention) => contention.needsSource !== true) + ) { + throw new Error(`model evidence warning was not preserved with a local source: ${JSON.stringify(localContentions)}`); + } } finally { globalThis.fetch = originalFetch; } diff --git a/supabase/functions/ai/handler-research.test.ts b/supabase/functions/ai/handler-research.test.ts index 10ebb8c58..b5109fc55 100644 --- a/supabase/functions/ai/handler-research.test.ts +++ b/supabase/functions/ai/handler-research.test.ts @@ -211,3 +211,116 @@ Deno.test("factoids handler carries Exa filters and consulted-source metadata ac await server.shutdown(); } }); + +Deno.test("Ask marks only surviving URL and research context as available", async () => { + const originalFetch = globalThis.fetch; + let scenario: "empty-search" | "empty-url" | "trimmed" = "empty-search"; + const providerBodies: Array> = []; + + globalThis.fetch = (async (input: Request | URL | string, init?: RequestInit) => { + const url = new URL(input instanceof Request ? input.url : String(input)); + + if (url.origin === "https://supabase.test") { + if (url.pathname === "/auth/v1/user") return jsonResponse({ id: "user-1", aud: "authenticated" }); + if (url.pathname.includes("/rest/v1/user_roles")) return jsonResponse({ role: "user" }); + if (url.pathname.includes("/rest/v1/user_settings")) { + return jsonResponse({ + subscription_tier: "trial", + daily_limit_override: 50, + trial_ends_at: "2099-01-01T00:00:00.000Z", + grace_period_ends_at: null, + }); + } + if (url.pathname.includes("/rest/v1/rate_limits")) return jsonResponse({ limit_value: 50 }); + if (url.pathname.includes("/rest/v1/rpc/consume_daily_quota")) { + return jsonResponse([{ allowed: true, current_count: 1 }]); + } + if (url.pathname.includes("/rest/v1/global_prompts")) return jsonResponse({ prompt_text: null }); + if (url.pathname.includes("/rest/v1/ai_requests")) return new Response(null, { status: 201 }); + throw new Error(`Unhandled Supabase request: ${url}`); + } + + if (url.href === "https://api.exa.ai/search") { + return scenario === "trimmed" + ? jsonResponse({ results: [{ title: "Official government evidence report", url: "https://energy.gov/reports/evidence-2026", text: "Official government statistics and evidence report.", publishedDate: "2026-01-01T00:00:00.000Z" }] }) + : jsonResponse({ results: [] }); + } + if (url.href === "https://api.exa.ai/contents") return jsonResponse({ results: [] }); + + if (url.href === "https://api.openai.com/v1/chat/completions") { + const body = JSON.parse(String(init?.body)) as Record; + providerBodies.push(body); + const messages = body.messages as Array<{ content?: unknown }> | undefined; + const userContent = messages?.at(-1)?.content; + if (typeof userContent === "string" && userContent.startsWith("User message:")) { + return jsonResponse({ + choices: [{ message: { content: JSON.stringify({ shouldSearch: true, query: "current debate evidence" }) } }], + usage: { prompt_tokens: 10, completion_tokens: 5, total_tokens: 15 }, + }); + } + if (typeof userContent === "string" && userContent.startsWith("Claim:")) { + return jsonResponse({ + choices: [{ message: { content: JSON.stringify({ queries: [] }) } }], + usage: { prompt_tokens: 10, completion_tokens: 5, total_tokens: 15 }, + }); + } + return jsonResponse({ + choices: [{ message: { content: JSON.stringify({ response: "ok", followUpQuestions: [], uncertainties: [] }) } }], + usage: { prompt_tokens: 10, completion_tokens: 5, total_tokens: 15 }, + }); + } + + throw new Error(`Unhandled request: ${url}`); + }) as typeof fetch; + + const runAsk = async (input: Record) => { + providerBodies.length = 0; + const response = await handleRequest(new Request("http://127.0.0.1/functions/v1/ai", { + method: "POST", + headers: { Authorization: "Bearer test-user-token", "Content-Type": "application/json" }, + body: JSON.stringify({ feature: "ask", model: "gpt-5.4-mini", input }), + })); + if (response.status !== 200) throw new Error(`Ask handler returned ${response.status}: ${await response.text()}`); + const mainRequest = providerBodies.find((body) => { + const messages = body.messages as Array<{ content?: unknown }> | undefined; + const lastMessage = messages?.at(-1); + return typeof lastMessage?.content === "string" && lastMessage.content.startsWith("Message:"); + }); + if (!mainRequest) throw new Error("Ask main provider request was not captured"); + const messages = mainRequest.messages as Array<{ role?: string; content?: unknown }>; + return { + system: String(messages.find((message) => message.role === "system")?.content ?? ""), + user: String(messages.at(-1)?.content ?? ""), + body: await response.json(), + }; + }; + + try { + const emptySearch = await runAsk({ message: "What is current evidence?", mode: "general", citationMode: true }); + if (!emptySearch.user.includes("Research Context: NONE")) throw new Error("empty search was marked available"); + if (emptySearch.user.includes("WEB RESEARCH RESULTS") || emptySearch.system.includes("RESEARCH CITATION RULES:")) { + throw new Error("empty search context leaked into the Ask provider prompt"); + } + + scenario = "empty-url"; + const emptyUrl = await runAsk({ message: "Read https://example.com/empty", mode: "general", citationMode: false }); + if (!emptyUrl.user.includes("Research Context: NONE") || emptyUrl.user.includes("WEB PAGE CONTEXT:")) { + throw new Error("empty URL fetch was marked as usable context"); + } + + scenario = "trimmed"; + const untrimmed = await runAsk({ message: "What evidence is available?", mode: "general", citationMode: true }); + if (untrimmed.body.output.searchInfo.sources.length === 0) { + throw new Error("trimming fixture must provide a citable source before trimming"); + } + const trimmed = await runAsk({ message: "x".repeat(50_000), mode: "general", citationMode: true }); + if (!trimmed.user.includes("Research Context: NONE") || trimmed.system.includes("RESEARCH CITATION RULES:")) { + throw new Error("trimmed research remained marked as available"); + } + if (trimmed.body.output.searchInfo.sources.length !== 0 || (trimmed.body.output.citations?.length ?? 0) !== 0) { + throw new Error("trimmed research leaked into response sources or fallback citations"); + } + } finally { + globalThis.fetch = originalFetch; + } +}); diff --git a/supabase/functions/ai/index.ts b/supabase/functions/ai/index.ts index 3358c9774..9456b623c 100644 --- a/supabase/functions/ai/index.ts +++ b/supabase/functions/ai/index.ts @@ -1465,9 +1465,14 @@ export async function handleRequest(req: Request): Promise { if (urls.size > 0) { const contents = await Promise.all(Array.from(urls).slice(0, 3).map(async (u) => { const content = await exaGetContents(u, req.signal); - return `--- Content from ${u} ---\n${content.slice(0, 8000)}\n--- End of ${u} ---`; + return content.trim() + ? `--- Content from ${u} ---\n${content.slice(0, 8000)}\n--- End of ${u} ---` + : ""; })); - urlContext = "\n\nWEB PAGE CONTEXT:\n" + contents.join("\n"); + const usableContents = contents.filter(Boolean); + if (usableContents.length > 0) { + urlContext = "\n\nWEB PAGE CONTEXT:\n" + usableContents.join("\n"); + } } // 2) Optional web search in Ask mode. When the user explicitly enables @@ -1478,14 +1483,14 @@ export async function handleRequest(req: Request): Promise { askSearchQuery = searchDecision.query || message.slice(0, 240); if (askSearchQuery) { askResearch = await deepResearch(askSearchQuery, "", researchDeps, ASK_RESEARCH); - if (askResearch.searches.length > 0 || askResearch.readContent.length > 0) { - webSearchContext = `\n\n${buildResearchContext(askResearch)}`; + const renderedResearch = hasResearch(askResearch) ? buildResearchContext(askResearch) : ""; + if (renderedResearch) { + webSearchContext = `\n\n${renderedResearch}`; } } } - const systemPrompt = await buildSystemPromptFor(feature, askResearch); - const basePrompt = buildAskBasePrompt({ + const askPromptInput = { message, mode, citationMode, @@ -1493,8 +1498,9 @@ export async function handleRequest(req: Request): Promise { prosConsMode, argumentMapMode, researchAvailable: Boolean(webSearchContext || urlContext), - }); - const contextBudget = Math.max(0, MAX_ASK_CONTEXT_LENGTH - basePrompt.length); + }; + const initialBasePrompt = buildAskBasePrompt(askPromptInput); + const contextBudget = Math.max(0, MAX_ASK_CONTEXT_LENGTH - initialBasePrompt.length); const webSearchBudget = Math.max(0, contextBudget - urlContext.length); const trimmedWebSearch = truncateTextToBudget(webSearchContext, webSearchBudget); webSearchContext = trimmedWebSearch.text; @@ -1505,6 +1511,12 @@ export async function handleRequest(req: Request): Promise { urlContext = trimmedUrlContext.text; responseTruncated = responseTruncated || trimmedUrlContext.truncated; + const researchAvailable = Boolean(webSearchContext || urlContext); + const basePrompt = researchAvailable === askPromptInput.researchAvailable + ? initialBasePrompt + : buildAskBasePrompt({ ...askPromptInput, researchAvailable }); + const effectiveResearch = webSearchContext && askResearch ? askResearch : null; + const systemPrompt = await buildSystemPromptFor(feature, effectiveResearch); const prompt = `${basePrompt}${urlContext}${webSearchContext}`; const messages = [...history, { role: "user", content: prompt }]; const aiResult = recordMainResult(await callAI(provider, model, systemPrompt, messages, temperature, req.signal)); @@ -1526,7 +1538,9 @@ export async function handleRequest(req: Request): Promise { })); } - const researchSources = askResearch ? getResearchSources(askResearch) : []; + const researchSources = effectiveResearch + ? getResearchSources(effectiveResearch).filter((source) => webSearchContext.includes(source.url)) + : []; if (shouldUseResearch) { typedOutput.searchInfo = { webSearchUsed: researchSources.length > 0,