From eb56f867bfe9c9ced1169ea7e1402922386986fd Mon Sep 17 00:00:00 2001 From: Adam Date: Mon, 21 Sep 2026 15:48:38 +0000 Subject: [PATCH 1/9] docs: specify Jev decision features --- .../specs/2026-09-21-jev-decisions-design.md | 388 ++++++++++++++++++ 1 file changed, 388 insertions(+) create mode 100644 docs/superpowers/specs/2026-09-21-jev-decisions-design.md diff --git a/docs/superpowers/specs/2026-09-21-jev-decisions-design.md b/docs/superpowers/specs/2026-09-21-jev-decisions-design.md new file mode 100644 index 000000000..9687f1408 --- /dev/null +++ b/docs/superpowers/specs/2026-09-21-jev-decisions-design.md @@ -0,0 +1,388 @@ +# Jev decisions for Crossfire, sources, and model routing + +**Status:** Draft for written review + +**Date:** 2026-09-21 + +**Owner:** DebateLab + +## Goal + +Use TypeSafe Jev through DebateLab's existing OpenRouter account for narrow, +typed judgments that improve three product flows: + +1. show live coaching signals during Crossfire; +2. rank, retain, and track the sources that materially inform AI work; +3. offer `Auto Fast` and `Auto Pro` choices that resolve to a concrete writing + model for each request. + +Jev is an internal decision model. It never appears as a prose-writing model, +never receives a browser-visible credential, and never replaces deterministic +authorization, quota, validation, citation, or conflict checks. + +## Product requirements + +### Crossfire + +- Every completed question-and-answer pair receives live coaching signals. +- The signals are: + - answer quality: `direct`, `partial`, or `evasive`; + - concession: `none`, `implicit`, or `explicit`; + - follow-up opportunity: a probability from 0 to 1. +- Signals appear beneath the answer they evaluate and remain visible in the + final transcript. +- The interface describes them as coaching judgments, not objective scores. +- A Jev failure must not discard or delay an otherwise valid Crossfire reply. + +### Sources + +- Jev evaluates candidate sources for relevance, evidence strength, and + importance to the current request. +- Jev ordering is applied only after existing deterministic URL, source-type, + quality-floor, and provenance rules. +- Active user sources are not deleted or silently deactivated because of a + Jev decision. +- The prompt receives the strongest relevant sources first within the existing + source-count and character budgets. +- After Generate completes, Jev identifies which retained sources materially + informed the generated case. +- The product labels that result `Used in this case` and explains that it is a + semantic model judgment, not an exact citation trace. +- Saved cases keep a bounded source-usage snapshot. +- The source library defaults to a `Smart` sort that places case-used sources + first, followed by importance, evidence strength, existing deterministic + quality, and recency. +- Existing sort choices remain available. +- Semantic metadata persists in the existing `sources.metadata` JSONB column. + No schema migration is required. + +### Automatic model routing + +- The model picker adds `Auto Fast` and `Auto Pro` above the concrete models. +- `Auto Fast` is available to Basic and Pro accounts. +- GLM 5.3 Flash is the normal `Auto Fast` selection, including for Basic + accounts. Manual GLM selection can retain its existing Pro restriction. +- Jev may divert `Auto Fast` to Luna, Gemini Flash, or DeepSeek Flash when the + bounded request clearly fits one of those models better. +- `Auto Pro` remains Pro-only and routes among GPT-5.6 Sol, Claude Opus 5, and + Claude Sonnet 5. +- Manual model choices continue to bypass Jev routing. +- OpenRouter requests, cost accounting, and admin reporting use the concrete + resolved model. +- Responses expose both the requested selection and resolved model in bounded + metadata. +- A routing failure falls back to GLM 5.3 Flash for `Auto Fast` and Claude + Sonnet 5 for `Auto Pro`. +- The daily user quota is consumed once per product request. Internal Jev calls + do not reserve additional daily quota. + +## Architecture + +### Shared Jev client + +Create a small shared Deno module under `supabase/functions/_shared/` that: + +- calls `POST https://openrouter.ai/api/alpha/decisions`; +- authenticates with the existing `OPENROUTER_API_KEY`; +- pins `typesafe/jev-1.13` so behavior cannot change without a code review; +- accepts only state plus `noul`, `choice`, and `score` questions; +- applies an eight-second timeout linked to the parent request signal; +- enforces the existing provider response byte limit; +- parses external JSON from `unknown` into narrow answer types; +- returns normalized usage and the actual returned model; +- allows an injected fetch function in tests without adding a runtime SDK. + +One request batches all related questions for a decision. No question asks Jev +to generate text, count items, or create model identifiers. + +### Failure policy + +- Crossfire: return the valid reply with no signal. +- Source ranking: preserve deterministic source ordering and existing retention + behavior. +- Source-use identification: return the case without use labels. +- Auto Fast: use GLM 5.3 Flash. +- Auto Pro: use Claude Sonnet 5 after the existing Pro entitlement check. +- Abort: propagate the caller's abort rather than running a fallback provider + request after the user has canceled. +- Invalid Jev output: treat it as unavailable; never coerce malformed values. + +### Usage accounting + +The main product request keeps one normal `ai_requests` row using the resolved +writing model. Each completed Jev call adds a separate internal row using: + +- provider: `openrouter`; +- model: `typesafe/jev-1.13`; +- feature: `jev_model_router`, `jev_source_ranking`, + `jev_source_usage`, or `jev_crossfire_signals`. + +This keeps cost and latency auditable without incrementing the daily quota a +second time. Failed calls with observed usage retain the existing incomplete +usage semantics. + +## Model routing contract + +### Selection types + +Keep the concrete `MODEL_CATALOG` unchanged. Define virtual selections beside +it: + +```ts +type AutoModelSelection = "auto-fast" | "auto-pro"; +type ModelSelection = ModelCatalogEntry["id"] | AutoModelSelection; +``` + +Settings normalization accepts the two virtual values. Server validation keeps +virtual selections separate from concrete OpenRouter model IDs. + +### Candidate criteria + +`Auto Fast`: + +- `z-ai/glm-5.3-flash`: default for ordinary structured DebateLab work and + efficient long-context requests; +- `openai/gpt-5.6-luna`: short latency-sensitive conversation and simple + coaching; +- `google/gemini-3.8-flash`: very long input, multilingual output, or a request + whose dominant need is broad context handling; +- `deepseek/deepseek-v4-flash-0731`: logic-heavy analysis, adversarial attack, + or structured reasoning at low cost. + +`Auto Pro`: + +- `openai/gpt-5.6-sol`: complex structured reasoning and reliable multi-part + analysis; +- `anthropic/claude-opus-5`: nuanced rhetoric, adversarial critique, and + demanding long-form writing; +- `anthropic/claude-sonnet-5`: balanced professional work and the fallback when + no specialist clearly wins. + +The Jev state contains the feature, requested language, controls, source count, +reasoning level, and bounded request text. It excludes credentials, user IDs, +and full source-library contents. A confidence below 0.55 uses the relevant +fallback. + +### Entitlements and request sequence + +1. Parse the requested selection without treating it as a concrete model. +2. Authenticate the user and resolve the current subscription. +3. Reject `Auto Pro` for non-Pro users with the existing upgrade response. +4. Reserve one daily quota unit. +5. Resolve an automatic selection through Jev or its fallback. +6. Apply the existing concrete-model allowlist and reasoning rules. +7. Call OpenRouter with the resolved model. +8. Log and return the resolved model plus the requested selection. + +The streaming Ask handler follows the same sequence before opening the upstream +stream. + +## Source decision contract + +### Request source identity + +The browser includes each active library source's stable ID in the authenticated +request. Prompt rendering continues to expose only bounded title, content, +origin, and type. The ID exists for assessment correlation and is not trusted +as authorization; the browser can only persist assessments through existing +user-scoped RLS. + +Research-discovered sources use their normalized public URL as a temporary +correlation key until autosave creates a source row. + +### Semantic fields + +Extend source provenance with an optional nested object: + +```ts +interface SourceSemanticMetadata { + relevance: number; // 0..1 + importance: 0 | 1 | 2 | 3; + strength: 0 | 1 | 2 | 3; + evaluatedAt: string; + useCount: number; + caseUseCount: number; + lastUsedAt?: string; + lastUsedInCase?: boolean; +} +``` + +All fields are parsed at the client boundary. Counts are non-negative integers, +probabilities are bounded, score levels are exact integers, and invalid nested +objects are discarded without discarding valid existing provenance. + +`updateSource` persists `metadata` as well as editable source fields. Assessment +updates preserve existing URL, type, query, quality, and capture provenance. + +### Ranking and retention + +For each bounded source candidate, one Jev batch asks: + +- whether it is relevant to the request; +- how strong its evidence is on a four-level rubric; +- how important it is to the requested output on a four-level rubric. + +The application sorts candidates by relevance, importance, strength, existing +quality score, then stable original order. A relevance probability below 0.45 +does not delete or deactivate an active source. It only moves the source below +better candidates. If all results fall below the threshold or Jev fails, the +existing deterministic order remains. + +Automatically discovered sources are autosaved only when they pass the existing +non-weak quality floor and the semantic relevance threshold, or when post-output +usage identification marks them as used. This replaces the current behavior of +saving every discovered source while preserving the existing Undo action. + +### Case-use identification + +After Generate output has passed schema and control validation, one Jev batch +asks independently whether each retained source materially informed the case. +The application uses a probability above 0.70 for `Used in this case`; values +from 0.30 through 0.70 remain uncertain and do not receive the label. + +The generated case metadata keeps a maximum of ten source snapshots containing +source ID when available, title, safe origin, importance, strength, and the +usage judgment. Saving the case therefore preserves the relationship without a +new table or migration. + +## Crossfire signal contract + +After the generative next-turn output passes its existing strict schema, the +server constructs the newest completed exchange by combining the transcript and +new reply. One batched Jev request asks: + +- answer quality as `direct`, `partial`, or `evasive`; +- concession as `none`, `implicit`, or `explicit`; +- whether the exchange presents a useful follow-up opportunity. + +The normalized response is: + +```ts +interface CrossfireTurnSignal { + answerTurnId: string; + answerQuality: "direct" | "partial" | "evasive"; + answerQualityConfidence: number; + concession: "none" | "implicit" | "explicit"; + concessionConfidence: number; + followUpOpportunity: number; +} +``` + +The server derives the answer turn ID from the validated sequence. The client +never accepts signals for an unknown or non-answer turn. Retried replies replace +the pending reply and signal together; they cannot attach a fresh signal to an +old answer. + +`CrossfireTurn` gains an optional signal. Draft schema version 1 remains valid +because the field is optional. The reducer owns signal attachment so restored, +retried, and newly received turns share one validation path. + +## User interface + +This is a targeted extension of the existing DebateLab interface. It introduces +no new visual system, icon package, motion language, or layout structure. + +### Model picker + +- Place `Auto Fast` and `Auto Pro` before concrete models. +- Add short descriptions inside their items. +- Show the existing Pro badge on `Auto Pro` and disable it for non-Pro users. +- Disable the separate reasoning selector for automatic selections because the + router owns the concrete model and effective reasoning choice. +- Preserve keyboard behavior, accessible names, touch targets, and loading-time + entitlement behavior. + +### Crossfire + +- Render signals beneath the answer text inside the existing transcript card. +- Use compact text labels with the current semantic color tokens. +- Show follow-up opportunity only when its probability exceeds 0.70. +- Provide accessible text that does not rely on color. +- Do not animate the signals independently from their turn card. + +### Source library and result source lists + +- Add `Smart`, `Most used`, `Importance`, and `Evidence strength` sort choices. +- Default to `Smart`. +- Show at most three compact semantic labels per row. +- `Used in this case` has priority over score labels. +- Add the usage explanation to the selected-source inspector. +- Reuse the same labels in researched-source result lists. +- Keep search, folders, active toggles, editing, deletion, and responsive layout + unchanged. + +All visible copy is added in English, Spanish, and Hungarian in the same change. + +## Security and privacy + +- `OPENROUTER_API_KEY` remains server-only. +- Jev receives only the bounded state required for its question set. +- Source IDs are correlation values, not authorization tokens. +- Existing RLS remains decisive for source writes. +- Jev cannot change subscriptions, consume extra quota, activate sources, delete + sources, accept edits, or bypass model allowlists. +- Existing prompt-injection, SSRF, URL, citation, and source-quality controls + remain deterministic. +- Logs contain request IDs, models, counts, timing, and error classes, not user + prompts, transcripts, source bodies, or credentials. + +## Verification + +Development follows test-first cycles. Each production branch is preceded by a +test that fails for the missing behavior. + +Required automated coverage: + +- Jev request serialization, strict response parsing, batching, timeout, abort, + malformed output, provider failure, and usage normalization; +- Auto Fast Basic access, GLM default, specialist diversion, low-confidence + fallback, Auto Pro denial for Basic, Pro routing, concrete model logging, and + streaming metadata; +- source identity bounds, semantic metadata parsing, preservation on update, + sorting, fallback order, retention, autosave, source-use snapshots, and saved + case reload; +- Crossfire signal normalization, turn association, reducer persistence, retry + replacement, Jev failure without reply failure, and debrief compatibility; +- English, Spanish, and Hungarian key parity; +- keyboard and accessible-name coverage for model and source controls. + +Required runtime verification: + +- a minimal live Jev request through the configured OpenRouter key with no + credential output; +- focused Deno and Vitest suites after each slice; +- real browser checks for desktop and mobile model selection, Sources smart + ordering, Generate source-use labels, and both Crossfire modes with live + signals; +- `npm run verify` on Node 22; +- final diff review, credential scan, and current-head GitHub CI and CodeRabbit + review. + +The pull request is pushed and babysat until its current head is green and has +no actionable review findings. It is not merged. + +## Non-goals + +- Replacing prose models with Jev. +- Letting Jev verify factual truth or citation authenticity. +- Changing source RLS or adding a source-usage database table. +- Changing the daily quota policy. +- Redesigning DebateLab. +- Deploying Edge Functions, migrations, or frontend code to production. +- Merging the pull request. + +## Rollback + +Reverting the feature commit restores manual model selection, deterministic +source ordering, and Crossfire without live signals. Existing source metadata +is additive JSON and can remain unread without affecting older clients. No +database rollback or destructive cleanup is required. + +## References + +- OpenRouter Decisions API: + +- OpenRouter Jev model page: +- TypeSafe confidence guidance: +- DebateLab architecture: [`docs/architecture.md`](../../architecture.md) +- DebateLab testing: [`docs/testing.md`](../../testing.md) From b70da1abf9c171c2df045969d54290f47bb1c4c5 Mon Sep 17 00:00:00 2001 From: Adam Date: Mon, 21 Sep 2026 15:53:13 +0000 Subject: [PATCH 2/9] docs: plan Jev decision implementation --- .../plans/2026-09-21-jev-decisions.md | 583 ++++++++++++++++++ 1 file changed, 583 insertions(+) create mode 100644 docs/superpowers/plans/2026-09-21-jev-decisions.md diff --git a/docs/superpowers/plans/2026-09-21-jev-decisions.md b/docs/superpowers/plans/2026-09-21-jev-decisions.md new file mode 100644 index 000000000..891dfddb3 --- /dev/null +++ b/docs/superpowers/plans/2026-09-21-jev-decisions.md @@ -0,0 +1,583 @@ +# Jev decisions implementation plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** Add Jev-powered live Crossfire signals, semantic source ranking and case-use tracking, and `Auto Fast`/`Auto Pro` model routing through DebateLab's existing OpenRouter account. + +**Architecture:** A small shared Deno module calls OpenRouter's Decisions API and validates typed Jev answers. The existing `ai` and `ai-stream` functions consume it for internal judgments while retaining their current authentication, quota, source-safety, and concrete-model paths. Client contracts carry bounded decision metadata into existing stores and UI without a new dependency or database migration. + +**Tech Stack:** React 18, TypeScript 5.8, Zustand, Zod 4, Supabase Edge Functions on Deno, OpenRouter Decisions API, Vitest, Playwright. + +**Spec:** `docs/superpowers/specs/2026-09-21-jev-decisions-design.md` + +## Global constraints + +- Use Node `22.23.2`, npm, and the committed `package-lock.json`. +- Use only `OPENROUTER_API_KEY`; do not add or expose a TypeSafe key. +- Pin internal decisions to `typesafe/jev-1.13`. +- Keep `MODEL_CATALOG` concrete-only and preserve every manual model choice. +- `Auto Fast` is available to Basic users and normally resolves to `z-ai/glm-5.3-flash`, even though manual GLM selection remains Pro-only. +- `Auto Pro` remains Pro-only. +- Reserve one daily quota unit per product request; internal Jev calls do not reserve quota. +- Preserve deterministic source-quality, SSRF, citation, auth, RLS, and Crossfire transcript controls. +- Persist semantic source data only inside the existing `sources.metadata` JSONB and saved-output JSONB. +- Update English, Spanish, and Hungarian together. +- Preserve DebateLab's existing component system and visual design; add no frontend dependency. +- Do not deploy, merge, or change production state. + +## Review focus + +- A Basic user selecting `Auto Fast` must be allowed to resolve to GLM without gaining manual access to Pro-only GLM. +- An unavailable, timed-out, aborted, or malformed Jev response must follow the feature-specific fallback without losing a valid main response or starting work after cancellation. +- Semantic source updates must preserve existing provenance and must not delete, deactivate, or falsely mark uncertain sources as used. +- Crossfire signals must attach only to the answer in the newest completed exchange, including retries and restored drafts. +- Cost, analytics, and visible model metadata must report the concrete resolved model while retaining the requested automatic selection. + +--- + +### Task 1: Shared Jev Decisions client + +**Files:** +- Create: `supabase/functions/_shared/jev.ts` +- Create: `supabase/functions/_shared/jev.test.ts` +- Modify: `supabase/functions/_shared/research.ts` + +**Interfaces:** +- Produces: `callJev({ state, questions, signal, fetchImpl? }): Promise`. +- Produces: validated `JevNoulAnswer`, `JevChoiceAnswer`, `JevScoreAnswer`, and normalized OpenRouter usage. +- Consumes: existing `fetchJsonWithTimeout`, response-size limits, `normalizeOpenRouterUsage`, and parent abort signals. + +- [ ] **Step 1: Write failing serialization and parsing tests** + +Add Deno tests with a real `Response` test double that assert: + +```ts +const result = await callJev({ + state: { feature: "crossfire", text: "Question and answer" }, + questions: { + directness: { + type: "choice", + instructions: "How directly does the answer address the question?", + criteria: { direct: "Fully answers it", partial: "Answers only part", evasive: "Avoids it" }, + }, + concession: { + type: "noul", + instructions: "Does the answer concede the challenged claim?", + criteria: { true: "Concedes it", false: "Does not concede it" }, + }, + }, + fetchImpl, +}); + +assertEquals(capturedUrl, "https://openrouter.ai/api/alpha/decisions"); +assertEquals(capturedBody.model, "typesafe/jev-1.13"); +assertEquals(result.answers.directness.choice, "direct"); +assertEquals(result.answers.concession.noul, 0.12); +``` + +Add literal fixtures for choice probabilities, score legend, usage, returned model, malformed answers, oversized bodies, non-2xx responses, timeout, and caller abort. The production mutation each test catches is stated in its name. + +- [ ] **Step 2: Run the focused test and verify RED** + +Run: + +```bash +deno test --allow-read --allow-env=OPENROUTER_API_KEY --allow-net=127.0.0.1 supabase/functions/_shared/jev.test.ts +``` + +Expected: FAIL because `jev.ts` and `callJev` do not exist. + +- [ ] **Step 3: Implement the minimum Decisions client** + +Define discriminated question and answer unions. Parse the response from +`unknown`; reject an answer whose type, chosen label, probability map, score, +legend, or bounds do not match its submitted question. Use the injected fetch +only in tests; production defaults to `globalThis.fetch`. Link the eight-second +timeout to the caller signal, and return normalized usage without logging state +or question content. + +- [ ] **Step 4: Run the focused test and verify GREEN** + +Run the command from Step 2. + +Expected: all `jev.test.ts` tests pass with no leaked credential or unhandled rejection. + +- [ ] **Step 5: Run shared backend regression tests** + +Run: + +```bash +PATH=/home/codex/.nvm/versions/node/v22.23.2/bin:$PATH npm run test:edge +``` + +Expected: all Edge tests pass. + +- [ ] **Step 6: Commit** + +```bash +git add supabase/functions/_shared/jev.ts supabase/functions/_shared/jev.test.ts supabase/functions/_shared/research.ts +git commit -m "feat(ai): add OpenRouter Jev decisions client" +``` + +### Task 2: Automatic model routing + +**Files:** +- Modify: `supabase/functions/_shared/model-catalog.ts` +- Modify: `supabase/functions/_shared/model-catalog.test.ts` +- Create: `supabase/functions/_shared/auto-model-router.ts` +- Create: `supabase/functions/_shared/auto-model-router.test.ts` +- Modify: `supabase/functions/ai/index.ts` +- Modify: `supabase/functions/ai-stream/index.ts` +- Modify: `supabase/functions/ai/handler-research.test.ts` +- Modify: `supabase/functions/ai-stream/handler_test.ts` +- Modify: `src/lib/ai/providers/types.ts` +- Modify: `src/lib/stores/settings-store.ts` +- Modify: `src/components/ModelSelector.tsx` +- Modify: `src/test/components/ModelSelector.test.tsx` +- Modify: `src/test/settings-store.test.ts` +- Modify: `src/locales/en.json` +- Modify: `src/locales/es.json` +- Modify: `src/locales/hu.json` +- Modify: `docs/architecture.md` + +**Interfaces:** +- Consumes: `callJev` from Task 1. +- Produces: `AUTO_MODEL_SELECTIONS`, `isAutoModelSelection()`, and `resolveModelSelection()` returning requested selection, resolved concrete ID, catalog entry, and effective reasoning. +- Produces: response `meta.requestedModel` and `meta.resolvedModel` for non-streaming and streaming requests. + +- [ ] **Step 1: Write failing pure-router tests** + +Cover these hand-derived cases: + +```ts +assertEquals(await resolveModelSelection(autoFastOrdinaryFixture), { + requestedModel: "auto-fast", + resolvedModel: "z-ai/glm-5.3-flash", + usedFallback: false, +}); + +assertEquals(await resolveModelSelection(autoFastLogicFixture), { + requestedModel: "auto-fast", + resolvedModel: "deepseek/deepseek-v4-flash-0731", + usedFallback: false, +}); +``` + +Also assert Basic Auto Fast access, GLM fallback, unsupported Jev labels ignored, +confidence below `0.55` fallback, Pro candidate allowlisting, Auto Pro rejection +before resolution for Basic, and caller abort propagation. + +- [ ] **Step 2: Run pure-router tests and verify RED** + +Run: + +```bash +deno test --allow-read --allow-env=OPENROUTER_API_KEY --allow-net=127.0.0.1 supabase/functions/_shared/auto-model-router.test.ts +``` + +Expected: FAIL because the router does not exist. + +- [ ] **Step 3: Implement virtual selections and router** + +Keep the ten-item concrete catalog unchanged. Add exact virtual definitions: + +```ts +export const AUTO_MODEL_SELECTIONS = [ + { id: "auto-fast", name: "Auto Fast", isProOnly: false }, + { id: "auto-pro", name: "Auto Pro", isProOnly: true }, +] as const; +``` + +Build one Jev choice whose criteria contain only the applicable candidate IDs +and the approved strengths from the spec. Bound routing state to 8,000 +characters. Validate the answer against the submitted candidate set. + +- [ ] **Step 4: Make the backend resolve before concrete validation** + +Refactor both Edge handlers to parse a `ModelSelection`, authenticate, enforce +`Auto Pro`, reserve one quota unit, resolve the selection, then apply existing +concrete catalog and reasoning rules. A validated `Auto Fast` route may use its +explicit candidate list even when GLM's manual catalog entry is Pro-only; a +manually requested GLM still uses the existing Pro gate. Use the resolved model +for provider calls, usage rows, cost, and admin aggregation. Return requested +and resolved IDs in response metadata. Add a separate internal Jev usage row +without calling the quota RPC again. + +- [ ] **Step 5: Run handler tests and verify routing GREEN** + +Run: + +```bash +deno test --allow-read --allow-env=DEMO_CODE_PEPPER,EXA_API_KEY,SUPABASE_URL,SUPABASE_ANON_KEY,SUPABASE_SERVICE_ROLE_KEY,SPARRING_INTEGRITY_SECRET,OPENROUTER_API_KEY,WORKSPACE_INVITE_SECRET,WORKSPACE_INVITE_MAILER_URL,WORKSPACE_INVITE_MAILER_TOKEN,WORKSPACE_INVITE_BASE_URL --allow-net=127.0.0.1 supabase/functions/_shared/model-catalog.test.ts supabase/functions/_shared/auto-model-router.test.ts supabase/functions/ai/handler-research.test.ts supabase/functions/ai-stream/handler_test.ts +``` + +Expected: Basic Auto Fast reaches GLM, Basic Auto Pro returns the existing 403, +manual GLM stays Pro-only, streaming reports the concrete model, and quota is +reserved once. + +- [ ] **Step 6: Write failing client picker and settings tests** + +Assert Auto Fast and Auto Pro render before concrete models, Auto Fast is enabled +for Basic, Auto Pro is disabled for Basic, automatic values survive store +normalization and server hydration, and reasoning is disabled for either auto +selection. + +- [ ] **Step 7: Run client tests and verify RED** + +Run: + +```bash +PATH=/home/codex/.nvm/versions/node/v22.23.2/bin:$PATH npm test -- src/test/components/ModelSelector.test.tsx src/test/settings-store.test.ts +``` + +Expected: FAIL because virtual choices are not yet rendered or accepted. + +- [ ] **Step 8: Implement the picker and persisted settings contract** + +Add the two localized choices using the existing Select, Badge, and model label +styles. Do not add a new icon or layout. Preserve saved Pro controls while +entitlement is loading. Extend settings normalization without putting virtual +IDs into `MODEL_CATALOG`. + +- [ ] **Step 9: Run client tests, typecheck, and docs check** + +Run: + +```bash +PATH=/home/codex/.nvm/versions/node/v22.23.2/bin:$PATH npm test -- src/test/components/ModelSelector.test.tsx src/test/settings-store.test.ts src/test/i18n.test.ts +PATH=/home/codex/.nvm/versions/node/v22.23.2/bin:$PATH npm run typecheck +PATH=/home/codex/.nvm/versions/node/v22.23.2/bin:$PATH npm run docs:check +``` + +Expected: all commands pass. + +- [ ] **Step 10: Commit** + +```bash +git add supabase/functions/_shared/model-catalog.ts supabase/functions/_shared/model-catalog.test.ts supabase/functions/_shared/auto-model-router.ts supabase/functions/_shared/auto-model-router.test.ts supabase/functions/ai/index.ts supabase/functions/ai-stream/index.ts supabase/functions/ai/handler-research.test.ts supabase/functions/ai-stream/handler_test.ts src/lib/ai/providers/types.ts src/lib/stores/settings-store.ts src/components/ModelSelector.tsx src/test/components/ModelSelector.test.tsx src/test/settings-store.test.ts src/locales/en.json src/locales/es.json src/locales/hu.json docs/architecture.md +git commit -m "feat(ai): add Jev automatic model routing" +``` + +### Task 3: Semantic source ranking, retention, and case usage + +**Files:** +- Create: `supabase/functions/_shared/source-decisions.ts` +- Create: `supabase/functions/_shared/source-decisions.test.ts` +- Modify: `supabase/functions/_shared/prompts.ts` +- Modify: `supabase/functions/_shared/research.ts` +- Modify: `supabase/functions/ai/index.ts` +- Modify: `supabase/functions/ai/handler-research.test.ts` +- Modify: `src/lib/stores/session-store.ts` +- Modify: `src/lib/ai/client.ts` +- Modify: `src/lib/ai/schemas/generate.ts` +- Modify: `src/components/features/shared/SourceQualityList.tsx` +- Modify: `src/pages/LibraryPage.tsx` +- Modify: `src/test/components/SourceQualityList.test.tsx` +- Modify: `src/test/pages/LibraryPage.test.tsx` +- Modify: `src/test/session-store.test.ts` +- Modify: `src/test/ai/client-branches.test.ts` +- Modify: `src/locales/en.json` +- Modify: `src/locales/es.json` +- Modify: `src/locales/hu.json` +- Modify: `docs/architecture.md` + +**Interfaces:** +- Consumes: `callJev` from Task 1. +- Produces: `rankSourcesForRequest()` and `identifyCaseSourceUsage()` with deterministic fallbacks. +- Produces: `SourceSemanticMetadata`, `SourceAssessment`, response `sourceAssessments`, and Generate `metadata.sourceUsage`. + +- [ ] **Step 1: Write failing source decision tests** + +Use literal source fixtures to assert: + +- existing deterministic weak-source exclusion remains decisive; +- Jev relevance, importance, and strength reorder non-weak candidates; +- an all-low-confidence batch preserves original order; +- malformed or missing answers preserve original order; +- case usage requires probability above `0.70`; +- values from `0.30` through `0.70` remain uncertain; +- assessment count and source excerpts remain bounded. + +- [ ] **Step 2: Run focused source decision tests and verify RED** + +Run: + +```bash +deno test --allow-read --allow-env=OPENROUTER_API_KEY --allow-net=127.0.0.1 supabase/functions/_shared/source-decisions.test.ts +``` + +Expected: FAIL because the module does not exist. + +- [ ] **Step 3: Implement server-side source decisions** + +Batch three typed questions per bounded candidate. Preserve stable original +indexes for ties. Apply semantic ordering only when at least one candidate has +usable confidence. After validated Generate output, batch one Noul per retained +source to identify material use and attach a maximum of ten snapshots. + +- [ ] **Step 4: Write failing source persistence and UI tests** + +Assert the client: + +- sends stable IDs for active library sources; +- parses valid semantic metadata and discards invalid nested fields; +- persists `metadata` during `updateSource`; +- merges assessment updates without losing quality provenance; +- autosaves only retained or used discovered sources; +- defaults Library to Smart sorting; +- orders case-used, importance, strength, quality, and recency in that order; +- renders `Used in this case`, importance, strength, and use count accessibly; +- reloads a saved case with its source-usage snapshot. + +- [ ] **Step 5: Run client tests and verify RED** + +Run: + +```bash +PATH=/home/codex/.nvm/versions/node/v22.23.2/bin:$PATH npm test -- src/test/session-store.test.ts src/test/ai/client-branches.test.ts src/test/components/SourceQualityList.test.tsx src/test/pages/LibraryPage.test.tsx +``` + +Expected: FAIL because semantic metadata and Smart sorting do not exist. + +- [ ] **Step 6: Implement source contracts, persistence, and UI** + +Extend provenance parsing with the exact bounded nested type from the spec. +Persist metadata on source updates. Add one store action that applies a bounded +assessment batch with user-scoped updates and optimistic rollback. Keep the +existing source row composition, adding no more than three semantic labels. +Add localized sort choices and the selected-source explanation. + +- [ ] **Step 7: Run focused backend and client tests GREEN** + +Run: + +```bash +deno test --allow-read --allow-env=DEMO_CODE_PEPPER,EXA_API_KEY,SUPABASE_URL,SUPABASE_ANON_KEY,SUPABASE_SERVICE_ROLE_KEY,SPARRING_INTEGRITY_SECRET,OPENROUTER_API_KEY,WORKSPACE_INVITE_SECRET,WORKSPACE_INVITE_MAILER_URL,WORKSPACE_INVITE_MAILER_TOKEN,WORKSPACE_INVITE_BASE_URL --allow-net=127.0.0.1 supabase/functions/_shared/source-decisions.test.ts supabase/functions/_shared/research.test.ts supabase/functions/ai/source-quality.test.ts supabase/functions/ai/handler-research.test.ts +PATH=/home/codex/.nvm/versions/node/v22.23.2/bin:$PATH npm test -- src/test/session-store.test.ts src/test/ai/client-branches.test.ts src/test/components/SourceQualityList.test.tsx src/test/pages/LibraryPage.test.tsx src/test/i18n.test.ts +``` + +Expected: all commands pass and no prior provenance or source-quality assertion changes meaning. + +- [ ] **Step 8: Commit** + +```bash +git add supabase/functions/_shared/source-decisions.ts supabase/functions/_shared/source-decisions.test.ts supabase/functions/_shared/prompts.ts supabase/functions/_shared/research.ts supabase/functions/ai/index.ts supabase/functions/ai/handler-research.test.ts src/lib/stores/session-store.ts src/lib/ai/client.ts src/lib/ai/schemas/generate.ts src/components/features/shared/SourceQualityList.tsx src/pages/LibraryPage.tsx src/test/components/SourceQualityList.test.tsx src/test/pages/LibraryPage.test.tsx src/test/session-store.test.ts src/test/ai/client-branches.test.ts src/locales/en.json src/locales/es.json src/locales/hu.json docs/architecture.md +git commit -m "feat(sources): rank and retain evidence with Jev" +``` + +### Task 4: Live Crossfire coaching signals + +**Files:** +- Create: `supabase/functions/_shared/crossfire-signals.ts` +- Create: `supabase/functions/_shared/crossfire-signals.test.ts` +- Modify: `supabase/functions/ai/index.ts` +- Modify: `supabase/functions/ai/handler-research.test.ts` +- Modify: `src/lib/crossfire/schemas.ts` +- Modify: `src/lib/crossfire/reducer.ts` +- Modify: `src/lib/ai/client.ts` +- Modify: `src/pages/CrossfirePage.tsx` +- Modify: `src/test/crossfire/schemas.test.ts` +- Modify: `src/test/crossfire/reducer.test.ts` +- Modify: `src/test/crossfire/storage.test.ts` +- Modify: `src/test/ai/crossfire-client.test.ts` +- Modify: `src/test/pages/CrossfirePage.test.tsx` +- Modify: `e2e/crossfire.spec.ts` +- Modify: `src/locales/en.json` +- Modify: `src/locales/es.json` +- Modify: `src/locales/hu.json` +- Modify: `docs/architecture.md` + +**Interfaces:** +- Consumes: `callJev` from Task 1 and resolved-model metadata from Task 2. +- Produces: `evaluateCrossfireExchange()` and optional `CrossfireTurnSignal` attached to an answer turn. +- Produces: next-turn response `{ output, signal?, meta }`; a missing signal is valid. + +- [ ] **Step 1: Write failing server signal tests** + +Cover both Crossfire modes: + +```ts +assertEquals(questionAi.signal?.answerTurnId, "T2"); +assertEquals(questionAi.signal?.answerQuality, "direct"); +assertEquals(aiQuestionsUser.signal?.answerTurnId, "T2"); +assertEquals(aiQuestionsUser.signal?.concession, "implicit"); +``` + +Also assert only the newest completed exchange is sent, no call occurs without a +question-answer pair, invalid answers return no signal, a Jev 500 returns the +valid main reply, and abort does not start or retry signal work. + +- [ ] **Step 2: Run signal tests and verify RED** + +Run: + +```bash +deno test --allow-read --allow-env=OPENROUTER_API_KEY --allow-net=127.0.0.1 supabase/functions/_shared/crossfire-signals.test.ts +``` + +Expected: FAIL because signal evaluation does not exist. + +- [ ] **Step 3: Implement signal evaluation and handler integration** + +Build the combined transcript plus validated new reply, derive the answer turn +ID from the sequence, and ask one choice plus concession choice plus one Noul. +Parse confidence and probabilities without generating prose. Catch non-abort +Jev errors after the main reply validates and return the reply without a signal. + +- [ ] **Step 4: Write failing client state and UI tests** + +Assert strict signal parsing, rejection of unknown/non-answer turn IDs, reducer +attachment, retry replacement, version-1 draft compatibility, restored signals, +visible localized labels, accessible probability explanation, and no labels on +questions. + +- [ ] **Step 5: Run client tests and verify RED** + +Run: + +```bash +PATH=/home/codex/.nvm/versions/node/v22.23.2/bin:$PATH npm test -- src/test/crossfire/schemas.test.ts src/test/crossfire/reducer.test.ts src/test/crossfire/storage.test.ts src/test/ai/crossfire-client.test.ts src/test/pages/CrossfirePage.test.tsx +``` + +Expected: FAIL because turns cannot carry signals. + +- [ ] **Step 6: Implement signal state and restrained transcript UI** + +Add the optional signal schema, pass it through the client, and let the reducer +attach it only to the matching answer turn. Reuse existing Badge and semantic +tokens. Show follow-up opportunity only above `0.70`; include its probability +in accessible text. Keep the turn card's existing motion and layout. + +- [ ] **Step 7: Run focused Crossfire suites GREEN** + +Run: + +```bash +deno test --allow-read --allow-env=DEMO_CODE_PEPPER,EXA_API_KEY,SUPABASE_URL,SUPABASE_ANON_KEY,SUPABASE_SERVICE_ROLE_KEY,SPARRING_INTEGRITY_SECRET,OPENROUTER_API_KEY,WORKSPACE_INVITE_SECRET,WORKSPACE_INVITE_MAILER_URL,WORKSPACE_INVITE_MAILER_TOKEN,WORKSPACE_INVITE_BASE_URL --allow-net=127.0.0.1 supabase/functions/_shared/crossfire-signals.test.ts supabase/functions/_shared/crossfire-response.test.ts supabase/functions/ai/prompt-builder.test.ts supabase/functions/ai/handler-research.test.ts +PATH=/home/codex/.nvm/versions/node/v22.23.2/bin:$PATH npm test -- src/test/crossfire/schemas.test.ts src/test/crossfire/reducer.test.ts src/test/crossfire/storage.test.ts src/test/ai/crossfire-client.test.ts src/test/pages/CrossfirePage.test.tsx src/test/i18n.test.ts +``` + +Expected: all tests pass. + +- [ ] **Step 8: Extend browser coverage** + +Update `e2e/crossfire.spec.ts` fixtures to return signals in both modes. Assert +that the answer receives the correct live label, questions do not, retry does +not duplicate labels, and the final transcript retains them. + +Run: + +```bash +PATH=/home/codex/.nvm/versions/node/v22.23.2/bin:$PATH CI=1 PORT=18080 PLAYWRIGHT_BASE_URL=http://127.0.0.1:18080 npx playwright test e2e/crossfire.spec.ts --project=desktop-chromium --project=mobile-chromium +``` + +Expected: desktop and mobile Crossfire tests pass. + +- [ ] **Step 9: Commit** + +```bash +git add supabase/functions/_shared/crossfire-signals.ts supabase/functions/_shared/crossfire-signals.test.ts supabase/functions/ai/index.ts supabase/functions/ai/handler-research.test.ts src/lib/crossfire/schemas.ts src/lib/crossfire/reducer.ts src/lib/ai/client.ts src/pages/CrossfirePage.tsx src/test/crossfire/schemas.test.ts src/test/crossfire/reducer.test.ts src/test/crossfire/storage.test.ts src/test/ai/crossfire-client.test.ts src/test/pages/CrossfirePage.test.tsx e2e/crossfire.spec.ts src/locales/en.json src/locales/es.json src/locales/hu.json docs/architecture.md +git commit -m "feat(crossfire): add live Jev coaching signals" +``` + +### Task 5: Live integration proof and release gate + +**Files:** +- Create: `scripts/smoke-jev.mjs` +- Modify: `package.json` +- Modify: `docs/testing.md` +- Modify: `supabase/README.md` + +**Interfaces:** +- Consumes: the final shared Jev request contract and existing local environment key. +- Produces: `npm run smoke:jev`, which prints only model, answer labels, latency, and usage counts. + +- [ ] **Step 1: Write the smoke script self-check first** + +The script accepts an injected fetch in its exported function. Add a Vitest test +that verifies it sends the pinned model, rejects missing keys before network +access, and never includes the key or submitted state in its returned summary. + +- [ ] **Step 2: Run the smoke test and verify RED** + +Run: + +```bash +PATH=/home/codex/.nvm/versions/node/v22.23.2/bin:$PATH npm test -- src/test/scripts/smoke-jev.test.ts +``` + +Expected: FAIL because the script does not exist. + +- [ ] **Step 3: Implement the reusable smoke command and docs** + +Use a tiny fixed classification state. Read `OPENROUTER_API_KEY` from the +environment, call the same endpoint and pinned model, print sanitized JSON, and +exit non-zero for malformed or unsuccessful results. Document what the smoke +proves and that it spends a negligible live request. + +- [ ] **Step 4: Run the live Jev smoke without exposing the key** + +Run: + +```bash +PATH=/home/codex/.nvm/versions/node/v22.23.2/bin:$PATH npm run smoke:jev +``` + +Expected: sanitized output identifies `typesafe/jev-1.13`, a valid typed answer, +latency, and token counts. If the key is unavailable locally, record the exact +missing-credential blocker and rely on mocked contract tests; do not print or +copy a credential. + +- [ ] **Step 5: Run the full local release gate** + +Run: + +```bash +PATH=/home/codex/.nvm/versions/node/v22.23.2/bin:$PATH npm run verify +git diff origin/main...HEAD --check +git status --short +``` + +Expected: `npm run verify` passes, diff check is clean, and the only working-tree +changes are the intended final task files before commit. + +- [ ] **Step 6: Verify the real UI path** + +Use the T3 collaborative preview against the current worktree. Verify desktop +and mobile: + +- Model picker selects and retains Auto Fast for Basic; Auto Pro is disabled. +- Sources defaults to Smart and visibly orders case-used evidence first. +- Generate displays used-source judgments and persists them after saving and + reloading the case. +- Both Crossfire modes show live signals on answers only. +- Browser console has no errors and keyboard controls remain usable. + +Expected: all four flows work on the running application, not only in fixtures. + +- [ ] **Step 7: Commit** + +```bash +git add scripts/smoke-jev.mjs src/test/scripts/smoke-jev.test.ts package.json docs/testing.md supabase/README.md +git commit -m "test(ai): add Jev integration verification" +``` + +- [ ] **Step 8: Whole-branch review and fix pass** + +Build the review package from the merge base and send it to a fresh reviewer. +Re-grade findings by user impact. Fix every Critical or Important finding in one +TDD pass, rerun the affected focused tests and full verification, and commit the +fixes. Record any deferred Minor item in the execution ledger. + +- [ ] **Step 9: Push, open, and babysit the pull request** + +```bash +git push -u origin HEAD +gh pr create --base main --title "feat: add Jev decisions to DebateLab" --body $'## Summary\n- add server-side Jev decisions through the existing OpenRouter key\n- add Auto Fast and Auto Pro routing\n- rank and retain sources with case-use judgments\n- add live Crossfire coaching signals\n\n## Verification\n- npm run verify\n- npm run smoke:jev\n- desktop and mobile browser checks\n\nMerge is intentionally left to Adam.' +gh pr checks --watch +``` + +Immediately register the PR with the T3 thread. Wait for current-head CI and a +substantive CodeRabbit review. Fix submitted-code failures and every valid +in-scope finding, push, and repeat until the current head is green and review is +clear. Do not merge or enable auto-merge. From 080de68752ed9d43de3881363d605c78b42306db Mon Sep 17 00:00:00 2001 From: Adam Date: Mon, 21 Sep 2026 15:56:32 +0000 Subject: [PATCH 3/9] feat(ai): add OpenRouter Jev decisions client --- supabase/functions/_shared/jev.test.ts | 205 ++++++++++++++++++++++ supabase/functions/_shared/jev.ts | 230 +++++++++++++++++++++++++ supabase/functions/_shared/research.ts | 2 +- 3 files changed, 436 insertions(+), 1 deletion(-) create mode 100644 supabase/functions/_shared/jev.test.ts create mode 100644 supabase/functions/_shared/jev.ts diff --git a/supabase/functions/_shared/jev.test.ts b/supabase/functions/_shared/jev.test.ts new file mode 100644 index 000000000..c7a7626ea --- /dev/null +++ b/supabase/functions/_shared/jev.test.ts @@ -0,0 +1,205 @@ +import { + callJev, + type JevQuestion, +} from "./jev.ts"; + +const QUESTIONS = { + directness: { + type: "choice", + instructions: "How directly does the answer address the question?", + criteria: { + direct: "Fully answers it", + partial: "Answers only part", + evasive: "Avoids it", + }, + }, + concession: { + type: "noul", + instructions: "Does the answer concede the challenged claim?", + criteria: { true: "Concedes it", false: "Does not concede it" }, + }, + strength: { + type: "score", + instructions: "How strong is the evidence?", + criteria: ["Weak", "Useful", "Decisive"], + }, +} as const satisfies Record; + +function jsonResponse(value: unknown, status = 200): Response { + return new Response(JSON.stringify(value), { + status, + headers: { "Content-Type": "application/json" }, + }); +} + +function validResponse() { + return { + model: "typesafe/jev-1.13", + answers: { + directness: { + type: "choice", + choice: "direct", + confidence: 0.91, + probabilities: { direct: 0.91, partial: 0.07, evasive: 0.02 }, + }, + concession: { type: "noul", noul: 0.12 }, + strength: { + type: "score", + score: 1.6, + confidence: 0.84, + probabilities: { "0": 0.05, "1": 0.3, "2": 0.65 }, + legend: { "0": "Weak", "1": "Useful", "2": "Decisive" }, + }, + }, + usage: { input_tokens: 41, output_tokens: 0, cost: 0.00000172 }, + }; +} + +async function rejectsWith(operation: () => Promise, expected: string): Promise { + try { + await operation(); + } catch (error) { + const message = error instanceof Error ? error.message : String(error); + if (!message.includes(expected)) throw new Error(`expected ${JSON.stringify(expected)} in ${JSON.stringify(message)}`); + return; + } + throw new Error(`expected rejection containing ${JSON.stringify(expected)}`); +} + +Deno.test("Jev sends the pinned model and parses typed answers", async () => { + let capturedUrl = ""; + let capturedBody: Record = {}; + let authorization = ""; + const fetchImpl: typeof fetch = (input, init) => { + capturedUrl = String(input); + capturedBody = JSON.parse(String(init?.body)); + authorization = new Headers(init?.headers).get("Authorization") ?? ""; + return Promise.resolve(jsonResponse(validResponse())); + }; + + const result = await callJev({ + state: { feature: "crossfire", text: "Question and answer" }, + questions: QUESTIONS, + apiKey: "test-openrouter-key", + fetchImpl, + }); + + if (capturedUrl !== "https://openrouter.ai/api/alpha/decisions") throw new Error(`unexpected URL: ${capturedUrl}`); + if (capturedBody.model !== "typesafe/jev-1.13") throw new Error(`unexpected model: ${String(capturedBody.model)}`); + if (authorization !== "Bearer test-openrouter-key") throw new Error("authorization header missing"); + if (result.answers.directness.type !== "choice" || result.answers.directness.choice !== "direct") { + throw new Error("choice answer was not parsed"); + } + if (result.answers.concession.type !== "noul" || result.answers.concession.noul !== 0.12) { + throw new Error("noul answer was not parsed"); + } + if (result.answers.strength.type !== "score" || result.answers.strength.score !== 1.6) { + throw new Error("score answer was not parsed"); + } + if (result.model !== "typesafe/jev-1.13") throw new Error("returned model missing"); + if (result.usage?.inputTokens !== 41 || result.usage.reportedCostUsd !== 0.00000172) { + throw new Error(`usage was not normalized: ${JSON.stringify(result.usage)}`); + } +}); + +Deno.test("Jev rejects a choice outside the submitted labels", async () => { + const response = validResponse(); + response.answers.directness.choice = "invented"; + await rejectsWith(() => callJev({ + state: "state", + questions: QUESTIONS, + apiKey: "test-key", + fetchImpl: () => Promise.resolve(jsonResponse(response)), + }), "directness"); +}); + +Deno.test("Jev rejects probabilities outside zero to one", async () => { + const response = validResponse(); + response.answers.directness.probabilities.direct = 1.2; + await rejectsWith(() => callJev({ + state: "state", + questions: QUESTIONS, + apiKey: "test-key", + fetchImpl: () => Promise.resolve(jsonResponse(response)), + }), "directness"); +}); + +Deno.test("Jev rejects a score outside the submitted rubric", async () => { + const response = validResponse(); + response.answers.strength.score = 3; + await rejectsWith(() => callJev({ + state: "state", + questions: QUESTIONS, + apiKey: "test-key", + fetchImpl: () => Promise.resolve(jsonResponse(response)), + }), "strength"); +}); + +Deno.test("Jev rejects a response missing a submitted answer", async () => { + const response = validResponse(); + delete (response.answers as Record).concession; + await rejectsWith(() => callJev({ + state: "state", + questions: QUESTIONS, + apiKey: "test-key", + fetchImpl: () => Promise.resolve(jsonResponse(response)), + }), "concession"); +}); + +Deno.test("Jev bounds successful response bodies", async () => { + const oversized = "x".repeat(1_048_577); + await rejectsWith(() => callJev({ + state: "state", + questions: QUESTIONS, + apiKey: "test-key", + fetchImpl: () => Promise.resolve(new Response(oversized, { status: 200 })), + }), "1048576"); +}); + +Deno.test("Jev reports bounded provider errors", async () => { + await rejectsWith(() => callJev({ + state: "state", + questions: QUESTIONS, + apiKey: "test-key", + fetchImpl: () => Promise.resolve(new Response("provider unavailable", { status: 503 })), + }), "503"); +}); + +Deno.test("Jev timeout aborts an in-flight fetch", async () => { + let observedAbort = false; + const fetchImpl: typeof fetch = (_input, init) => new Promise((_resolve, reject) => { + const onAbort = () => { + observedAbort = true; + reject(init?.signal?.reason ?? new DOMException("Aborted", "AbortError")); + }; + if (init?.signal?.aborted) onAbort(); + else init?.signal?.addEventListener("abort", onAbort, { once: true }); + }); + + await rejectsWith(() => callJev({ + state: "state", + questions: QUESTIONS, + apiKey: "test-key", + fetchImpl, + timeoutMs: 5, + }), "timed out"); + if (!observedAbort) throw new Error("timeout did not abort fetch"); +}); + +Deno.test("Jev propagates caller cancellation", async () => { + const controller = new AbortController(); + const fetchImpl: typeof fetch = (_input, init) => new Promise((_resolve, reject) => { + const onAbort = () => reject(init?.signal?.reason ?? new DOMException("Aborted", "AbortError")); + if (init?.signal?.aborted) onAbort(); + else init?.signal?.addEventListener("abort", onAbort, { once: true }); + }); + const pending = callJev({ + state: "state", + questions: QUESTIONS, + apiKey: "test-key", + fetchImpl, + signal: controller.signal, + }); + controller.abort(new DOMException("User cancelled", "AbortError")); + await rejectsWith(() => pending, "User cancelled"); +}); diff --git a/supabase/functions/_shared/jev.ts b/supabase/functions/_shared/jev.ts new file mode 100644 index 000000000..89c24e8a2 --- /dev/null +++ b/supabase/functions/_shared/jev.ts @@ -0,0 +1,230 @@ +import { + normalizeOpenRouterUsage, + type NormalizedUsage, +} from "./cost.ts"; +import { + MAX_PROVIDER_ERROR_BODY_BYTES, + MAX_PROVIDER_ERROR_BODY_CHARS, + MAX_PROVIDER_RESPONSE_BYTES, + readResponseText, +} from "./research.ts"; + +export const JEV_MODEL = "typesafe/jev-1.13"; +export const JEV_DECISIONS_URL = "https://openrouter.ai/api/alpha/decisions"; +const JEV_TIMEOUT_MS = 8_000; + +type JsonPrimitive = string | number | boolean | null; +export type JevState = JsonPrimitive | JevState[] | { [key: string]: JevState }; + +export interface JevNoulQuestion { + type: "noul"; + instructions: string; + criteria?: { true: string; false: string }; +} + +export interface JevChoiceQuestion { + type: "choice"; + instructions: string; + criteria: Record; +} + +export interface JevScoreQuestion { + type: "score"; + instructions: string; + criteria: readonly [string, string, ...string[]]; +} + +export type JevQuestion = JevNoulQuestion | JevChoiceQuestion | JevScoreQuestion; + +export interface JevNoulAnswer { + type: "noul"; + noul: number; +} + +export interface JevChoiceAnswer { + type: "choice"; + choice: string; + confidence: number; + probabilities: Record; +} + +export interface JevScoreAnswer { + type: "score"; + score: number; + confidence: number; + probabilities: Record; + legend: Record; +} + +export type JevAnswer = JevNoulAnswer | JevChoiceAnswer | JevScoreAnswer; + +export interface JevDecisionResult { + model: string; + answers: Record; + usage?: NormalizedUsage; +} + +interface CallJevArgs { + state: JevState; + questions: Record; + signal?: AbortSignal; + fetchImpl?: typeof fetch; + apiKey?: string; + timeoutMs?: number; +} + +function asRecord(value: unknown): Record | null { + return value && typeof value === "object" && !Array.isArray(value) + ? value as Record + : null; +} + +function boundedProbability(value: unknown): number | null { + return typeof value === "number" && Number.isFinite(value) && value >= 0 && value <= 1 + ? value + : null; +} + +function sameKeys(value: Record, expected: readonly string[]): boolean { + const actual = Object.keys(value).sort(); + const wanted = [...expected].sort(); + return actual.length === wanted.length && actual.every((key, index) => key === wanted[index]); +} + +function parseProbabilities(value: unknown, expected: readonly string[]): Record | null { + const record = asRecord(value); + if (!record || !sameKeys(record, expected)) return null; + const parsed: Record = {}; + for (const key of expected) { + const probability = boundedProbability(record[key]); + if (probability === null) return null; + parsed[key] = probability; + } + return parsed; +} + +function parseAnswer(id: string, question: JevQuestion, value: unknown): JevAnswer { + const answer = asRecord(value); + if (!answer || answer.type !== question.type) throw new Error(`Invalid Jev answer for ${id}`); + + if (question.type === "noul") { + const noul = boundedProbability(answer.noul); + if (noul === null) throw new Error(`Invalid Jev answer for ${id}`); + return { type: "noul", noul }; + } + + const confidence = boundedProbability(answer.confidence); + if (confidence === null) throw new Error(`Invalid Jev answer for ${id}`); + + if (question.type === "choice") { + const labels = Object.keys(question.criteria); + const choice = typeof answer.choice === "string" && labels.includes(answer.choice) + ? answer.choice + : null; + const probabilities = parseProbabilities(answer.probabilities, labels); + if (!choice || !probabilities) throw new Error(`Invalid Jev answer for ${id}`); + return { type: "choice", choice, confidence, probabilities }; + } + + const score = typeof answer.score === "number" && Number.isFinite(answer.score) && + answer.score >= 0 && answer.score <= question.criteria.length - 1 + ? answer.score + : null; + const indexes = question.criteria.map((_, index) => String(index)); + const probabilities = parseProbabilities(answer.probabilities, indexes); + const rawLegend = asRecord(answer.legend); + if (score === null || !probabilities || !rawLegend || !sameKeys(rawLegend, indexes)) { + throw new Error(`Invalid Jev answer for ${id}`); + } + const legend: Record = {}; + for (const index of indexes) { + if (rawLegend[index] !== question.criteria[Number(index)]) { + throw new Error(`Invalid Jev answer for ${id}`); + } + legend[index] = question.criteria[Number(index)]; + } + return { type: "score", score, confidence, probabilities, legend }; +} + +function normalizeJevUsage(value: unknown): NormalizedUsage | undefined { + const usage = asRecord(value); + if (!usage) return undefined; + const normalized = normalizeOpenRouterUsage({ + ...usage, + prompt_tokens: usage.prompt_tokens ?? usage.input_tokens, + completion_tokens: usage.completion_tokens ?? usage.output_tokens, + }); + return { ...normalized, raw: usage }; +} + +export async function callJev({ + state, + questions, + signal, + fetchImpl = globalThis.fetch, + apiKey = Deno.env.get("OPENROUTER_API_KEY"), + timeoutMs = JEV_TIMEOUT_MS, +}: CallJevArgs): Promise { + if (!apiKey) throw new Error("OPENROUTER_API_KEY missing"); + const questionIds = Object.keys(questions); + if (questionIds.length === 0) throw new Error("Jev requires at least one question"); + + const controller = new AbortController(); + const abortFromCaller = () => controller.abort(signal?.reason ?? new DOMException("The operation was aborted", "AbortError")); + if (signal?.aborted) abortFromCaller(); + else signal?.addEventListener("abort", abortFromCaller, { once: true }); + const timeoutId = setTimeout(() => { + controller.abort(new DOMException("Jev request timed out", "TimeoutError")); + }, timeoutMs); + + try { + const response = await fetchImpl(JEV_DECISIONS_URL, { + method: "POST", + headers: { + "Authorization": `Bearer ${apiKey}`, + "Content-Type": "application/json", + }, + body: JSON.stringify({ model: JEV_MODEL, state, questions }), + signal: controller.signal, + }); + + if (!response.ok) { + const body = await readResponseText( + response, + MAX_PROVIDER_ERROR_BODY_BYTES, + MAX_PROVIDER_ERROR_BODY_CHARS, + controller.signal, + true, + ); + throw new Error(`OpenRouter Jev error ${response.status}: ${body.text}`); + } + + const body = await readResponseText( + response, + MAX_PROVIDER_RESPONSE_BYTES, + MAX_PROVIDER_RESPONSE_BYTES, + controller.signal, + false, + ); + const payload = asRecord(JSON.parse(body.text)); + const rawAnswers = asRecord(payload?.answers); + const model = typeof payload?.model === "string" && payload.model ? payload.model : null; + if (!payload || !rawAnswers || !model) { + throw new Error("Invalid Jev response envelope"); + } + for (const id of questionIds) { + if (!(id in rawAnswers)) throw new Error(`Missing Jev answer for ${id}`); + } + if (!sameKeys(rawAnswers, questionIds)) throw new Error("Invalid Jev response envelope"); + + const answers: Record = {}; + for (const id of questionIds) answers[id] = parseAnswer(id, questions[id], rawAnswers[id]); + return { model, answers, usage: normalizeJevUsage(payload.usage) }; + } catch (error) { + if (controller.signal.aborted) throw controller.signal.reason; + throw error; + } finally { + clearTimeout(timeoutId); + signal?.removeEventListener("abort", abortFromCaller); + } +} diff --git a/supabase/functions/_shared/research.ts b/supabase/functions/_shared/research.ts index 3caea4bd8..9f21f9f24 100644 --- a/supabase/functions/_shared/research.ts +++ b/supabase/functions/_shared/research.ts @@ -184,7 +184,7 @@ function joinBytes(chunks: Uint8Array[], totalBytes: number): Uint8Array { return bytes; } -async function readResponseText( +export async function readResponseText( response: Response, maxBytes: number, maxChars: number, From 2128ff4b4f248d17396aceabc8d6676498e48e6e Mon Sep 17 00:00:00 2001 From: Adam Date: Mon, 21 Sep 2026 16:07:44 +0000 Subject: [PATCH 4/9] feat(ai): add Jev automatic model routing --- docs/architecture.md | 2 + src/components/ModelSelector.tsx | 38 +++- src/lib/ai/client.ts | 9 +- src/lib/ai/providers/types.ts | 15 +- src/lib/ai/streaming.ts | 14 +- src/lib/stores/settings-store.ts | 17 +- src/locales/en.json | 2 + src/locales/es.json | 2 + src/locales/hu.json | 2 + src/test/ai/client-analytics.test.ts | 20 ++ src/test/ai/streaming-integration.test.ts | 18 ++ src/test/components/ModelSelector.test.tsx | 38 ++++ src/test/settings-store.test.ts | 68 +++++++ .../_shared/auto-model-router.test.ts | 184 ++++++++++++++++++ .../functions/_shared/auto-model-router.ts | 126 ++++++++++++ supabase/functions/_shared/jev.ts | 2 +- supabase/functions/_shared/model-catalog.ts | 18 ++ supabase/functions/ai-stream/handler_test.ts | 26 ++- supabase/functions/ai-stream/index.ts | 58 ++++-- .../functions/ai/handler-research.test.ts | 42 +++- supabase/functions/ai/index.ts | 68 +++++-- 21 files changed, 722 insertions(+), 47 deletions(-) create mode 100644 supabase/functions/_shared/auto-model-router.test.ts create mode 100644 supabase/functions/_shared/auto-model-router.ts diff --git a/docs/architecture.md b/docs/architecture.md index 0992fa7c9..6aba0a8c0 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -91,6 +91,8 @@ Checkout and portal functions create Stripe sessions after authenticated ownersh All model requests use the single OpenRouter backend route and `OPENROUTER_API_KEY`. The shared ten-model catalog, Luna Basic default, Pro-only entries, and request controls live in [`supabase/functions/_shared/model-catalog.ts`](../supabase/functions/_shared/model-catalog.ts). OpenRouter usage remains raw in the audit row, and the reported charge is unknown when the provider does not return a complete cost. +`Auto Fast` and `Auto Pro` are virtual selections, not provider model IDs. The Edge Functions resolve them through TypeSafe Jev on OpenRouter before calling the concrete writing model. Auto Fast is available to Basic accounts and normally resolves to GLM 5.3 Flash; manual GLM selection keeps its Pro restriction. Auto Pro remains Pro-only. Provider calls, usage rows, and response metadata record the concrete resolved model, while response metadata also retains the requested automatic selection. A Jev router call uses the same OpenRouter key and does not consume a second daily quota unit. + Historical provider pricing evidence is retained under [`docs/pricing-snapshots/`](pricing-snapshots/). Those snapshots support legacy calculator records and are evidence, not runtime configuration for the OpenRouter route. ## Observability diff --git a/src/components/ModelSelector.tsx b/src/components/ModelSelector.tsx index a7c666d62..e081f35fe 100644 --- a/src/components/ModelSelector.tsx +++ b/src/components/ModelSelector.tsx @@ -1,7 +1,13 @@ import { Cpu } from 'lucide-react'; import { Select, SelectContent, SelectItem, SelectTrigger, SelectValue } from '@/components/ui/select'; import { Badge } from '@/components/ui/badge'; -import { DEFAULT_MODEL, DEFAULT_PROVIDERS, isReasoningEffort } from '@/lib/ai/providers/types'; +import { + AUTO_MODEL_SELECTIONS, + DEFAULT_MODEL, + DEFAULT_PROVIDERS, + isAutoModelSelection, + isReasoningEffort, +} from '@/lib/ai/providers/types'; import type { ReasoningEffort } from '@/lib/ai/providers/types'; import { useSettingsStore, type FeatureTab } from '@/lib/stores/settings-store'; import { useSubscription } from '@/hooks/use-subscription'; @@ -15,8 +21,12 @@ interface ModelSelectorProps { showLabel?: boolean; } -const allModels = DEFAULT_PROVIDERS - .flatMap(p => p.models.map(m => ({ ...m, provider: p.id, providerName: p.name }))); +const allModels = [ + ...AUTO_MODEL_SELECTIONS.map((model) => ({ ...model, provider: 'openrouter', providerName: 'OpenRouter', isAuto: true })), + ...DEFAULT_PROVIDERS.flatMap((provider) => + provider.models.map((model) => ({ ...model, provider: provider.id, providerName: provider.name, isAuto: false })) + ), +]; const reasoningOptions: { value: ReasoningEffort; labelKey: string; fallback: string }[] = [ { value: 'low', labelKey: 'modelSelector.reasoningLow', fallback: 'Low' }, @@ -31,7 +41,8 @@ export function ModelSelector({ feature, className, showLabel = false }: ModelSe const config = featureConfigs?.[feature]; const selectedModel = allModels.find((model) => model.id === config?.model); const currentModel = selectedModel?.id ?? DEFAULT_MODEL; - const currentReasoning = (isPro || subscriptionLoading) && config?.reasoningEffort && isReasoningEffort(config.reasoningEffort) + const automatic = isAutoModelSelection(currentModel); + const currentReasoning = !automatic && (isPro || subscriptionLoading) && config?.reasoningEffort && isReasoningEffort(config.reasoningEffort) ? config.reasoningEffort : 'low'; @@ -65,8 +76,19 @@ export function ModelSelector({ feature, className, showLabel = false }: ModelSe disabled={m.isProOnly && !isPro} > - {m.name} - {m.isProOnly && Pro} + + + {m.name} + {m.isProOnly && Pro} + + {m.isAuto && ( + + {m.id === 'auto-fast' + ? t('modelSelector.autoFastDescription', 'Fast routing, usually GLM') + : t('modelSelector.autoProDescription', 'Routes among the strongest models')} + + )} + ))} @@ -78,9 +100,9 @@ export function ModelSelector({ feature, className, showLabel = false }: ModelSe { beforeEach(() => { vi.clearAllMocks(); + sessionState.sourceLibrary = [ + { title: 'Active source', content: 'Useful evidence', origin: 'https://example.com', type: 'url', isActive: true }, + { title: 'Inactive source', content: 'Ignore me', origin: 'https://example.com/2', type: 'url', isActive: false }, + ]; getSessionMock.mockResolvedValue({ data: { session: { access_token: 'token-123' } }, }); @@ -266,6 +272,33 @@ describe('AI client branch coverage', () => { }); }); + it('sends stable source IDs and applies returned semantic assessments', async () => { + sessionState.sourceLibrary[0] = { ...sessionState.sourceLibrary[0], id: 'source-active', metadata: { qualityLabel: 'strong' } } as any; + invokeMock.mockResolvedValueOnce({ + data: { + output: validGenerateOutput, + sourceAssessments: [{ key: 'source-active', id: 'source-active', relevance: 0.9, importance: 3, strength: 2, evaluatedAt: '2026-09-21T00:00:00.000Z', retained: true, usedInCase: true }], + }, + error: null, + }); + + await generateCase(validGenerateInput, config); + expect(invokeMock).toHaveBeenCalledWith('ai', expect.objectContaining({ body: expect.objectContaining({ sources: [expect.objectContaining({ id: 'source-active' })] }) })); + expect(sessionState.applySourceAssessments).toHaveBeenCalledWith([expect.objectContaining({ id: 'source-active', usedInCase: true })]); + }); + + it('does not autosave a semantically irrelevant discovered source', async () => { + invokeMock.mockResolvedValueOnce({ + data: { + output: validGenerateOutput, + discoveredSources: [{ title: 'Irrelevant', url: 'https://example.com/irrelevant', qualityLabel: 'strong', semantic: { relevance: 0.2, importance: 0, strength: 2, evaluatedAt: '2026-09-21T00:00:00.000Z', useCount: 1, caseUseCount: 0, lastUsedInCase: false } }], + }, + error: null, + }); + await generateCase(validGenerateInput, config); + expect(addSourceMock).not.toHaveBeenCalled(); + }); + it('rejects malformed nested direct-format generate output before saving sources', async () => { invokeMock.mockResolvedValueOnce({ data: { diff --git a/src/test/components/SourceQualityList.test.tsx b/src/test/components/SourceQualityList.test.tsx index 65773c47d..584cdb36e 100644 --- a/src/test/components/SourceQualityList.test.tsx +++ b/src/test/components/SourceQualityList.test.tsx @@ -73,4 +73,25 @@ describe('SourceQualityList', () => { expect(screen.getByText('Kormányzati')).toBeInTheDocument(); expect(screen.getByText('Erős')).toBeInTheDocument(); }); + + it('shows case use before semantic importance and strength', () => { + render(); + + expect(screen.getByText('Used in this case')).toBeInTheDocument(); + expect(screen.getByText('High importance')).toBeInTheDocument(); + expect(screen.getByText('Medium strength')).toBeInTheDocument(); + }); }); diff --git a/src/test/pages/LibraryPage.test.tsx b/src/test/pages/LibraryPage.test.tsx index 30295b36e..7540beb65 100644 --- a/src/test/pages/LibraryPage.test.tsx +++ b/src/test/pages/LibraryPage.test.tsx @@ -110,6 +110,19 @@ describe('LibraryPage', () => { expect(screen.getByText('Source B')).toBeDefined(); }); + it('defaults to Smart ordering and puts case-used evidence first', () => { + mockStore.sourceLibrary = [ + { id: 'new', title: 'New source', content: 'A', type: 'text', isActive: true, createdAt: 2 }, + { id: 'used', title: 'Used source', content: 'B', type: 'text', isActive: true, createdAt: 1, metadata: { semantic: { relevance: 0.8, importance: 2, strength: 3, evaluatedAt: '2026-09-21T00:00:00.000Z', useCount: 4, caseUseCount: 1, lastUsedInCase: true } } }, + ]; + render(); + + expect(screen.getByRole('combobox', { name: 'Sort by' })).toHaveTextContent('Smart'); + const labels = screen.getAllByText(/source$/i).map((element) => element.textContent); + expect(labels.indexOf('Used source')).toBeLessThan(labels.indexOf('New source')); + expect(screen.getByText('Used in this case')).toBeInTheDocument(); + }); + it('shows grounding status', () => { mockStore.sourceLibrary = [ { id: '1', title: 'Active Source', content: 'C', type: 'text', isActive: true, createdAt: Date.now() }, diff --git a/src/test/session-store.test.ts b/src/test/session-store.test.ts index d76ff729d..f9e6d8710 100644 --- a/src/test/session-store.test.ts +++ b/src/test/session-store.test.ts @@ -391,6 +391,81 @@ describe("session-store", () => { }); }); + it("parses valid semantic metadata and discards only an invalid semantic object", async () => { + setMockTableResponse("sources", { + order: [ + { + id: "semantic-valid", title: "Valid", content: "c", type: "url", origin: "https://example.com/valid", is_active: true, created_at: new Date().toISOString(), + metadata: { qualityLabel: "strong", semantic: { relevance: 0.9, importance: 3, strength: 2, evaluatedAt: "2026-09-21T00:00:00.000Z", useCount: 4, caseUseCount: 2, lastUsedInCase: true } }, + }, + { + id: "semantic-invalid", title: "Invalid", content: "c", type: "url", origin: "https://example.com/invalid", is_active: true, created_at: new Date().toISOString(), + metadata: { qualityLabel: "moderate", semantic: { relevance: 7, importance: 5 } }, + }, + ], + }); + + await useSessionStore.getState().hydrateFromServer(); + expect(useSessionStore.getState().sourceLibrary.find((source) => source.id === "semantic-valid")?.metadata?.semantic?.caseUseCount).toBe(2); + expect(useSessionStore.getState().sourceLibrary.find((source) => source.id === "semantic-invalid")?.metadata).toEqual({ qualityLabel: "moderate" }); + }); + + it("persists metadata when updating a source", async () => { + setMockAuthUser({ id: "user_1" }); + useSessionStore.setState({ sourceLibrary: [{ id: "source_1", title: "Source", content: "Content", type: "text", createdAt: 1, isActive: true }] }); + const updates: Array> = []; + const originalFrom = supabaseMock.from.getMockImplementation() as unknown as (table: string) => ReturnType; + supabaseMock.from.mockImplementation(((table: string) => { + const builder = originalFrom(table); + if (table === "sources") { + const originalUpdate = builder.update; + builder.update = ((payload: Record) => { + updates.push(payload); + return (originalUpdate as unknown as (p: Record) => ReturnType)(payload); + }) as typeof builder.update; + } + return builder; + }) as never); + const semantic = { relevance: 0.8, importance: 2 as const, strength: 3 as const, evaluatedAt: "2026-09-21T00:00:00.000Z", useCount: 1, caseUseCount: 1, lastUsedInCase: true }; + try { + await useSessionStore.getState().updateSource("source_1", { metadata: { semantic } }); + } finally { + supabaseMock.from.mockImplementation(originalFrom as never); + } + expect(updates[0].metadata).toEqual({ semantic }); + }); + + it("merges semantic assessments without losing source provenance", async () => { + useSessionStore.setState({ sourceLibrary: [{ + id: "source_1", + title: "Source", + content: "Content", + type: "url", + origin: "https://example.com/source", + createdAt: 1, + isActive: true, + metadata: { url: "https://example.com/source", query: "original query", qualityLabel: "strong" }, + }] }); + + await useSessionStore.getState().applySourceAssessments([{ + key: "source_1", + id: "source_1", + relevance: 0.9, + importance: 3, + strength: 2, + evaluatedAt: "2026-09-21T00:00:00.000Z", + retained: true, + usedInCase: true, + }]); + + expect(useSessionStore.getState().sourceLibrary[0].metadata).toEqual(expect.objectContaining({ + url: "https://example.com/source", + query: "original query", + qualityLabel: "strong", + semantic: expect.objectContaining({ useCount: 1, caseUseCount: 1, lastUsedInCase: true }), + })); + }); + it("persists local collections but excludes pending session state", () => { const partialize = useSessionStore.persist.getOptions().partialize!; const savedCases = [{ id: "case_1", title: "Case", createdAt: Date.now(), data: sampleGenerate }]; diff --git a/supabase/functions/_shared/source-decisions.test.ts b/supabase/functions/_shared/source-decisions.test.ts new file mode 100644 index 000000000..a51bca6d2 --- /dev/null +++ b/supabase/functions/_shared/source-decisions.test.ts @@ -0,0 +1,96 @@ +import { + identifyCaseSourceUsage, + rankSourcesForRequest, + type SourceDecisionJev, +} from "./source-decisions.ts"; +import type { CallJevArgs, JevDecisionResult } from "./jev.ts"; + +const sources = [ + { key: "official", title: "Official report", content: "Primary statistics", type: "url", qualityScore: 90, qualityLabel: "strong" as const }, + { key: "study", title: "Peer reviewed study", content: "Measured effects", type: "url", qualityScore: 80, qualityLabel: "strong" as const }, + { key: "forum", title: "Forum post", content: "Personal opinion", type: "url", qualityScore: 10, qualityLabel: "weak" as const }, +]; + +function answersFor(args: CallJevArgs, values: Array<{ relevance: number; importance: number; strength: number; confidence?: number }>): JevDecisionResult { + const answers: JevDecisionResult["answers"] = {}; + values.forEach((value, index) => { + answers[`source_${index}_relevance`] = { type: "noul", noul: value.relevance }; + for (const field of ["importance", "strength"] as const) { + const score = value[field]; + answers[`source_${index}_${field}`] = { + type: "score", + score, + confidence: value.confidence ?? 0.9, + probabilities: { "0": 0.01, "1": 0.02, "2": 0.07, "3": 0.9 }, + legend: { "0": "None", "1": "Low", "2": "Medium", "3": "High" }, + }; + } + }); + for (const id of Object.keys(args.questions)) if (!answers[id]) throw new Error(`missing fixture answer ${id}`); + return { model: "typesafe/jev-1.13", answers }; +} + +Deno.test("semantic source ranking keeps weak evidence below non-weak candidates", async () => { + const callJevImpl: SourceDecisionJev = (args) => Promise.resolve(answersFor(args, [ + { relevance: 0.8, importance: 1, strength: 1 }, + { relevance: 0.9, importance: 3, strength: 3 }, + { relevance: 0.99, importance: 3, strength: 3 }, + ])); + const result = await rankSourcesForRequest({ feature: "generate", requestText: "motion", sources, callJevImpl }); + if (result.sources.map((source) => source.key).join(",") !== "study,official,forum") { + throw new Error(`unexpected order: ${result.sources.map((source) => source.key)}`); + } + if (result.assessments.find((item) => item.key === "forum")?.retained) throw new Error("weak source was retained"); +}); + +Deno.test("all low-confidence semantic scores preserve deterministic source order", async () => { + const callJevImpl: SourceDecisionJev = (args) => Promise.resolve(answersFor(args, [ + { relevance: 0.9, importance: 1, strength: 1, confidence: 0.3 }, + { relevance: 0.95, importance: 3, strength: 3, confidence: 0.3 }, + { relevance: 0.99, importance: 3, strength: 3, confidence: 0.3 }, + ])); + const result = await rankSourcesForRequest({ feature: "generate", requestText: "motion", sources, callJevImpl }); + if (result.sources.map((source) => source.key).join(",") !== "official,study,forum") throw new Error("low-confidence batch reordered sources"); +}); + +Deno.test("failed or malformed Jev source decisions preserve deterministic source order", async () => { + for (const callJevImpl of [ + () => Promise.reject(new Error("unavailable")), + () => Promise.resolve({ model: "typesafe/jev-1.13", answers: {} }), + ] satisfies SourceDecisionJev[]) { + const result = await rankSourcesForRequest({ feature: "generate", requestText: "motion", sources, callJevImpl }); + if (result.sources.map((source) => source.key).join(",") !== "official,study,forum") throw new Error("failed batch reordered sources"); + } +}); + +Deno.test("case use requires probability above 0.70 and keeps the middle band uncertain", async () => { + const callJevImpl: SourceDecisionJev = (args) => Promise.resolve({ + model: "typesafe/jev-1.13", + answers: { + source_0_used: { type: "noul", noul: 0.71 }, + source_1_used: { type: "noul", noul: 0.7 }, + source_2_used: { type: "noul", noul: 0.29 }, + }, + }); + const result = await identifyCaseSourceUsage({ caseText: "generated case", sources, callJevImpl }); + if (result.sourceUsage.map((item) => item.judgment).join(",") !== "used,uncertain,not_used") { + throw new Error(`unexpected judgments: ${result.sourceUsage.map((item) => item.judgment)}`); + } +}); + +Deno.test("source decisions bound candidate count, excerpts, and case snapshots", async () => { + let observedState = ""; + const manySources = Array.from({ length: 14 }, (_, index) => ({ + key: `s${index}`, + title: `Source ${index}`, + content: "x".repeat(9_000), + type: "text", + })); + const callJevImpl: SourceDecisionJev = (args) => { + observedState = JSON.stringify(args.state); + return Promise.resolve(answersFor(args, Array.from({ length: 10 }, () => ({ relevance: 0.9, importance: 2, strength: 2 })))); + }; + const ranked = await rankSourcesForRequest({ feature: "ask", requestText: "y".repeat(20_000), sources: manySources, callJevImpl }); + if (ranked.assessments.length !== 10 || ranked.sources.length !== 14) throw new Error("candidate bounds changed source count"); + if (observedState.length > 32_000) throw new Error(`source state exceeded bound: ${observedState.length}`); +}); diff --git a/supabase/functions/_shared/source-decisions.ts b/supabase/functions/_shared/source-decisions.ts new file mode 100644 index 000000000..64adc3e93 --- /dev/null +++ b/supabase/functions/_shared/source-decisions.ts @@ -0,0 +1,210 @@ +import { + callJev, + type CallJevArgs, + type JevDecisionResult, + type JevNoulAnswer, + type JevScoreAnswer, +} from "./jev.ts"; + +const MAX_SOURCES = 10; +const MAX_REQUEST_CHARS = 8_000; +const MAX_SOURCE_CHARS = 1_500; +const SCORE_CONFIDENCE = 0.55; +const RETAIN_RELEVANCE = 0.45; +const SCORE_CRITERIA = ["None", "Low", "Medium", "High"] as const; + +export interface SourceDecisionCandidate { + key: string; + id?: string; + title: string; + content: string; + origin?: string; + type: string; + qualityScore?: number; + qualityLabel?: "strong" | "moderate" | "weak"; +} + +export interface SourceAssessment { + key: string; + id?: string; + relevance: number; + importance: 0 | 1 | 2 | 3; + strength: 0 | 1 | 2 | 3; + evaluatedAt: string; + retained: boolean; +} + +export interface CaseSourceUsage { + key: string; + id?: string; + title: string; + origin?: string; + importance: 0 | 1 | 2 | 3; + strength: 0 | 1 | 2 | 3; + probability: number; + judgment: "used" | "uncertain" | "not_used"; +} + +export type SourceDecisionJev = (args: CallJevArgs) => Promise; + +function isAbort(error: unknown, signal?: AbortSignal): boolean { + return signal?.aborted === true || error instanceof DOMException && error.name === "AbortError"; +} + +function scoreAnswer(value: unknown): JevScoreAnswer | null { + if (!value || typeof value !== "object" || (value as { type?: unknown }).type !== "score") return null; + const answer = value as JevScoreAnswer; + return Number.isInteger(answer.score) && answer.score >= 0 && answer.score <= 3 && + Number.isFinite(answer.confidence) && answer.confidence >= 0 && answer.confidence <= 1 + ? answer + : null; +} + +function noulAnswer(value: unknown): JevNoulAnswer | null { + if (!value || typeof value !== "object" || (value as { type?: unknown }).type !== "noul") return null; + const answer = value as JevNoulAnswer; + return Number.isFinite(answer.noul) && answer.noul >= 0 && answer.noul <= 1 ? answer : null; +} + +function boundedSources(sources: readonly SourceDecisionCandidate[]) { + return sources.slice(0, MAX_SOURCES).map((source, index) => ({ + index, + key: source.key.slice(0, 500), + title: source.title.slice(0, 300), + excerpt: source.content.slice(0, MAX_SOURCE_CHARS), + type: source.type.slice(0, 64), + ...(source.origin ? { origin: source.origin.slice(0, 500) } : {}), + ...(source.qualityLabel ? { qualityLabel: source.qualityLabel } : {}), + ...(source.qualityScore !== undefined ? { qualityScore: source.qualityScore } : {}), + })); +} + +export async function rankSourcesForRequest({ + feature, + requestText, + sources, + signal, + callJevImpl = callJev, +}: { + feature: string; + requestText: string; + sources: SourceDecisionCandidate[]; + signal?: AbortSignal; + callJevImpl?: SourceDecisionJev; +}): Promise<{ sources: SourceDecisionCandidate[]; assessments: SourceAssessment[]; jev?: JevDecisionResult }> { + const candidates = boundedSources(sources); + if (candidates.length === 0) return { sources, assessments: [] }; + const questions: CallJevArgs["questions"] = {}; + candidates.forEach((_, index) => { + questions[`source_${index}_relevance`] = { + type: "noul", + instructions: "Is this source relevant to the user's requested output?", + criteria: { true: "Materially relevant", false: "Not materially relevant" }, + }; + questions[`source_${index}_importance`] = { + type: "score", + instructions: "How important is this source to producing the requested output?", + criteria: SCORE_CRITERIA, + }; + questions[`source_${index}_strength`] = { + type: "score", + instructions: "How strong is the source as evidence for the requested output?", + criteria: SCORE_CRITERIA, + }; + }); + + try { + const jev = await callJevImpl({ + state: { feature: feature.slice(0, 64), request: requestText.slice(0, MAX_REQUEST_CHARS), sources: candidates }, + questions, + signal, + }); + const evaluatedAt = new Date().toISOString(); + const assessments = candidates.map((candidate, index): SourceAssessment | null => { + const relevance = noulAnswer(jev.answers[`source_${index}_relevance`]); + const importance = scoreAnswer(jev.answers[`source_${index}_importance`]); + const strength = scoreAnswer(jev.answers[`source_${index}_strength`]); + if (!relevance || !importance || !strength) return null; + return { + key: candidate.key, + ...(sources[index].id ? { id: sources[index].id } : {}), + relevance: relevance.noul, + importance: importance.score as 0 | 1 | 2 | 3, + strength: strength.score as 0 | 1 | 2 | 3, + evaluatedAt, + retained: sources[index].qualityLabel !== "weak" && relevance.noul >= RETAIN_RELEVANCE, + }; + }); + if (assessments.some((assessment) => assessment === null)) return { sources, assessments: [] }; + const valid = assessments as SourceAssessment[]; + const hasConfidentAssessment = candidates.some((_, index) => { + const importance = scoreAnswer(jev.answers[`source_${index}_importance`]); + const strength = scoreAnswer(jev.answers[`source_${index}_strength`]); + return Boolean(importance && strength && importance.confidence >= SCORE_CONFIDENCE && strength.confidence >= SCORE_CONFIDENCE); + }); + if (!hasConfidentAssessment) return { sources, assessments: valid, jev }; + + const assessmentByKey = new Map(valid.map((assessment) => [assessment.key, assessment])); + const ranked = sources.map((source, index) => ({ source, index })).sort((a, b) => { + const weakDifference = Number(a.source.qualityLabel === "weak") - Number(b.source.qualityLabel === "weak"); + if (weakDifference !== 0) return weakDifference; + const left = assessmentByKey.get(a.source.key); + const right = assessmentByKey.get(b.source.key); + if (!left || !right) return a.index - b.index; + return right.relevance - left.relevance || + right.importance - left.importance || + right.strength - left.strength || + (b.source.qualityScore ?? 0) - (a.source.qualityScore ?? 0) || + a.index - b.index; + }).map(({ source }) => source); + return { sources: ranked, assessments: valid, jev }; + } catch (error) { + if (isAbort(error, signal)) throw error; + return { sources, assessments: [] }; + } +} + +export async function identifyCaseSourceUsage({ + caseText, + sources, + signal, + callJevImpl = callJev, +}: { + caseText: string; + sources: Array>>; + signal?: AbortSignal; + callJevImpl?: SourceDecisionJev; +}): Promise<{ sourceUsage: CaseSourceUsage[]; jev?: JevDecisionResult }> { + const candidates = boundedSources(sources); + if (candidates.length === 0) return { sourceUsage: [] }; + const questions: CallJevArgs["questions"] = {}; + candidates.forEach((_, index) => { + questions[`source_${index}_used`] = { + type: "noul", + instructions: "Did this source materially inform a claim, warrant, evidence statement, or impact in the generated case?", + criteria: { true: "Materially used", false: "Not materially used" }, + }; + }); + try { + const jev = await callJevImpl({ state: { case: caseText.slice(0, 16_000), sources: candidates }, questions, signal }); + const sourceUsage = candidates.flatMap((candidate, index): CaseSourceUsage[] => { + const answer = noulAnswer(jev.answers[`source_${index}_used`]); + if (!answer) return []; + const source = sources[index]; + return [{ + key: source.key, + ...(source.id ? { id: source.id } : {}), + title: source.title.slice(0, 300), + ...(source.origin ? { origin: source.origin.slice(0, 500) } : {}), + importance: source.importance ?? 0, + strength: source.strength ?? 0, + probability: answer.noul, + judgment: answer.noul > 0.70 ? "used" : answer.noul >= 0.30 ? "uncertain" : "not_used", + }]; + }); + return sourceUsage.length === candidates.length ? { sourceUsage, jev } : { sourceUsage: [] }; + } catch (error) { + if (isAbort(error, signal)) throw error; + return { sourceUsage: [] }; + } +} diff --git a/supabase/functions/ai/handler-research.test.ts b/supabase/functions/ai/handler-research.test.ts index 960c34229..109542c3c 100644 --- a/supabase/functions/ai/handler-research.test.ts +++ b/supabase/functions/ai/handler-research.test.ts @@ -385,6 +385,69 @@ function askRequest(model: string, reasoningEffort?: string): Request { }); } +Deno.test("AI handler ranks stable library sources and returns bounded assessments", async () => { + const originalFetch = globalThis.fetch; + const providerBodies: Array> = []; + globalThis.fetch = (async (input: Request | URL | string, init?: RequestInit) => { + const url = new URL(input instanceof Request ? input.url : String(input)); + const supabaseResponse = supabaseTestResponse(url); + if (supabaseResponse) return supabaseResponse; + if (url.href === "https://openrouter.ai/api/alpha/decisions") { + const body = JSON.parse(String(init?.body)) as { questions: Record }; + const answers: Record = {}; + for (const [id, question] of Object.entries(body.questions)) { + const second = id.startsWith("source_1_"); + if (question.type === "noul") { + answers[id] = { type: "noul", noul: second ? 0.95 : 0.7 }; + } else { + const criteria = question.criteria ?? ["None", "Low", "Medium", "High"]; + const score = second ? 3 : 1; + answers[id] = { + type: "score", + score, + confidence: 0.9, + probabilities: Object.fromEntries(criteria.map((_, index) => [String(index), index === score ? 0.9 : 0.03])), + legend: Object.fromEntries(criteria.map((label, index) => [String(index), label])), + }; + } + } + return jsonResponse({ model: "typesafe/jev-1.13", answers, usage: { input_tokens: 10, output_tokens: 0, cost: 0.000001 } }); + } + if (url.href === "https://openrouter.ai/api/v1/chat/completions") { + providerBodies.push(JSON.parse(String(init?.body)) as Record); + return jsonResponse({ + choices: [{ message: { content: JSON.stringify({ response: "ok", followUpQuestions: [], uncertainties: [] }) } }], + usage: { prompt_tokens: 10, completion_tokens: 5, total_tokens: 15, cost: 0.000001 }, + }); + } + throw new Error(`Unhandled request: ${url}`); + }) as typeof fetch; + + try { + const response = await handleRequest(new Request("http://127.0.0.1/functions/v1/ai", { + method: "POST", + headers: { Authorization: "Bearer test-user-token", "Content-Type": "application/json" }, + body: JSON.stringify({ + feature: "ask", + model: "openai/gpt-5.6-luna", + input: { message: "Compare the evidence", mode: "general", webSearch: false }, + sources: [ + { id: "source-a", title: "Source A", content: "Evidence A", type: "text", metadata: { qualityLabel: "strong", qualityScore: 90 } }, + { id: "source-b", title: "Source B", content: "Evidence B", type: "text", metadata: { qualityLabel: "strong", qualityScore: 80 } }, + ], + }), + })); + if (response.status !== 200) throw new Error(`source-ranking request returned ${response.status}: ${await response.text()}`); + const payload = await response.json() as { sourceAssessments?: Array<{ id?: string; importance?: number }> }; + if (payload.sourceAssessments?.map((item) => item.id).join(",") !== "source-a,source-b") throw new Error("stable source IDs were not returned"); + if (payload.sourceAssessments?.[1]?.importance !== 3) throw new Error("semantic assessment was not returned"); + const providerText = JSON.stringify(providerBodies); + if (providerText.indexOf("Source B") > providerText.indexOf("Source A")) throw new Error("semantic source order did not reach the provider prompt"); + } finally { + globalThis.fetch = originalFetch; + } +}); + Deno.test("AI handlers use strict OpenRouter routing and server-side reasoning tiers", async () => { const originalFetch = globalThis.fetch; const originalTier = (globalThis as typeof globalThis & { __testTier?: string }).__testTier; diff --git a/supabase/functions/ai/index.ts b/supabase/functions/ai/index.ts index 4fe5ee9b1..1f12d9d7a 100644 --- a/supabase/functions/ai/index.ts +++ b/supabase/functions/ai/index.ts @@ -109,6 +109,12 @@ import { } from "../_shared/model-catalog.ts"; import { resolveModelSelection } from "../_shared/auto-model-router.ts"; import { JEV_MODEL } from "../_shared/jev.ts"; +import { + identifyCaseSourceUsage, + rankSourcesForRequest, + type SourceAssessment, + type SourceDecisionCandidate, +} from "../_shared/source-decisions.ts"; import { ASK_RESEARCH, applyFactoidsVerification, @@ -1087,7 +1093,7 @@ export async function handleRequest(req: Request): Promise { return badRequest(controlError, corsHeaders); } let responseTruncated = sanitizeResult.truncated || crossfireSourcesTruncated; - const sources: PromptSource[] = feature === "crossfire" + let sources: PromptSource[] = feature === "crossfire" ? crossfireSources : Array.isArray(rawSources) ? rawSources as PromptSource[] : []; if (customSystemPrompt) { @@ -1107,12 +1113,14 @@ export async function handleRequest(req: Request): Promise { } validatedSparringRequest = sparringResult.request; } + let sourceAssessments: SourceAssessment[] = []; const successBody = (payload: Record) => { const meta = { ...(responseTruncated ? { truncated: true } : {}), ...(resolvedModelMeta ? { requestedModel, resolvedModel: resolvedModelMeta } : {}), }; - return Object.keys(meta).length > 0 ? { ...payload, meta } : payload; + const enriched = sourceAssessments.length > 0 ? { ...payload, sourceAssessments } : payload; + return Object.keys(meta).length > 0 ? { ...enriched, meta } : enriched; }; // Require a valid user session for all AI requests. @@ -1253,6 +1261,42 @@ export async function handleRequest(req: Request): Promise { return badRequest(message.includes("Pro") ? message : "Unable to route this request", corsHeaders, message.includes("Pro") ? 403 : 500); } + const sourceDecisionCandidates = Array.isArray(rawSources) + ? rawSources.slice(0, MAX_PROMPT_SOURCE_COUNT).flatMap((value, index): SourceDecisionCandidate[] => { + if (!isRecord(value) || typeof value.title !== "string" || typeof value.content !== "string" || typeof value.type !== "string") return []; + const metadata = isRecord(value.metadata) ? value.metadata : {}; + const id = typeof value.id === "string" && value.id.trim() ? value.id.trim().slice(0, 128) : undefined; + const origin = typeof value.origin === "string" && value.origin.trim() ? value.origin.trim().slice(0, 500) : undefined; + const qualityLabel = metadata.qualityLabel === "strong" || metadata.qualityLabel === "moderate" || metadata.qualityLabel === "weak" + ? metadata.qualityLabel + : undefined; + const qualityScore = typeof metadata.qualityScore === "number" && Number.isFinite(metadata.qualityScore) + ? metadata.qualityScore + : undefined; + return [{ + key: id ?? origin ?? `source-${index}`, + ...(id ? { id } : {}), + title: value.title, + content: value.content, + ...(origin ? { origin } : {}), + type: value.type, + ...(qualityLabel ? { qualityLabel } : {}), + ...(qualityScore !== undefined ? { qualityScore } : {}), + }]; + }) + : []; + if (feature !== "sparring" && feature !== "crossfire" && sourceDecisionCandidates.length > 0) { + const ranked = await rankSourcesForRequest({ + feature, + requestText: JSON.stringify(sanitizedInput), + sources: sourceDecisionCandidates, + signal: req.signal, + }); + sourceAssessments = ranked.assessments; + sources = ranked.sources.map(({ title, content, origin, type }) => ({ title, content, ...(origin ? { origin } : {}), type })); + if (ranked.jev?.usage) await logUsage(userId, "jev_source_ranking", provider, JEV_MODEL, ranked.jev.usage); + } + const reasoningEffort = effectiveReasoningEffort(model, requestedReasoningEffort ?? undefined, Boolean(rateLimit.canAccessProModels)); log(requestId, "INFO", "Resolved model controls", { model: modelEntry.id, @@ -1618,6 +1662,42 @@ export async function handleRequest(req: Request): Promise { const research = await deepResearch(motion, "", researchDeps, withResearchPreferences(DEEP_RESEARCH, { preferredSourceType: "any", })); + const discoveredSources = buildDiscoveredSources(research); + const discoveredCandidates: SourceDecisionCandidate[] = discoveredSources.map((source) => ({ + key: source.url, + title: source.title, + content: source.content, + origin: source.url, + type: source.sourceType ?? "url", + qualityScore: source.qualityScore, + qualityLabel: source.qualityLabel, + })); + const discoveredRanking = await rankSourcesForRequest({ + feature, + requestText: JSON.stringify(sanitizedInput), + sources: discoveredCandidates, + signal: req.signal, + }); + if (discoveredRanking.jev?.usage) await logUsage(userId, "jev_source_ranking", provider, JEV_MODEL, discoveredRanking.jev.usage); + const discoveredAssessmentByKey = new Map(discoveredRanking.assessments.map((assessment) => [assessment.key, assessment])); + const rankedDiscoveredSources = discoveredRanking.sources.map((candidate) => { + const source = discoveredSources.find((item) => item.url === candidate.key)!; + const assessment = discoveredAssessmentByKey.get(candidate.key); + if (!assessment) return source; + const semantic: { + relevance: number; + importance: 0 | 1 | 2 | 3; + strength: 0 | 1 | 2 | 3; + evaluatedAt: string; + useCount: number; + caseUseCount: number; + lastUsedInCase?: boolean; + lastUsedAt?: string; + } = { relevance: assessment.relevance, importance: assessment.importance, strength: assessment.strength, evaluatedAt: assessment.evaluatedAt, useCount: 1, caseUseCount: 0 }; + return { ...source, semantic }; + }); + const discoveredOrder = new Map(discoveredRanking.sources.map((source, index) => [source.key, index])); + research.readContent.sort((a, b) => (discoveredOrder.get(a.url) ?? Number.MAX_SAFE_INTEGER) - (discoveredOrder.get(b.url) ?? Number.MAX_SAFE_INTEGER)); const searchContext = buildResearchContext(research); // Build prompt with research context @@ -1656,7 +1736,41 @@ export async function handleRequest(req: Request): Promise { enforceGenerateControls(output, side, format, length, role); // Build discovered sources from research - const discoveredSources = buildDiscoveredSources(research); + const activeAssessmentByKey = new Map(sourceAssessments.map((assessment) => [assessment.key, assessment])); + const usageCandidates = [ + ...sourceDecisionCandidates.filter((source) => activeAssessmentByKey.get(source.key)?.retained).map((source) => ({ + ...source, + importance: activeAssessmentByKey.get(source.key)?.importance ?? 0, + strength: activeAssessmentByKey.get(source.key)?.strength ?? 0, + })), + ...discoveredRanking.sources.filter((source) => discoveredAssessmentByKey.get(source.key)?.retained).map((source) => ({ + ...source, + importance: discoveredAssessmentByKey.get(source.key)?.importance ?? 0, + strength: discoveredAssessmentByKey.get(source.key)?.strength ?? 0, + })), + ].slice(0, 10); + const caseUsage = await identifyCaseSourceUsage({ + caseText: JSON.stringify(output), + sources: usageCandidates, + signal: req.signal, + }); + if (caseUsage.jev?.usage) await logUsage(userId, "jev_source_usage", provider, JEV_MODEL, caseUsage.jev.usage); + const usedByKey = new Map(caseUsage.sourceUsage.map((usage) => [usage.key, usage])); + for (const assessment of sourceAssessments) { + (assessment as SourceAssessment & { usedInCase?: boolean }).usedInCase = usedByKey.get(assessment.key)?.judgment === "used"; + } + for (const source of rankedDiscoveredSources) { + const semantic = "semantic" in source ? source.semantic : undefined; + if (semantic) { + const used = usedByKey.get(source.url)?.judgment === "used"; + semantic.lastUsedInCase = used; + semantic.caseUseCount = used ? 1 : 0; + if (used) semantic.lastUsedAt = new Date().toISOString(); + } + } + if (output && typeof output === "object" && "metadata" in output && isRecord((output as { metadata?: unknown }).metadata)) { + (output as { metadata: Record }).metadata.sourceUsage = caseUsage.sourceUsage; + } console.log(`[GENERATE] Found ${discoveredSources.length} sources`); // Log usage and return with sources @@ -1666,7 +1780,7 @@ export async function handleRequest(req: Request): Promise { metadata.durationMs = latencyMs; } await logUsage(userId, feature, provider, model, mergeNormalizedUsage([aiResult.usage, ...researchUsageForLog()]), latencyMs); - return new Response(JSON.stringify(successBody({ output, discoveredSources })), { headers: { ...corsHeaders, "Content-Type": "application/json" } }); + return new Response(JSON.stringify(successBody({ output, discoveredSources: rankedDiscoveredSources, sourceAssessments })), { headers: { ...corsHeaders, "Content-Type": "application/json" } }); } // BRAINSTORM: Use research to inform argument angles From 5a6da416ad3d84b0e630b663bee7ad2c7200487b Mon Sep 17 00:00:00 2001 From: Adam Date: Mon, 21 Sep 2026 16:40:56 +0000 Subject: [PATCH 6/9] feat(crossfire): add live Jev coaching signals --- docs/architecture.md | 2 + e2e/crossfire.spec.ts | 48 ++++++- src/lib/ai/client.ts | 28 +++- src/lib/crossfire/reducer.ts | 15 ++- src/lib/crossfire/schemas.ts | 18 ++- src/locales/en.json | 6 + src/locales/es.json | 6 + src/locales/hu.json | 6 + src/pages/CrossfirePage.tsx | 34 +++++ src/test/ai/crossfire-client.test.ts | 17 +++ src/test/crossfire/reducer.test.ts | 25 ++++ src/test/crossfire/schemas.test.ts | 15 +++ src/test/crossfire/storage.test.ts | 13 ++ src/test/pages/CrossfirePage.test.tsx | 21 +++ .../_shared/crossfire-signals.test.ts | 118 +++++++++++++++++ .../functions/_shared/crossfire-signals.ts | 123 ++++++++++++++++++ .../functions/ai/handler-research.test.ts | 72 ++++++++++ supabase/functions/ai/index.ts | 14 +- 18 files changed, 569 insertions(+), 12 deletions(-) create mode 100644 supabase/functions/_shared/crossfire-signals.test.ts create mode 100644 supabase/functions/_shared/crossfire-signals.ts diff --git a/docs/architecture.md b/docs/architecture.md index e85243d85..7e657f757 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -58,6 +58,8 @@ Client-side environment selection is a developer convenience, not authorization. [`supabase/functions/ai/index.ts`](../supabase/functions/ai/index.ts) serves non-streaming structured tools and practice modes. [`supabase/functions/ai-stream/index.ts`](../supabase/functions/ai-stream/index.ts) serves Ask over server-sent events. +For a validated Crossfire next turn, the server may ask Jev to classify only the newest completed question-answer exchange. The optional directness, concession, and follow-up signal is attached by turn ID to the answer; a missing or failed Jev decision never invalidates the main Crossfire reply. Draft version 1 remains compatible because turn signals are optional. + A configured mock provider bypasses Supabase for local and test behavior. It is not authenticated backend evidence. ## Research and source import diff --git a/e2e/crossfire.spec.ts b/e2e/crossfire.spec.ts index 8faf1fbb3..c376079cb 100644 --- a/e2e/crossfire.spec.ts +++ b/e2e/crossfire.spec.ts @@ -7,12 +7,13 @@ type CrossfireRequest = { operation: 'next-turn' | 'debrief'; expectedReply?: 'question' | 'answer'; config: { mode: 'question-ai' | 'ai-questions-user'; useSourceLibrary: boolean }; - transcript: Array<{ id: string; text: string }>; + transcript: Array<{ id: string; kind: 'question' | 'answer'; text: string }>; }; sources: unknown[]; }; -async function installDeterministicBackend(page: Page, requests: CrossfireRequest[]) { +async function installDeterministicBackend(page: Page, requests: CrossfireRequest[], failFirstAnswer = false) { + let answerFailed = false; await page.route('**/*', async (route) => { const url = new URL(route.request().url()); if (url.hostname === 'localhost' || url.hostname === '127.0.0.1') { @@ -43,6 +44,11 @@ async function installDeterministicBackend(page: Page, requests: CrossfireReques const body = route.request().postDataJSON() as CrossfireRequest; requests.push(body); if (body.input.operation === 'next-turn') { + if (failFirstAnswer && !answerFailed && body.input.expectedReply === 'answer') { + answerFailed = true; + await route.fulfill({ status: 500, contentType: 'application/json', body: JSON.stringify({ error: 'Temporary Crossfire failure' }) }); + return; + } const questionCount = requests.filter((request) => request.input.operation === 'next-turn' && request.input.expectedReply === 'question' ).length; @@ -54,7 +60,18 @@ async function installDeterministicBackend(page: Page, requests: CrossfireReques ? 'What evidence connects fewer private cars to safer streets?' : 'How would you answer the strongest enforcement objection?', }; - await route.fulfill({ status: 200, contentType: 'application/json', body: JSON.stringify({ output }) }); + const answerTurnId = body.input.expectedReply === 'answer' + ? `T${body.input.transcript.length + 1}` + : body.input.transcript.at(-1)?.kind === 'answer' ? body.input.transcript.at(-1)?.id : undefined; + const signal = answerTurnId ? { + answerTurnId, + answerQuality: body.input.expectedReply === 'answer' ? 'direct' : 'partial', + answerQualityConfidence: 0.91, + concession: body.input.expectedReply === 'answer' ? 'none' : 'implicit', + concessionConfidence: 0.76, + followUpOpportunity: 0.82, + } : undefined; + await route.fulfill({ status: 200, contentType: 'application/json', body: JSON.stringify({ output, ...(signal ? { signal } : {}) }) }); return; } @@ -75,7 +92,7 @@ async function installDeterministicBackend(page: Page, requests: CrossfireReques }); } -async function prepare(page: Page, requests: CrossfireRequest[]) { +async function prepare(page: Page, requests: CrossfireRequest[], failFirstAnswer = false) { await setLocale(page, 'en'); await stubSupabaseSession(page); await page.addInitScript(() => { @@ -83,7 +100,7 @@ async function prepare(page: Page, requests: CrossfireRequest[]) { if (key.startsWith('debatelab-crossfire-draft:')) window.localStorage.removeItem(key); } }); - await installDeterministicBackend(page, requests); + await installDeterministicBackend(page, requests, failFirstAnswer); await page.goto('/app/crossfire'); await page.getByTestId('crossfire-resolution').fill('Resolved: cities should ban private cars downtown.'); await page.getByTestId('crossfire-argument').fill('Private cars create congestion and unsafe streets.'); @@ -100,9 +117,13 @@ test('Question the AI completes a grounded Crossfire debrief', async ({ page }, await page.getByTestId('crossfire-input').fill('What evidence supports the safety mechanism?'); await page.getByTestId('submit-turn').click(); await expect(page.getByText('The case relies on reduced traffic exposure and documented collision risk.')).toBeVisible(); + await expect(page.getByText('Direct answer')).toBeVisible(); + await expect(page.getByText('What evidence supports the safety mechanism?').locator('..')).not.toContainText('Direct answer'); + await expect(page.getByText('Follow-up opportunity')).toHaveCount(1); await page.getByTestId('end-crossfire').click(); await expect(page.getByTestId('crossfire-debrief')).toBeVisible(); + await expect(page.getByText('Direct answer')).toBeVisible(); await expect(page.getByText('T1', { exact: true }).first()).toBeVisible(); await expect(page.getByText('T2', { exact: true }).first()).toBeVisible(); expect(requests[0].input).toMatchObject({ operation: 'next-turn', expectedReply: 'answer', config: { mode: 'question-ai', useSourceLibrary: true } }); @@ -122,11 +143,28 @@ test('Let the AI question me alternates roles and completes a debrief', async ({ await page.getByTestId('crossfire-input').fill('Traffic exposure falls when trips shift to transit and walking.'); await page.getByTestId('submit-turn').click(); await expect(page.getByText('How would you answer the strongest enforcement objection?')).toBeVisible(); + await expect(page.getByText('Partial answer')).toBeVisible(); + await expect(page.getByText('Implicit concession')).toBeVisible(); + await expect(page.getByText('What evidence connects fewer private cars to safer streets?').locator('..')).not.toContainText('Partial answer'); await page.getByTestId('end-crossfire').click(); await expect(page.getByTestId('crossfire-debrief')).toBeVisible(); + await expect(page.getByText('Partial answer')).toBeVisible(); expect(requests[0].input).toMatchObject({ operation: 'next-turn', expectedReply: 'question', config: { mode: 'ai-questions-user', useSourceLibrary: false } }); expect(requests[1].input.transcript.at(-1)?.text).toBe('Traffic exposure falls when trips shift to transit and walking.'); expect(requests.at(-1)?.input.operation).toBe('debrief'); await page.screenshot({ path: testInfo.outputPath('crossfire-ai-questions-user-final.png'), fullPage: true }); }); + +test('retry attaches one signal to the recovered answer', async ({ page }) => { + const requests: CrossfireRequest[] = []; + await prepare(page, requests, true); + await page.getByTestId('start-crossfire').click(); + await page.getByTestId('crossfire-input').fill('What evidence supports the safety mechanism?'); + await page.getByTestId('submit-turn').click(); + await expect(page.getByTestId('retry-ai')).toBeVisible(); + await page.getByTestId('retry-ai').click(); + await expect(page.getByText('The case relies on reduced traffic exposure and documented collision risk.')).toBeVisible(); + await expect(page.getByText('Direct answer')).toHaveCount(1); + await expect(page.getByText('Follow-up opportunity')).toHaveCount(1); +}); diff --git a/src/lib/ai/client.ts b/src/lib/ai/client.ts index 597f3928e..f88c99a79 100644 --- a/src/lib/ai/client.ts +++ b/src/lib/ai/client.ts @@ -39,10 +39,12 @@ import { CrossfireAIInputSchema, CrossfireDebriefSchema, CrossfireReplySchema, + CrossfireTurnSignalSchema, collectDebriefTurnIds, type CrossfireAIInput, type CrossfireDebrief, type CrossfireReply, + type CrossfireTurnSignal, } from '@/lib/crossfire/schemas'; // Request timeout in milliseconds (200 seconds) @@ -737,7 +739,7 @@ export async function crossfire( rawInput: CrossfireAIInput, config: ProviderConfig, options: Pick = {}, -): Promise { +): Promise<(CrossfireReply & { signal?: CrossfireTurnSignal }) | CrossfireDebrief> { const input = CrossfireAIInputSchema.safeParse(rawInput); if (!input.success) { throw new AIResponseError('Crossfire request did not match the expected schema.', { @@ -755,7 +757,7 @@ export async function crossfire( timeoutMs: CROSSFIRE_REQUEST_TIMEOUT_MS, signal: options.signal, transformResponse: (rawData) => { - const data = rawData as { output: unknown }; + const data = rawData as { output: unknown; signal?: unknown }; if (input.data.operation === 'next-turn') { const reply = CrossfireReplySchema.safeParse(data.output); if (!reply.success || reply.data.type !== input.data.expectedReply) { @@ -765,7 +767,27 @@ export async function crossfire( : { issues: reply.error.issues }, }); } - return reply.data; + if (data.signal === undefined) return reply.data; + const signal = CrossfireTurnSignalSchema.safeParse(data.signal); + if (!signal.success) { + throw new AIResponseError('Crossfire signal did not match the expected schema.', { details: { issues: signal.error.issues } }); + } + const answerTurns = [ + ...input.data.transcript, + { + id: `T${input.data.transcript.length + 1}`, + sequence: input.data.transcript.length + 1, + speaker: 'ai' as const, + kind: reply.data.type, + text: reply.data.text, + createdAt: 0, + }, + ]; + const target = answerTurns.find((turn) => turn.id === signal.data.answerTurnId); + if (!target || target.kind !== 'answer') { + throw new AIResponseError('Crossfire signal referenced an unknown or non-answer turn.'); + } + return { ...reply.data, signal: signal.data }; } const debrief = CrossfireDebriefSchema.safeParse(data.output); diff --git a/src/lib/crossfire/reducer.ts b/src/lib/crossfire/reducer.ts index af7b890fb..46ec88066 100644 --- a/src/lib/crossfire/reducer.ts +++ b/src/lib/crossfire/reducer.ts @@ -4,19 +4,21 @@ import { CrossfireSetupConfigSchema, CrossfireStateSchema, CrossfireTurnSchema, + CrossfireTurnSignalSchema, type CrossfireConfig, type CrossfireDebrief, type CrossfirePrefill, type CrossfireState, type CrossfireSetupConfig, type CrossfireTurn, + type CrossfireTurnSignal, } from './schemas'; export type CrossfireAction = | { type: 'config-patched'; patch: Partial } | { type: 'session-started'; sessionId: string; now: number } | { type: 'user-turn-submitted'; turn: CrossfireTurn; expectedReply: 'question' | 'answer' } - | { type: 'ai-turn-received'; turn: CrossfireTurn } + | { type: 'ai-turn-received'; turn: CrossfireTurn; signal?: CrossfireTurnSignal } | { type: 'request-failed'; expectedReply: 'question' | 'answer'; @@ -106,8 +108,17 @@ export function crossfireReducer(state: CrossfireState, action: CrossfireAction) case 'ai-turn-received': { if (state.phase !== 'active' || state.request.status !== 'pending') return state; if (action.turn.speaker !== 'ai' || action.turn.kind !== state.request.expectedReply) return state; - const turns = appendTurn(state.turns, action.turn); + let turns = appendTurn(state.turns, action.turn); if (!turns) return state; + if (action.signal) { + const signal = CrossfireTurnSignalSchema.safeParse(action.signal); + if (!signal.success) return state; + const targetIndex = turns.findIndex((turn) => turn.id === signal.data.answerTurnId && turn.kind === 'answer'); + if (targetIndex < 0) return state; + const target = CrossfireTurnSchema.safeParse({ ...turns[targetIndex], signal: signal.data }); + if (!target.success) return state; + turns = turns.map((turn, index) => index === targetIndex ? target.data : turn); + } return { ...state, turns, request: { status: 'idle' } }; } case 'request-failed': { diff --git a/src/lib/crossfire/schemas.ts b/src/lib/crossfire/schemas.ts index 339ce90a2..fcf3bc47d 100644 --- a/src/lib/crossfire/schemas.ts +++ b/src/lib/crossfire/schemas.ts @@ -47,6 +47,16 @@ export const CrossfirePrefillSchema = z.object({ }).strict(); export type CrossfirePrefill = z.infer; +export const CrossfireTurnSignalSchema = z.object({ + answerTurnId: z.string().trim().min(1).max(40), + answerQuality: z.enum(['direct', 'partial', 'evasive']), + answerQualityConfidence: z.number().min(0).max(1), + concession: z.enum(['none', 'implicit', 'explicit']), + concessionConfidence: z.number().min(0).max(1), + followUpOpportunity: z.number().min(0).max(1), +}).strict(); +export type CrossfireTurnSignal = z.infer; + export const CrossfireTurnSchema = z.object({ id: z.string().trim().min(1).max(40), sequence: z.number().int().positive().max(MAX_CROSSFIRE_TRANSCRIPT_TURNS), @@ -54,7 +64,13 @@ export const CrossfireTurnSchema = z.object({ kind: z.enum(['question', 'answer']), text: boundedText(1, MAX_CROSSFIRE_TURN_CHARS), createdAt: z.number().int().nonnegative(), -}).strict(); + signal: CrossfireTurnSignalSchema.optional(), +}).strict().superRefine((turn, context) => { + if (!turn.signal) return; + if (turn.kind !== 'answer' || turn.signal.answerTurnId !== turn.id) { + context.addIssue({ code: 'custom', path: ['signal', 'answerTurnId'], message: 'Signal must reference this answer turn' }); + } +}); export type CrossfireTurn = z.infer; export const CrossfireTranscriptSchema = z.array(CrossfireTurnSchema) diff --git a/src/locales/en.json b/src/locales/en.json index 96424c06d..3c826a26e 100644 --- a/src/locales/en.json +++ b/src/locales/en.json @@ -1861,6 +1861,12 @@ "coachTurn": "Coach turn", "question": "Question", "answer": "Answer", + "liveSignals": "Live coaching signals", + "signalQuality": { "direct": "Direct answer", "partial": "Partial answer", "evasive": "Evasive answer" }, + "signalConcession": { "none": "No concession", "implicit": "Implicit concession", "explicit": "Explicit concession" }, + "signalFollowUp": "Follow-up opportunity", + "signalConfidence": "{{label}}, confidence {{percent}}%", + "signalProbability": "{{label}}, probability {{percent}}%", "questionLabel": "Your question", "answerLabel": "Your answer", "questionPlaceholder": "Ask a focused question...", diff --git a/src/locales/es.json b/src/locales/es.json index a4677a72e..edb11107a 100644 --- a/src/locales/es.json +++ b/src/locales/es.json @@ -1861,6 +1861,12 @@ "coachTurn": "Turno del coach", "question": "Pregunta", "answer": "Respuesta", + "liveSignals": "Señales de entrenamiento en vivo", + "signalQuality": { "direct": "Respuesta directa", "partial": "Respuesta parcial", "evasive": "Respuesta evasiva" }, + "signalConcession": { "none": "Sin concesión", "implicit": "Concesión implícita", "explicit": "Concesión explícita" }, + "signalFollowUp": "Oportunidad de seguimiento", + "signalConfidence": "{{label}}, confianza {{percent}}%", + "signalProbability": "{{label}}, probabilidad {{percent}}%", "questionLabel": "Tu pregunta", "answerLabel": "Tu respuesta", "questionPlaceholder": "Haz una pregunta precisa...", diff --git a/src/locales/hu.json b/src/locales/hu.json index d41255f52..0e6cd0147 100644 --- a/src/locales/hu.json +++ b/src/locales/hu.json @@ -1861,6 +1861,12 @@ "coachTurn": "Az edző jön", "question": "Kérdés", "answer": "Válasz", + "liveSignals": "Élő edzői jelzések", + "signalQuality": { "direct": "Közvetlen válasz", "partial": "Részleges válasz", "evasive": "Kitérő válasz" }, + "signalConcession": { "none": "Nincs engedmény", "implicit": "Burkolt engedmény", "explicit": "Kifejezett engedmény" }, + "signalFollowUp": "Visszakérdezési lehetőség", + "signalConfidence": "{{label}}, bizonyosság {{percent}}%", + "signalProbability": "{{label}}, valószínűség {{percent}}%", "questionLabel": "A kérdésed", "answerLabel": "A válaszod", "questionPlaceholder": "Tegyél fel egy célzott kérdést...", diff --git a/src/pages/CrossfirePage.tsx b/src/pages/CrossfirePage.tsx index d88173d5f..5e94a03ed 100644 --- a/src/pages/CrossfirePage.tsx +++ b/src/pages/CrossfirePage.tsx @@ -319,6 +319,7 @@ export default function CrossfirePage() { dispatch({ type: 'ai-turn-received', turn: nextTurn(snapshot.turns.length + 1, 'ai', result.type, result.text), + signal: result.signal, }); } catch (error) { if (controller.signal.aborted) return; @@ -814,6 +815,39 @@ function Transcript({ turns, reduceMotion }: { turns: CrossfireTurn[]; reduceMot {turn.kind === 'question' ? t('crossfire.question') : t('crossfire.answer')}

{turn.text}

+ {turn.kind === 'answer' && turn.signal ? ( +
+ + {t(`crossfire.signalQuality.${turn.signal.answerQuality}`)} + + + {t(`crossfire.signalConcession.${turn.signal.concession}`)} + + {turn.signal.followUpOpportunity > 0.70 ? ( + + {t('crossfire.signalFollowUp')} + + ) : null} +
+ ) : null} ))} diff --git a/src/test/ai/crossfire-client.test.ts b/src/test/ai/crossfire-client.test.ts index c711593ec..97a535b74 100644 --- a/src/test/ai/crossfire-client.test.ts +++ b/src/test/ai/crossfire-client.test.ts @@ -148,6 +148,23 @@ describe('Crossfire AI client', () => { await expect(crossfire(nextTurnInput(), config)).rejects.toBeInstanceOf(AIResponseError); }); + it('parses a valid live signal and rejects unknown or non-answer targets', async () => { + const signal = { answerTurnId: 'T2', answerQuality: 'direct', answerQualityConfidence: 0.9, concession: 'none', concessionConfidence: 0.8, followUpOpportunity: 0.75 }; + mocks.invoke.mockResolvedValueOnce({ data: { output: { type: 'answer', text: 'Answer.' }, signal }, error: null }); + await expect(crossfire(nextTurnInput(), config)).resolves.toEqual({ type: 'answer', text: 'Answer.', signal }); + + mocks.invoke.mockResolvedValueOnce({ data: { output: { type: 'answer', text: 'Answer.' }, signal: { ...signal, answerTurnId: 'T99' } }, error: null }); + await expect(crossfire(nextTurnInput(), config)).rejects.toBeInstanceOf(AIResponseError); + + mocks.invoke.mockResolvedValueOnce({ data: { output: { type: 'question', text: 'Next?' }, signal: { ...signal, answerTurnId: 'T1' } }, error: null }); + const aiQuestionInput = nextTurnInput(); + if (aiQuestionInput.operation !== 'next-turn') throw new Error('Unexpected fixture'); + aiQuestionInput.config.mode = 'ai-questions-user'; + aiQuestionInput.transcript = []; + aiQuestionInput.expectedReply = 'question'; + await expect(crossfire(aiQuestionInput, config)).rejects.toBeInstanceOf(AIResponseError); + }); + it('validates debrief shape and every transcript reference', async () => { mocks.invoke.mockResolvedValueOnce({ data: { output: validDebrief }, error: null }); await expect(crossfire(debriefInput(), config)).resolves.toEqual(validDebrief); diff --git a/src/test/crossfire/reducer.test.ts b/src/test/crossfire/reducer.test.ts index c38959a97..0dc6688b7 100644 --- a/src/test/crossfire/reducer.test.ts +++ b/src/test/crossfire/reducer.test.ts @@ -70,6 +70,31 @@ describe('crossfireReducer', () => { expect(canEndCrossfireSession(state)).toBe(true); }); + it('attaches a live signal only to its answer and replaces it on retry', () => { + let state = crossfireReducer(configured(), { type: 'session-started', sessionId: 'signal-session', now: 1 }); + state = crossfireReducer(state, { type: 'user-turn-submitted', turn: turn(1, 'user', 'question'), expectedReply: 'answer' }); + const signal = { answerTurnId: 'T2', answerQuality: 'direct' as const, answerQualityConfidence: 0.9, concession: 'none' as const, concessionConfidence: 0.8, followUpOpportunity: 0.75 }; + state = crossfireReducer(state, { type: 'ai-turn-received', turn: turn(2, 'ai', 'answer'), signal }); + expect(state.phase === 'active' && state.turns[0].signal).toBeUndefined(); + expect(state.phase === 'active' && state.turns[1].signal).toEqual(signal); + + const invalidTarget = crossfireReducer(state, { type: 'state-restored', state: { + ...state, + turns: state.phase === 'active' ? state.turns.map((item) => item.id === 'T2' ? { ...item, signal: { ...signal, answerTurnId: 'T9' } } : item) : [], + } as CrossfireState }); + expect(invalidTarget).toEqual(createCrossfireSetupState()); + }); + + it('attaches a signal to the previous user answer when the AI returns the next question', () => { + let state = crossfireReducer(configured('ai-questions-user'), { type: 'session-started', sessionId: 'signal-session-2', now: 1 }); + state = crossfireReducer(state, { type: 'ai-turn-received', turn: turn(1, 'ai', 'question') }); + state = crossfireReducer(state, { type: 'user-turn-submitted', turn: turn(2, 'user', 'answer'), expectedReply: 'question' }); + const signal = { answerTurnId: 'T2', answerQuality: 'partial' as const, answerQualityConfidence: 0.8, concession: 'implicit' as const, concessionConfidence: 0.7, followUpOpportunity: 0.9 }; + state = crossfireReducer(state, { type: 'ai-turn-received', turn: turn(3, 'ai', 'question'), signal }); + expect(state.phase === 'active' && state.turns[1].signal).toEqual(signal); + expect(state.phase === 'active' && state.turns[2].signal).toBeUndefined(); + }); + it('models Let the AI question me and requests a new question after the user answer', () => { let state = crossfireReducer(configured('ai-questions-user'), { type: 'session-started', diff --git a/src/test/crossfire/schemas.test.ts b/src/test/crossfire/schemas.test.ts index a963e6d47..9ea24a2f7 100644 --- a/src/test/crossfire/schemas.test.ts +++ b/src/test/crossfire/schemas.test.ts @@ -4,6 +4,7 @@ import { CrossfireDebriefSchema, CrossfireReplySchema, CrossfireStateSchema, + CrossfireTurnSchema, MAX_CROSSFIRE_ARGUMENT_CHARS, MAX_CROSSFIRE_REPLY_CHARS, MAX_CROSSFIRE_TRANSCRIPT_CHARS, @@ -95,6 +96,20 @@ describe('Crossfire schemas', () => { expect(CrossfireReplySchema.safeParse({ type: 'answer', text: 'Valid', extra: true }).success).toBe(false); }); + it('accepts signals only on their matching answer turn', () => { + const signal = { + answerTurnId: 'T2', + answerQuality: 'direct', + answerQualityConfidence: 0.9, + concession: 'implicit', + concessionConfidence: 0.7, + followUpOpportunity: 0.8, + } as const; + expect(CrossfireTurnSchema.safeParse({ ...turn, id: 'T2', sequence: 2, speaker: 'ai', kind: 'answer', signal }).success).toBe(true); + expect(CrossfireTurnSchema.safeParse({ ...turn, signal: { ...signal, answerTurnId: 'T2' } }).success).toBe(false); + expect(CrossfireTurnSchema.safeParse({ ...turn, id: 'T2', sequence: 2, kind: 'answer', signal: { ...signal, answerTurnId: 'T9' } }).success).toBe(false); + }); + it('requires every debrief claim and suggestion to cite transcript turns', () => { const valid: CrossfireDebrief = { concessionsSecured: [{ turnIds: ['T2'], claim: 'The answer concedes implementation has a cost.' }], diff --git a/src/test/crossfire/storage.test.ts b/src/test/crossfire/storage.test.ts index 015e1084b..9f2d5896f 100644 --- a/src/test/crossfire/storage.test.ts +++ b/src/test/crossfire/storage.test.ts @@ -25,6 +25,19 @@ describe('Crossfire draft storage', () => { expect(loadCrossfireDraft('user-a', storage)).toEqual(state); }); + it('round-trips version-one drafts containing optional answer signals', () => { + const storage = new MemoryStorage(); + let state = crossfireReducer(createCrossfireSetupState({ resolution: 'Resolved: test', argument: 'Case text' }), { type: 'session-started', sessionId: 's', now: 1 }); + state = crossfireReducer(state, { type: 'user-turn-submitted', turn: { id: 'T1', sequence: 1, speaker: 'user', kind: 'question', text: 'Why?', createdAt: 1 }, expectedReply: 'answer' }); + state = crossfireReducer(state, { + type: 'ai-turn-received', + turn: { id: 'T2', sequence: 2, speaker: 'ai', kind: 'answer', text: 'Because.', createdAt: 2 }, + signal: { answerTurnId: 'T2', answerQuality: 'direct', answerQualityConfidence: 0.9, concession: 'none', concessionConfidence: 0.8, followUpOpportunity: 0.75 }, + }); + expect(saveCrossfireDraft('user-a', state, storage, 10)).toBe(true); + expect(loadCrossfireDraft('user-a', storage)).toEqual(state); + }); + it('clears malformed, oversized, or differently owned data', () => { const malformed = new MemoryStorage(); malformed.setItem(crossfireStorageKey('user-a'), '{broken'); diff --git a/src/test/pages/CrossfirePage.test.tsx b/src/test/pages/CrossfirePage.test.tsx index 0bf346484..a01e5a279 100644 --- a/src/test/pages/CrossfirePage.test.tsx +++ b/src/test/pages/CrossfirePage.test.tsx @@ -132,4 +132,25 @@ describe('CrossfirePage', () => { expect(await screen.findByTestId('retry-debrief')).toBeInTheDocument(); expect(screen.getByText('The coach returned an incomplete debrief. Try again to continue.')).toBeInTheDocument(); }); + + it('shows live coaching on the answer only', async () => { + mocks.crossfire.mockResolvedValueOnce({ + type: 'answer', + text: 'A bounded answer.', + signal: { answerTurnId: 'T2', answerQuality: 'direct', answerQualityConfidence: 0.91, concession: 'implicit', concessionConfidence: 0.74, followUpOpportunity: 0.82 }, + }); + renderPage(); + fireEvent.change(screen.getByTestId('crossfire-resolution'), { target: { value: 'Resolved: test' } }); + fireEvent.change(screen.getByTestId('crossfire-argument'), { target: { value: 'Case text' } }); + fireEvent.click(screen.getByTestId('start-crossfire')); + fireEvent.change(screen.getByTestId('crossfire-input'), { target: { value: 'What is the mechanism?' } }); + fireEvent.click(screen.getByTestId('submit-turn')); + + const answer = await screen.findByText('A bounded answer.'); + expect(answer.closest('article')).toHaveTextContent('Direct answer'); + expect(answer.closest('article')).toHaveTextContent('Implicit concession'); + expect(answer.closest('article')).toHaveTextContent('Follow-up opportunity'); + expect(screen.getByText('What is the mechanism?').closest('article')).not.toHaveTextContent('Direct answer'); + expect(answer.closest('article')?.querySelector('[aria-label*="82%"]')).not.toBeNull(); + }); }); diff --git a/supabase/functions/_shared/crossfire-signals.test.ts b/supabase/functions/_shared/crossfire-signals.test.ts new file mode 100644 index 000000000..65498ce5f --- /dev/null +++ b/supabase/functions/_shared/crossfire-signals.test.ts @@ -0,0 +1,118 @@ +import { + evaluateCrossfireExchange, + type CrossfireSignalJev, + type CrossfireSignalTurn, +} from "./crossfire-signals.ts"; +import type { CallJevArgs, JevDecisionResult } from "./jev.ts"; + +const turn = (sequence: number, speaker: "user" | "ai", kind: "question" | "answer", text: string): CrossfireSignalTurn => ({ + id: `T${sequence}`, + sequence, + speaker, + kind, + text, +}); + +function signalResult(answerQuality = "direct", concession = "implicit"): JevDecisionResult { + return { + model: "typesafe/jev-1.13", + answers: { + answer_quality: { type: "choice", choice: answerQuality, confidence: 0.88, probabilities: { direct: 0.88, partial: 0.1, evasive: 0.02 } }, + concession: { type: "choice", choice: concession, confidence: 0.76, probabilities: { none: 0.2, implicit: 0.76, explicit: 0.04 } }, + follow_up: { type: "noul", noul: 0.82 }, + }, + }; +} + +Deno.test("Crossfire signal evaluates an AI answer added after a user question", async () => { + const result = await evaluateCrossfireExchange({ + transcript: [turn(1, "user", "question", "What evidence supports that?")], + reply: { type: "answer", text: "A national survey supports it." }, + callJevImpl: () => Promise.resolve(signalResult("direct", "none")), + }); + if (result.signal?.answerTurnId !== "T2" || result.signal.answerQuality !== "direct") throw new Error("AI answer signal was not attached to T2"); +}); + +Deno.test("Crossfire signal evaluates the user answer before the next AI question", async () => { + let state = ""; + const callJevImpl: CrossfireSignalJev = (args) => { + state = JSON.stringify(args.state); + return Promise.resolve(signalResult("partial", "implicit")); + }; + const result = await evaluateCrossfireExchange({ + transcript: [ + turn(1, "ai", "question", "Would costs rise?"), + turn(2, "user", "answer", "Some costs may rise."), + ], + reply: { type: "question", text: "How much would they rise?" }, + callJevImpl, + }); + if (result.signal?.answerTurnId !== "T2" || result.signal.concession !== "implicit") throw new Error("user answer signal was not attached to T2"); + if (state.includes("How much would they rise?")) throw new Error("next question leaked into the completed exchange state"); +}); + +Deno.test("Crossfire signal sends only the newest completed exchange", async () => { + let observed: CallJevArgs["state"] | undefined; + await evaluateCrossfireExchange({ + transcript: [ + turn(1, "user", "question", "Old question"), + turn(2, "ai", "answer", "Old answer"), + turn(3, "user", "question", "Newest question"), + ], + reply: { type: "answer", text: "Newest answer" }, + callJevImpl: (args) => { + observed = args.state; + return Promise.resolve(signalResult()); + }, + }); + const serialized = JSON.stringify(observed); + if (serialized.includes("Old question") || !serialized.includes("Newest answer")) throw new Error("signal state was not limited to newest exchange"); +}); + +Deno.test("Crossfire signal skips missing exchanges and invalid or failed decisions", async () => { + let calls = 0; + const noPair = await evaluateCrossfireExchange({ + transcript: [], + reply: { type: "question", text: "Opening question" }, + callJevImpl: () => { + calls += 1; + return Promise.resolve(signalResult()); + }, + }); + if (noPair.signal || calls !== 0) throw new Error("Jev ran without a completed exchange"); + + const malformed = await evaluateCrossfireExchange({ + transcript: [turn(1, "user", "question", "Question")], + reply: { type: "answer", text: "Answer" }, + callJevImpl: () => Promise.resolve({ model: "typesafe/jev-1.13", answers: {} }), + }); + if (malformed.signal) throw new Error("malformed decision produced a signal"); + + const failed = await evaluateCrossfireExchange({ + transcript: [turn(1, "user", "question", "Question")], + reply: { type: "answer", text: "Answer" }, + callJevImpl: () => Promise.reject(new Error("Jev 500")), + }); + if (failed.signal) throw new Error("failed decision discarded the optional-signal contract"); +}); + +Deno.test("Crossfire signal does not start work after caller cancellation", async () => { + const controller = new AbortController(); + controller.abort(new DOMException("Cancelled", "AbortError")); + let calls = 0; + try { + await evaluateCrossfireExchange({ + transcript: [turn(1, "user", "question", "Question")], + reply: { type: "answer", text: "Answer" }, + signal: controller.signal, + callJevImpl: () => { + calls += 1; + return Promise.resolve(signalResult()); + }, + }); + } catch (error) { + if (!(error instanceof Error) || error.name !== "AbortError" || calls !== 0) throw error; + return; + } + throw new Error("pre-aborted signal was ignored"); +}); diff --git a/supabase/functions/_shared/crossfire-signals.ts b/supabase/functions/_shared/crossfire-signals.ts new file mode 100644 index 000000000..4f7e2ff8d --- /dev/null +++ b/supabase/functions/_shared/crossfire-signals.ts @@ -0,0 +1,123 @@ +import { + callJev, + type CallJevArgs, + type JevChoiceAnswer, + type JevDecisionResult, + type JevNoulAnswer, +} from "./jev.ts"; + +export interface CrossfireSignalTurn { + id: string; + sequence: number; + speaker: "user" | "ai"; + kind: "question" | "answer"; + text: string; +} + +export interface CrossfireTurnSignal { + answerTurnId: string; + answerQuality: "direct" | "partial" | "evasive"; + answerQualityConfidence: number; + concession: "none" | "implicit" | "explicit"; + concessionConfidence: number; + followUpOpportunity: number; +} + +export type CrossfireSignalJev = (args: CallJevArgs) => Promise; + +function isAbort(error: unknown, signal?: AbortSignal): boolean { + return signal?.aborted === true || error instanceof DOMException && error.name === "AbortError"; +} + +function choice(value: unknown, labels: readonly string[]): JevChoiceAnswer | null { + if (!value || typeof value !== "object" || (value as { type?: unknown }).type !== "choice") return null; + const answer = value as JevChoiceAnswer; + return labels.includes(answer.choice) && Number.isFinite(answer.confidence) && answer.confidence >= 0 && answer.confidence <= 1 + ? answer + : null; +} + +function noul(value: unknown): JevNoulAnswer | null { + if (!value || typeof value !== "object" || (value as { type?: unknown }).type !== "noul") return null; + const answer = value as JevNoulAnswer; + return Number.isFinite(answer.noul) && answer.noul >= 0 && answer.noul <= 1 ? answer : null; +} + +function newestCompletedExchange(transcript: CrossfireSignalTurn[], reply: { type: "question" | "answer"; text: string }) { + const turns = reply.type === "answer" + ? [...transcript, { + id: `T${transcript.length + 1}`, + sequence: transcript.length + 1, + speaker: "ai" as const, + kind: "answer" as const, + text: reply.text, + }] + : transcript; + for (let index = turns.length - 1; index >= 1; index -= 1) { + const answer = turns[index]; + const question = turns[index - 1]; + if (answer.kind === "answer" && question.kind === "question") return { question, answer }; + } + return null; +} + +export async function evaluateCrossfireExchange({ + transcript, + reply, + signal, + callJevImpl = callJev, +}: { + transcript: CrossfireSignalTurn[]; + reply: { type: "question" | "answer"; text: string }; + signal?: AbortSignal; + callJevImpl?: CrossfireSignalJev; +}): Promise<{ signal?: CrossfireTurnSignal; jev?: JevDecisionResult }> { + if (signal?.aborted) throw signal.reason ?? new DOMException("The operation was aborted", "AbortError"); + const exchange = newestCompletedExchange(transcript, reply); + if (!exchange) return {}; + try { + const jev = await callJevImpl({ + state: { + question: { id: exchange.question.id, speaker: exchange.question.speaker, text: exchange.question.text.slice(0, 2_000) }, + answer: { id: exchange.answer.id, speaker: exchange.answer.speaker, text: exchange.answer.text.slice(0, 2_000) }, + }, + questions: { + answer_quality: { + type: "choice", + instructions: "How directly and completely does the answer address the question?", + criteria: { direct: "Direct and complete", partial: "Addresses only part", evasive: "Avoids the question" }, + }, + concession: { + type: "choice", + instructions: "What level of concession does the answer make to the questioner's challenged claim?", + criteria: { none: "No concession", implicit: "Implicit concession", explicit: "Explicit concession" }, + }, + follow_up: { + type: "noul", + instructions: "Does this exchange create a useful immediate follow-up opportunity for the questioner?", + criteria: { true: "Useful follow-up exists", false: "No useful follow-up" }, + }, + }, + signal, + }); + if (signal?.aborted) throw signal.reason ?? new DOMException("The operation was aborted", "AbortError"); + const quality = choice(jev.answers.answer_quality, ["direct", "partial", "evasive"]); + const concession = choice(jev.answers.concession, ["none", "implicit", "explicit"]); + const followUp = noul(jev.answers.follow_up); + if (!quality || !concession || !followUp) return {}; + return { + signal: { + answerTurnId: exchange.answer.id, + answerQuality: quality.choice as CrossfireTurnSignal["answerQuality"], + answerQualityConfidence: quality.confidence, + concession: concession.choice as CrossfireTurnSignal["concession"], + concessionConfidence: concession.confidence, + followUpOpportunity: followUp.noul, + }, + jev, + }; + } catch (error) { + if (isAbort(error, signal)) throw error; + return {}; + } +} diff --git a/supabase/functions/ai/handler-research.test.ts b/supabase/functions/ai/handler-research.test.ts index 109542c3c..7d0dbda5a 100644 --- a/supabase/functions/ai/handler-research.test.ts +++ b/supabase/functions/ai/handler-research.test.ts @@ -448,6 +448,78 @@ Deno.test("AI handler ranks stable library sources and returns bounded assessmen } }); +Deno.test("Crossfire returns newest-exchange signals without making them a reply dependency", async () => { + const originalFetch = globalThis.fetch; + let decisionFails = false; + globalThis.fetch = (async (input: Request | URL | string, init?: RequestInit) => { + const url = new URL(input instanceof Request ? input.url : String(input)); + const supabaseResponse = supabaseTestResponse(url); + if (supabaseResponse) return supabaseResponse; + if (url.href === "https://openrouter.ai/api/v1/chat/completions") { + const body = JSON.parse(String(init?.body)) as { messages?: Array<{ content?: string }> }; + const prompt = body.messages?.at(-1)?.content ?? ""; + const output = prompt.includes('must be type "question"') + ? { type: "question", text: "What follows from that concession?" } + : { type: "answer", text: "The evidence directly supports the claim." }; + return jsonResponse({ choices: [{ message: { content: JSON.stringify(output) } }], usage: { prompt_tokens: 10, completion_tokens: 5, cost: 0.000001 } }); + } + if (url.href === "https://openrouter.ai/api/alpha/decisions") { + if (decisionFails) return jsonResponse({ error: "unavailable" }, 500); + return jsonResponse({ + model: "typesafe/jev-1.13", + answers: { + answer_quality: { type: "choice", choice: "direct", confidence: 0.9, probabilities: { direct: 0.9, partial: 0.08, evasive: 0.02 } }, + concession: { type: "choice", choice: "implicit", confidence: 0.8, probabilities: { none: 0.15, implicit: 0.8, explicit: 0.05 } }, + follow_up: { type: "noul", noul: 0.82 }, + }, + usage: { input_tokens: 10, output_tokens: 0, cost: 0.000001 }, + }); + } + throw new Error(`Unhandled request: ${url}`); + }) as typeof fetch; + + const request = (mode: "question-ai" | "ai-questions-user", transcript: unknown[], expectedReply: "question" | "answer") => handleRequest(new Request("http://127.0.0.1/functions/v1/ai", { + method: "POST", + headers: { Authorization: "Bearer test-user-token", "Content-Type": "application/json" }, + body: JSON.stringify({ + feature: "crossfire", + model: "openai/gpt-5.6-luna", + input: { + operation: "next-turn", + config: { resolution: "Cities should ban cars.", argument: "Cars cause congestion.", mode, style: "cooperative", durationSeconds: 180, useSourceLibrary: false, seedQuestions: [] }, + transcript, + expectedReply, + }, + }), + })); + + try { + const questionAi = await request("question-ai", [{ id: "T1", sequence: 1, speaker: "user", kind: "question", text: "What evidence supports that?", createdAt: 1 }], "answer"); + const questionAiPayload = await questionAi.json() as { signal?: { answerTurnId?: string; answerQuality?: string } }; + if (questionAi.status !== 200 || questionAiPayload.signal?.answerTurnId !== "T2" || questionAiPayload.signal.answerQuality !== "direct") { + throw new Error(`question-ai signal missing: ${JSON.stringify(questionAiPayload)}`); + } + + const aiQuestionsUser = await request("ai-questions-user", [ + { id: "T1", sequence: 1, speaker: "ai", kind: "question", text: "Would costs rise?", createdAt: 1 }, + { id: "T2", sequence: 2, speaker: "user", kind: "answer", text: "Some costs may rise.", createdAt: 2 }, + ], "question"); + const aiQuestionsPayload = await aiQuestionsUser.json() as { signal?: { answerTurnId?: string; concession?: string } }; + if (aiQuestionsUser.status !== 200 || aiQuestionsPayload.signal?.answerTurnId !== "T2" || aiQuestionsPayload.signal.concession !== "implicit") { + throw new Error(`ai-questions-user signal missing: ${JSON.stringify(aiQuestionsPayload)}`); + } + + decisionFails = true; + const fallback = await request("question-ai", [{ id: "T1", sequence: 1, speaker: "user", kind: "question", text: "What evidence supports that?", createdAt: 1 }], "answer"); + const fallbackPayload = await fallback.json() as { output?: { type?: string }; signal?: unknown }; + if (fallback.status !== 200 || fallbackPayload.output?.type !== "answer" || fallbackPayload.signal !== undefined) { + throw new Error(`failed optional signal discarded the reply: ${JSON.stringify(fallbackPayload)}`); + } + } finally { + globalThis.fetch = originalFetch; + } +}); + Deno.test("AI handlers use strict OpenRouter routing and server-side reasoning tiers", async () => { const originalFetch = globalThis.fetch; const originalTier = (globalThis as typeof globalThis & { __testTier?: string }).__testTier; diff --git a/supabase/functions/ai/index.ts b/supabase/functions/ai/index.ts index 1f12d9d7a..4b5232379 100644 --- a/supabase/functions/ai/index.ts +++ b/supabase/functions/ai/index.ts @@ -115,6 +115,7 @@ import { type SourceAssessment, type SourceDecisionCandidate, } from "../_shared/source-decisions.ts"; +import { evaluateCrossfireExchange } from "../_shared/crossfire-signals.ts"; import { ASK_RESEARCH, applyFactoidsVerification, @@ -1451,9 +1452,20 @@ export async function handleRequest(req: Request): Promise { : validateCrossfireDebrief(cleanedOutput, crossfireInput.transcript); if ("error" in output) throw new Error(output.error); + let signal; + if (crossfireInput.operation === "next-turn") { + const evaluated = await evaluateCrossfireExchange({ + transcript: crossfireInput.transcript, + reply: output.value as { type: "question" | "answer"; text: string }, + signal: req.signal, + }); + signal = evaluated.signal; + if (evaluated.jev?.usage) await logUsage(userId, "jev_crossfire_signal", provider, JEV_MODEL, evaluated.jev.usage); + } + const latencyMs = Date.now() - startTime; await logUsage(userId, feature, provider, model, aiResult.usage, latencyMs); - return new Response(JSON.stringify(successBody({ output: output.value })), { headers: { ...corsHeaders, "Content-Type": "application/json" } }); + return new Response(JSON.stringify(successBody({ output: output.value, ...(signal ? { signal } : {}) })), { headers: { ...corsHeaders, "Content-Type": "application/json" } }); } // FACTOIDS: Use deep research with multiple searches From 6dad7ec37432bdd0a7284b7cca7c2b5225dd8a15 Mon Sep 17 00:00:00 2001 From: Adam Date: Mon, 21 Sep 2026 17:00:20 +0000 Subject: [PATCH 7/9] test(ai): add Jev integration verification --- docs/testing.md | 8 +++ package.json | 1 + scripts/smoke-jev.d.mts | 12 ++++ scripts/smoke-jev.mjs | 56 +++++++++++++++++++ src/test/scripts/smoke-jev.test.ts | 41 ++++++++++++++ src/test/security-hardening.test.ts | 3 +- supabase/README.md | 3 + .../_shared/auto-model-router.test.ts | 2 +- .../_shared/crossfire-signals.test.ts | 2 +- 9 files changed, 125 insertions(+), 3 deletions(-) create mode 100644 scripts/smoke-jev.d.mts create mode 100644 scripts/smoke-jev.mjs create mode 100644 src/test/scripts/smoke-jev.test.ts diff --git a/docs/testing.md b/docs/testing.md index 5a3ab7df3..3bdf38aa2 100644 --- a/docs/testing.md +++ b/docs/testing.md @@ -66,6 +66,14 @@ This uses Deno directly for `supabase/functions/`, including tests that import d A browser-oriented Vitest pass does not replace native Deno tests. The stream accounting handler tests use synthetic OpenRouter responses and inspect audit inserts for complete, interrupted, and truncated streams. The permission list includes the OpenRouter key name because the handler reads it, but these tests require no real provider credentials. +### Live Jev contract smoke + +```bash +OPENROUTER_API_KEY=... npm run smoke:jev +``` + +This sends one tiny typed choice to OpenRouter's Decisions endpoint using the pinned `typesafe/jev-1.13` model. It prints only the returned model, answer labels, latency, and token counts; it never prints the key or submitted state. The command makes a negligible billable live request and is separate from the credential-free `verify` gate. + ## Production build ```bash diff --git a/package.json b/package.json index f2967e178..52097de90 100644 --- a/package.json +++ b/package.json @@ -16,6 +16,7 @@ "lint": "eslint .", "typecheck": "tsc --noEmit -p tsconfig.app.json", "preview": "vite preview", + "smoke:jev": "node scripts/smoke-jev.mjs", "test": "vitest run", "test:collaboration": "vitest run --config services/collaboration/vitest.config.ts", "test:watch": "vitest", diff --git a/scripts/smoke-jev.d.mts b/scripts/smoke-jev.d.mts new file mode 100644 index 000000000..a78a5a1a9 --- /dev/null +++ b/scripts/smoke-jev.d.mts @@ -0,0 +1,12 @@ +export interface JevSmokeSummary { + model: string; + answers: { fit: 'specific' | 'vague' }; + latencyMs: number; + usage: { inputTokens: number | null; outputTokens: number | null }; +} + +export function runJevSmoke(options?: { + apiKey?: string; + fetchImpl?: typeof fetch; + now?: () => number; +}): Promise; diff --git a/scripts/smoke-jev.mjs b/scripts/smoke-jev.mjs new file mode 100644 index 000000000..1a91137dd --- /dev/null +++ b/scripts/smoke-jev.mjs @@ -0,0 +1,56 @@ +import { pathToFileURL } from 'node:url'; + +const DECISIONS_URL = 'https://openrouter.ai/api/alpha/decisions'; +const JEV_MODEL = 'typesafe/jev-1.13'; + +export async function runJevSmoke({ + apiKey = process.env.OPENROUTER_API_KEY, + fetchImpl = globalThis.fetch, + now = () => Date.now(), +} = {}) { + if (!apiKey) throw new Error('OPENROUTER_API_KEY missing'); + const startedAt = now(); + const response = await fetchImpl(DECISIONS_URL, { + method: 'POST', + headers: { Authorization: `Bearer ${apiKey}`, 'Content-Type': 'application/json' }, + body: JSON.stringify({ + model: JEV_MODEL, + state: { text: 'DebateLab smoke test: label this request.' }, + questions: { + fit: { + type: 'choice', + instructions: 'Is this request specific or vague?', + criteria: { specific: 'The request is concrete', vague: 'The request is ambiguous' }, + }, + }, + }), + signal: AbortSignal.timeout(15_000), + }); + if (!response.ok) throw new Error(`OpenRouter Jev smoke failed with ${response.status}`); + const payload = await response.json(); + const answer = payload?.answers?.fit; + if (typeof payload?.model !== 'string' || !payload.model.startsWith(JEV_MODEL) || answer?.type !== 'choice' || + (answer.choice !== 'specific' && answer.choice !== 'vague')) { + throw new Error('OpenRouter Jev smoke returned a malformed response'); + } + const inputTokens = payload.usage?.input_tokens ?? payload.usage?.prompt_tokens; + const outputTokens = payload.usage?.output_tokens ?? payload.usage?.completion_tokens; + return { + model: payload.model, + answers: { fit: answer.choice }, + latencyMs: Math.max(0, now() - startedAt), + usage: { + inputTokens: Number.isFinite(inputTokens) ? inputTokens : null, + outputTokens: Number.isFinite(outputTokens) ? outputTokens : null, + }, + }; +} + +if (process.argv[1] && import.meta.url === pathToFileURL(process.argv[1]).href) { + runJevSmoke() + .then((summary) => console.log(JSON.stringify(summary))) + .catch((error) => { + console.error(error instanceof Error ? error.message : 'Jev smoke failed'); + process.exitCode = 1; + }); +} diff --git a/src/test/scripts/smoke-jev.test.ts b/src/test/scripts/smoke-jev.test.ts new file mode 100644 index 000000000..f741e830a --- /dev/null +++ b/src/test/scripts/smoke-jev.test.ts @@ -0,0 +1,41 @@ +import { describe, expect, it, vi } from 'vitest'; +import { runJevSmoke } from '../../../scripts/smoke-jev.mjs'; + +describe('Jev smoke command', () => { + it('sends the pinned model and returns only a sanitized summary', async () => { + let requestBody: Record | undefined; + const fetchImpl = vi.fn(async (_url: string | URL | Request, init?: RequestInit) => { + requestBody = JSON.parse(String(init?.body)); + return new Response(JSON.stringify({ + model: 'typesafe/jev-1.13', + answers: { + fit: { type: 'choice', choice: 'specific', confidence: 0.91, probabilities: { specific: 0.91, vague: 0.09 } }, + }, + usage: { input_tokens: 12, output_tokens: 1 }, + }), { status: 200, headers: { 'Content-Type': 'application/json' } }); + }); + + const summary = await runJevSmoke({ apiKey: 'secret-key', fetchImpl, now: () => 100 }); + expect(requestBody).toMatchObject({ model: 'typesafe/jev-1.13' }); + expect(summary).toEqual({ model: 'typesafe/jev-1.13', answers: { fit: 'specific' }, latencyMs: 0, usage: { inputTokens: 12, outputTokens: 1 } }); + expect(JSON.stringify(summary)).not.toContain('secret-key'); + expect(JSON.stringify(summary)).not.toContain('DebateLab smoke test'); + }); + + it('rejects a missing key before network access', async () => { + const fetchImpl = vi.fn(); + await expect(runJevSmoke({ apiKey: '', fetchImpl })).rejects.toThrow('OPENROUTER_API_KEY missing'); + expect(fetchImpl).not.toHaveBeenCalled(); + }); + + it('rejects unsuccessful and malformed responses', async () => { + await expect(runJevSmoke({ + apiKey: 'key', + fetchImpl: vi.fn(async () => new Response('unavailable', { status: 503 })), + })).rejects.toThrow('OpenRouter Jev smoke failed with 503'); + await expect(runJevSmoke({ + apiKey: 'key', + fetchImpl: vi.fn(async () => new Response(JSON.stringify({ model: 'typesafe/jev-1.13', answers: {} }), { status: 200 })), + })).rejects.toThrow('malformed'); + }); +}); diff --git a/src/test/security-hardening.test.ts b/src/test/security-hardening.test.ts index d11f033d4..aba52d94f 100644 --- a/src/test/security-hardening.test.ts +++ b/src/test/security-hardening.test.ts @@ -69,7 +69,8 @@ describe('security hardening regressions', () => { expect(modelCatalog).toContain('id: "openai/gpt-5.6-luna"'); expect(modelCatalog).toContain('id: "google/gemini-3.8-flash"'); expect(ai).toContain('function modelForRequest'); - expect(stream).toContain('function modelForRequest'); + expect(stream).toContain('isModelSelection(rawModel)'); + expect(stream).toContain('resolveModelSelection({'); expect(ai).toContain('const provider: Provider = OPENROUTER_PROVIDER'); expect(stream).toContain('const OPENROUTER_PROVIDER = "openrouter"'); expect(ai).toContain('https://openrouter.ai/api/v1/chat/completions'); diff --git a/supabase/README.md b/supabase/README.md index ff9206687..e8ec17265 100644 --- a/supabase/README.md +++ b/supabase/README.md @@ -67,6 +67,7 @@ Other shared modules own: - model pricing and normalized cost calculations; - control enforcement; - crossfire response validation; +- the pinned OpenRouter Jev Decisions client, automatic model routing, semantic source decisions, and Crossfire signals; - announcement input validation. Move code into `_shared` only when multiple functions use the same boundary or policy. Feature-specific behavior stays with its handler. @@ -144,6 +145,8 @@ Document names, never values. Depending on deployed features, server-side config Use `supabase secrets` or protected deployment configuration for server values. Never place server secrets in `VITE_*`, source files, docs, screenshots, traces, logs, or comments. +Jev uses the same `OPENROUTER_API_KEY` as every other model request; there is no separate TypeSafe credential. `npm run smoke:jev` exercises one fixed, sanitized typed decision against the live Decisions endpoint and incurs a negligible provider charge. + The invitation handler does not select an email provider. Its mail boundary sends `{ to, inviteUrl, idempotencyKey }` to the configured relay. The relay must deduplicate repeated acceptance of the same key, which stays stable across pending-delivery retries. The handler marks delivery only after relay acceptance and suppresses later sends only after that mark commits. Email creation fails before storing an invite when the three mailer values are absent. Local tests inject an in-memory sender and make no external mail request. Public browser configuration in [`../src/lib/supabase/env.ts`](../src/lib/supabase/env.ts) is not a replacement for server authorization. diff --git a/supabase/functions/_shared/auto-model-router.test.ts b/supabase/functions/_shared/auto-model-router.test.ts index 72c7ddcc1..8ac417f13 100644 --- a/supabase/functions/_shared/auto-model-router.test.ts +++ b/supabase/functions/_shared/auto-model-router.test.ts @@ -176,7 +176,7 @@ Deno.test("router bounds state and propagates caller cancellation", async () => try { await pending; } catch (error) { - if (!(error instanceof Error) || !error.message.includes("User cancelled routing")) throw error; + if (!error || typeof error !== "object" || !("message" in error) || !String(error.message).includes("User cancelled routing")) throw error; if (JSON.stringify(observed.state).length > 8_200) throw new Error("routing state exceeded its bound"); return; } diff --git a/supabase/functions/_shared/crossfire-signals.test.ts b/supabase/functions/_shared/crossfire-signals.test.ts index 65498ce5f..f56d0e596 100644 --- a/supabase/functions/_shared/crossfire-signals.test.ts +++ b/supabase/functions/_shared/crossfire-signals.test.ts @@ -111,7 +111,7 @@ Deno.test("Crossfire signal does not start work after caller cancellation", asyn }, }); } catch (error) { - if (!(error instanceof Error) || error.name !== "AbortError" || calls !== 0) throw error; + if (!error || typeof error !== "object" || !("name" in error) || error.name !== "AbortError" || calls !== 0) throw error; return; } throw new Error("pre-aborted signal was ignored"); From 5c908254101ad66eadc371cdf69b909447733078 Mon Sep 17 00:00:00 2001 From: Adam Date: Mon, 21 Sep 2026 17:40:38 +0000 Subject: [PATCH 8/9] fix(ai): harden Jev fallback and Crossfire signals --- docs/architecture.md | 2 +- e2e/crossfire.spec.ts | 35 +++-- src/lib/ai/client.ts | 65 ++++++---- src/lib/crossfire/reducer.ts | 25 +++- src/lib/crossfire/schemas.ts | 8 ++ src/pages/CrossfirePage.tsx | 16 ++- src/test/ai/crossfire-client.test.ts | 25 ++-- src/test/crossfire/reducer.test.ts | 7 +- src/test/pages/CrossfirePage.test.tsx | 7 +- .../_shared/auto-model-router.test.ts | 3 +- .../functions/_shared/auto-model-router.ts | 6 +- .../_shared/crossfire-signals.test.ts | 10 +- .../functions/_shared/crossfire-signals.ts | 12 +- supabase/functions/_shared/jev.test.ts | 21 +++ supabase/functions/_shared/jev.ts | 25 +++- .../_shared/source-decisions.test.ts | 17 +++ .../functions/_shared/source-decisions.ts | 20 ++- supabase/functions/ai-stream/index.ts | 9 +- .../functions/ai/handler-research.test.ts | 51 +++++--- supabase/functions/ai/index.ts | 122 +++++++++++++++--- 20 files changed, 371 insertions(+), 115 deletions(-) diff --git a/docs/architecture.md b/docs/architecture.md index 7e657f757..0a28f4026 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -58,7 +58,7 @@ Client-side environment selection is a developer convenience, not authorization. [`supabase/functions/ai/index.ts`](../supabase/functions/ai/index.ts) serves non-streaming structured tools and practice modes. [`supabase/functions/ai-stream/index.ts`](../supabase/functions/ai-stream/index.ts) serves Ask over server-sent events. -For a validated Crossfire next turn, the server may ask Jev to classify only the newest completed question-answer exchange. The optional directness, concession, and follow-up signal is attached by turn ID to the answer; a missing or failed Jev decision never invalidates the main Crossfire reply. Draft version 1 remains compatible because turn signals are optional. +For a validated Crossfire next turn, the server returns the reply first with a short-lived signed sidecar token. The client then requests a Jev classification of only the newest completed question-answer exchange, without reserving a second daily quota unit. The optional directness, concession, and follow-up signal is attached by turn ID to the answer; a missing, slow, forged, or failed sidecar never delays or invalidates the main reply. Draft version 1 remains compatible because turn signals are optional. A configured mock provider bypasses Supabase for local and test behavior. It is not authenticated backend evidence. diff --git a/e2e/crossfire.spec.ts b/e2e/crossfire.spec.ts index c376079cb..60ec74fc1 100644 --- a/e2e/crossfire.spec.ts +++ b/e2e/crossfire.spec.ts @@ -4,8 +4,10 @@ import { setLocale, stubSupabaseSession } from './helpers/auth'; type CrossfireRequest = { feature: string; input: { - operation: 'next-turn' | 'debrief'; + operation: 'next-turn' | 'debrief' | 'signal'; expectedReply?: 'question' | 'answer'; + reply?: { type: 'question' | 'answer'; text: string }; + signalToken?: string; config: { mode: 'question-ai' | 'ai-questions-user'; useSourceLibrary: boolean }; transcript: Array<{ id: string; kind: 'question' | 'answer'; text: string }>; }; @@ -43,6 +45,21 @@ async function installDeterministicBackend(page: Page, requests: CrossfireReques if (url.pathname.endsWith('/functions/v1/ai')) { const body = route.request().postDataJSON() as CrossfireRequest; requests.push(body); + if (body.input.operation === 'signal') { + const answerTurnId = body.input.reply?.type === 'answer' + ? `T${body.input.transcript.length + 1}` + : body.input.transcript.at(-1)?.kind === 'answer' ? body.input.transcript.at(-1)?.id : undefined; + const signal = answerTurnId ? { + answerTurnId, + answerQuality: body.input.reply?.type === 'answer' ? 'direct' : 'partial', + answerQualityConfidence: 0.91, + concession: body.input.reply?.type === 'answer' ? 'none' : 'implicit', + concessionConfidence: 0.76, + followUpOpportunity: 0.82, + } : undefined; + await route.fulfill({ status: 200, contentType: 'application/json', body: JSON.stringify({ output: signal ? { signal } : {} }) }); + return; + } if (body.input.operation === 'next-turn') { if (failFirstAnswer && !answerFailed && body.input.expectedReply === 'answer') { answerFailed = true; @@ -60,18 +77,8 @@ async function installDeterministicBackend(page: Page, requests: CrossfireReques ? 'What evidence connects fewer private cars to safer streets?' : 'How would you answer the strongest enforcement objection?', }; - const answerTurnId = body.input.expectedReply === 'answer' - ? `T${body.input.transcript.length + 1}` - : body.input.transcript.at(-1)?.kind === 'answer' ? body.input.transcript.at(-1)?.id : undefined; - const signal = answerTurnId ? { - answerTurnId, - answerQuality: body.input.expectedReply === 'answer' ? 'direct' : 'partial', - answerQualityConfidence: 0.91, - concession: body.input.expectedReply === 'answer' ? 'none' : 'implicit', - concessionConfidence: 0.76, - followUpOpportunity: 0.82, - } : undefined; - await route.fulfill({ status: 200, contentType: 'application/json', body: JSON.stringify({ output, ...(signal ? { signal } : {}) }) }); + const hasCompletedExchange = output.type === 'answer' || body.input.transcript.at(-1)?.kind === 'answer'; + await route.fulfill({ status: 200, contentType: 'application/json', body: JSON.stringify({ output, ...(hasCompletedExchange ? { signalToken: 'e2e-signal-token' } : {}) }) }); return; } @@ -151,7 +158,7 @@ test('Let the AI question me alternates roles and completes a debrief', async ({ await expect(page.getByTestId('crossfire-debrief')).toBeVisible(); await expect(page.getByText('Partial answer')).toBeVisible(); expect(requests[0].input).toMatchObject({ operation: 'next-turn', expectedReply: 'question', config: { mode: 'ai-questions-user', useSourceLibrary: false } }); - expect(requests[1].input.transcript.at(-1)?.text).toBe('Traffic exposure falls when trips shift to transit and walking.'); + expect(requests.filter((request) => request.input.operation === 'next-turn')[1].input.transcript.at(-1)?.text).toBe('Traffic exposure falls when trips shift to transit and walking.'); expect(requests.at(-1)?.input.operation).toBe('debrief'); await page.screenshot({ path: testInfo.outputPath('crossfire-ai-questions-user-final.png'), fullPage: true }); }); diff --git a/src/lib/ai/client.ts b/src/lib/ai/client.ts index f88c99a79..a727f6738 100644 --- a/src/lib/ai/client.ts +++ b/src/lib/ai/client.ts @@ -39,11 +39,13 @@ import { CrossfireAIInputSchema, CrossfireDebriefSchema, CrossfireReplySchema, + CrossfireSignalInputSchema, CrossfireTurnSignalSchema, collectDebriefTurnIds, type CrossfireAIInput, type CrossfireDebrief, type CrossfireReply, + type CrossfireSignalInput, type CrossfireTurnSignal, } from '@/lib/crossfire/schemas'; @@ -739,7 +741,7 @@ export async function crossfire( rawInput: CrossfireAIInput, config: ProviderConfig, options: Pick = {}, -): Promise<(CrossfireReply & { signal?: CrossfireTurnSignal }) | CrossfireDebrief> { +): Promise<(CrossfireReply & { signalToken?: string }) | CrossfireDebrief> { const input = CrossfireAIInputSchema.safeParse(rawInput); if (!input.success) { throw new AIResponseError('Crossfire request did not match the expected schema.', { @@ -757,7 +759,7 @@ export async function crossfire( timeoutMs: CROSSFIRE_REQUEST_TIMEOUT_MS, signal: options.signal, transformResponse: (rawData) => { - const data = rawData as { output: unknown; signal?: unknown }; + const data = rawData as { output: unknown; signalToken?: unknown }; if (input.data.operation === 'next-turn') { const reply = CrossfireReplySchema.safeParse(data.output); if (!reply.success || reply.data.type !== input.data.expectedReply) { @@ -767,27 +769,11 @@ export async function crossfire( : { issues: reply.error.issues }, }); } - if (data.signal === undefined) return reply.data; - const signal = CrossfireTurnSignalSchema.safeParse(data.signal); - if (!signal.success) { - throw new AIResponseError('Crossfire signal did not match the expected schema.', { details: { issues: signal.error.issues } }); + if (data.signalToken === undefined) return reply.data; + if (typeof data.signalToken !== 'string' || !data.signalToken.trim() || data.signalToken.length > 128) { + throw new AIResponseError('Crossfire signal token was invalid.'); } - const answerTurns = [ - ...input.data.transcript, - { - id: `T${input.data.transcript.length + 1}`, - sequence: input.data.transcript.length + 1, - speaker: 'ai' as const, - kind: reply.data.type, - text: reply.data.text, - createdAt: 0, - }, - ]; - const target = answerTurns.find((turn) => turn.id === signal.data.answerTurnId); - if (!target || target.kind !== 'answer') { - throw new AIResponseError('Crossfire signal referenced an unknown or non-answer turn.'); - } - return { ...reply.data, signal: signal.data }; + return { ...reply.data, signalToken: data.signalToken }; } const debrief = CrossfireDebriefSchema.safeParse(data.output); @@ -803,6 +789,41 @@ export async function crossfire( ); } +export async function crossfireSignal( + rawInput: CrossfireSignalInput, + config: ProviderConfig, + options: Pick = {}, +): Promise { + const input = CrossfireSignalInputSchema.safeParse(rawInput); + if (!input.success) { + throw new AIResponseError('Crossfire signal request did not match the expected schema.', { details: { issues: input.error.issues } }); + } + return await callSupabaseAI( + 'crossfire', + withOutputLanguage({ operation: 'signal', ...input.data }), + config, + { + includeActiveSources: false, + includeCustomSystemPrompt: false, + timeoutMs: 5_000, + signal: options.signal, + transformResponse: (rawData) => { + const output = (rawData as { output?: unknown }).output; + if (!output || typeof output !== 'object' || Array.isArray(output) || !('signal' in output)) return undefined; + const parsed = CrossfireTurnSignalSchema.safeParse((output as { signal: unknown }).signal); + if (!parsed.success) throw new AIResponseError('Crossfire signal did not match the expected schema.', { details: { issues: parsed.error.issues } }); + const turns = [ + ...input.data.transcript, + { id: `T${input.data.transcript.length + 1}`, kind: input.data.reply.type }, + ]; + const target = turns.find((turn) => turn.id === parsed.data.answerTurnId); + if (!target || target.kind !== 'answer') throw new AIResponseError('Crossfire signal referenced an unknown or non-answer turn.'); + return parsed.data; + }, + }, + ); +} + export function mapAIError( error: { message: string; context?: unknown }, requestId: string, diff --git a/src/lib/crossfire/reducer.ts b/src/lib/crossfire/reducer.ts index 46ec88066..dcda7f041 100644 --- a/src/lib/crossfire/reducer.ts +++ b/src/lib/crossfire/reducer.ts @@ -19,6 +19,7 @@ export type CrossfireAction = | { type: 'session-started'; sessionId: string; now: number } | { type: 'user-turn-submitted'; turn: CrossfireTurn; expectedReply: 'question' | 'answer' } | { type: 'ai-turn-received'; turn: CrossfireTurn; signal?: CrossfireTurnSignal } + | { type: 'signal-received'; signal: CrossfireTurnSignal } | { type: 'request-failed'; expectedReply: 'question' | 'answer'; @@ -67,6 +68,16 @@ function appendTurn(turns: CrossfireTurn[], turn: CrossfireTurn): CrossfireTurn[ return [...turns, parsed.data]; } +function attachSignal(turns: CrossfireTurn[], value: CrossfireTurnSignal): CrossfireTurn[] | null { + const signal = CrossfireTurnSignalSchema.safeParse(value); + if (!signal.success) return null; + const targetIndex = turns.findIndex((turn) => turn.id === signal.data.answerTurnId && turn.kind === 'answer'); + if (targetIndex < 0) return null; + const target = CrossfireTurnSchema.safeParse({ ...turns[targetIndex], signal: signal.data }); + if (!target.success) return null; + return turns.map((turn, index) => index === targetIndex ? target.data : turn); +} + export function crossfireReducer(state: CrossfireState, action: CrossfireAction): CrossfireState { switch (action.type) { case 'config-patched': { @@ -111,16 +122,16 @@ export function crossfireReducer(state: CrossfireState, action: CrossfireAction) let turns = appendTurn(state.turns, action.turn); if (!turns) return state; if (action.signal) { - const signal = CrossfireTurnSignalSchema.safeParse(action.signal); - if (!signal.success) return state; - const targetIndex = turns.findIndex((turn) => turn.id === signal.data.answerTurnId && turn.kind === 'answer'); - if (targetIndex < 0) return state; - const target = CrossfireTurnSchema.safeParse({ ...turns[targetIndex], signal: signal.data }); - if (!target.success) return state; - turns = turns.map((turn, index) => index === targetIndex ? target.data : turn); + turns = attachSignal(turns, action.signal); + if (!turns) return state; } return { ...state, turns, request: { status: 'idle' } }; } + case 'signal-received': { + if (state.phase === 'setup') return state; + const turns = attachSignal(state.turns, action.signal); + return turns ? { ...state, turns } : state; + } case 'request-failed': { if (state.phase !== 'active' || state.request.status !== 'pending') return state; if (action.expectedReply !== state.request.expectedReply) return state; diff --git a/src/lib/crossfire/schemas.ts b/src/lib/crossfire/schemas.ts index fcf3bc47d..1e0fe8e02 100644 --- a/src/lib/crossfire/schemas.ts +++ b/src/lib/crossfire/schemas.ts @@ -150,6 +150,14 @@ export const CrossfireAIInputSchema = z.discriminatedUnion('operation', [ ]); export type CrossfireAIInput = z.infer; +export const CrossfireSignalInputSchema = z.object({ + config: CrossfireConfigSchema, + transcript: CrossfireTranscriptSchema, + reply: CrossfireReplySchema, + signalToken: z.string().trim().min(1).max(128), +}).strict(); +export type CrossfireSignalInput = z.infer; + export const CrossfireRequestStateSchema = z.discriminatedUnion('status', [ z.object({ status: z.literal('idle') }).strict(), z.object({ diff --git a/src/pages/CrossfirePage.tsx b/src/pages/CrossfirePage.tsx index 5e94a03ed..68036dcea 100644 --- a/src/pages/CrossfirePage.tsx +++ b/src/pages/CrossfirePage.tsx @@ -22,7 +22,7 @@ import { Card, CardContent, CardDescription, CardHeader, CardTitle } from '@/com import { Checkbox } from '@/components/ui/checkbox'; import { Label } from '@/components/ui/label'; import { Textarea } from '@/components/ui/textarea'; -import { crossfire, AIResponseError, isTimeoutError } from '@/lib/ai/client'; +import { crossfire, crossfireSignal, AIResponseError, isTimeoutError } from '@/lib/ai/client'; import { getUserFacingAIError } from '@/lib/ai/error-display'; import { canEndCrossfireSession, @@ -316,11 +316,23 @@ export default function CrossfirePage() { }); return; } + const reply = { type: result.type, text: result.text }; dispatch({ type: 'ai-turn-received', turn: nextTurn(snapshot.turns.length + 1, 'ai', result.type, result.text), - signal: result.signal, }); + if (result.signalToken) { + void crossfireSignal({ + config: snapshot.config, + transcript: snapshot.turns, + reply, + signalToken: result.signalToken, + }, modelConfig, { signal: controller.signal }) + .then((signal) => { + if (signal && !controller.signal.aborted) dispatch({ type: 'signal-received', signal }); + }) + .catch((): void => {}); + } } catch (error) { if (controller.signal.aborted) return; const classified = classifyError(error); diff --git a/src/test/ai/crossfire-client.test.ts b/src/test/ai/crossfire-client.test.ts index 97a535b74..9edd0e3e0 100644 --- a/src/test/ai/crossfire-client.test.ts +++ b/src/test/ai/crossfire-client.test.ts @@ -34,7 +34,7 @@ vi.mock('@/lib/stores/session-store', () => ({ vi.mock('@/hooks/use-toast', () => ({ toast: vi.fn() })); -import { AIResponseError, crossfire } from '@/lib/ai/client'; +import { AIResponseError, crossfire, crossfireSignal } from '@/lib/ai/client'; const config: ProviderConfig = { provider: 'google', @@ -150,19 +150,20 @@ describe('Crossfire AI client', () => { it('parses a valid live signal and rejects unknown or non-answer targets', async () => { const signal = { answerTurnId: 'T2', answerQuality: 'direct', answerQualityConfidence: 0.9, concession: 'none', concessionConfidence: 0.8, followUpOpportunity: 0.75 }; - mocks.invoke.mockResolvedValueOnce({ data: { output: { type: 'answer', text: 'Answer.' }, signal }, error: null }); - await expect(crossfire(nextTurnInput(), config)).resolves.toEqual({ type: 'answer', text: 'Answer.', signal }); + mocks.invoke.mockResolvedValueOnce({ data: { output: { type: 'answer', text: 'Answer.' }, signalToken: 'signed-token' }, error: null }); + await expect(crossfire(nextTurnInput(), config)).resolves.toEqual({ type: 'answer', text: 'Answer.', signalToken: 'signed-token' }); - mocks.invoke.mockResolvedValueOnce({ data: { output: { type: 'answer', text: 'Answer.' }, signal: { ...signal, answerTurnId: 'T99' } }, error: null }); - await expect(crossfire(nextTurnInput(), config)).rejects.toBeInstanceOf(AIResponseError); + const next = nextTurnInput(); + if (next.operation !== 'next-turn') throw new Error('Unexpected fixture'); + const signalInput = { config: next.config, transcript: next.transcript, reply: { type: 'answer' as const, text: 'Answer.' }, signalToken: 'signed-token' }; + mocks.invoke.mockResolvedValueOnce({ data: { output: { signal } }, error: null }); + await expect(crossfireSignal(signalInput, config)).resolves.toEqual(signal); + + mocks.invoke.mockResolvedValueOnce({ data: { output: { signal: { ...signal, answerTurnId: 'T99' } } }, error: null }); + await expect(crossfireSignal(signalInput, config)).rejects.toBeInstanceOf(AIResponseError); - mocks.invoke.mockResolvedValueOnce({ data: { output: { type: 'question', text: 'Next?' }, signal: { ...signal, answerTurnId: 'T1' } }, error: null }); - const aiQuestionInput = nextTurnInput(); - if (aiQuestionInput.operation !== 'next-turn') throw new Error('Unexpected fixture'); - aiQuestionInput.config.mode = 'ai-questions-user'; - aiQuestionInput.transcript = []; - aiQuestionInput.expectedReply = 'question'; - await expect(crossfire(aiQuestionInput, config)).rejects.toBeInstanceOf(AIResponseError); + mocks.invoke.mockResolvedValueOnce({ data: { output: { signal: { ...signal, answerTurnId: 'T1' } } }, error: null }); + await expect(crossfireSignal(signalInput, config)).rejects.toBeInstanceOf(AIResponseError); }); it('validates debrief shape and every transcript reference', async () => { diff --git a/src/test/crossfire/reducer.test.ts b/src/test/crossfire/reducer.test.ts index 0dc6688b7..1191c070b 100644 --- a/src/test/crossfire/reducer.test.ts +++ b/src/test/crossfire/reducer.test.ts @@ -74,7 +74,9 @@ describe('crossfireReducer', () => { let state = crossfireReducer(configured(), { type: 'session-started', sessionId: 'signal-session', now: 1 }); state = crossfireReducer(state, { type: 'user-turn-submitted', turn: turn(1, 'user', 'question'), expectedReply: 'answer' }); const signal = { answerTurnId: 'T2', answerQuality: 'direct' as const, answerQualityConfidence: 0.9, concession: 'none' as const, concessionConfidence: 0.8, followUpOpportunity: 0.75 }; - state = crossfireReducer(state, { type: 'ai-turn-received', turn: turn(2, 'ai', 'answer'), signal }); + state = crossfireReducer(state, { type: 'ai-turn-received', turn: turn(2, 'ai', 'answer') }); + expect(state.phase === 'active' && state.turns[1].signal).toBeUndefined(); + state = crossfireReducer(state, { type: 'signal-received', signal }); expect(state.phase === 'active' && state.turns[0].signal).toBeUndefined(); expect(state.phase === 'active' && state.turns[1].signal).toEqual(signal); @@ -90,7 +92,8 @@ describe('crossfireReducer', () => { state = crossfireReducer(state, { type: 'ai-turn-received', turn: turn(1, 'ai', 'question') }); state = crossfireReducer(state, { type: 'user-turn-submitted', turn: turn(2, 'user', 'answer'), expectedReply: 'question' }); const signal = { answerTurnId: 'T2', answerQuality: 'partial' as const, answerQualityConfidence: 0.8, concession: 'implicit' as const, concessionConfidence: 0.7, followUpOpportunity: 0.9 }; - state = crossfireReducer(state, { type: 'ai-turn-received', turn: turn(3, 'ai', 'question'), signal }); + state = crossfireReducer(state, { type: 'ai-turn-received', turn: turn(3, 'ai', 'question') }); + state = crossfireReducer(state, { type: 'signal-received', signal }); expect(state.phase === 'active' && state.turns[1].signal).toEqual(signal); expect(state.phase === 'active' && state.turns[2].signal).toBeUndefined(); }); diff --git a/src/test/pages/CrossfirePage.test.tsx b/src/test/pages/CrossfirePage.test.tsx index a01e5a279..1a80e95c2 100644 --- a/src/test/pages/CrossfirePage.test.tsx +++ b/src/test/pages/CrossfirePage.test.tsx @@ -5,6 +5,7 @@ import { beforeEach, describe, expect, it, vi } from 'vitest'; const mocks = vi.hoisted(() => ({ userId: 'user-a', crossfire: vi.fn<() => Promise>(() => new Promise(() => undefined)), + crossfireSignal: vi.fn<() => Promise>(() => Promise.resolve(undefined)), })); vi.mock('@/lib/supabase/AuthProvider', () => ({ @@ -24,6 +25,7 @@ vi.mock('@/lib/stores/settings-store', () => ({ vi.mock('@/lib/ai/client', () => ({ AIResponseError: class AIResponseError extends Error {}, crossfire: mocks.crossfire, + crossfireSignal: mocks.crossfireSignal, isTimeoutError: () => false, })); @@ -44,6 +46,8 @@ describe('CrossfirePage', () => { mocks.userId = 'user-a'; mocks.crossfire.mockReset(); mocks.crossfire.mockImplementation(() => new Promise(() => undefined)); + mocks.crossfireSignal.mockReset(); + mocks.crossfireSignal.mockResolvedValue(undefined); localStorage.clear(); }); @@ -137,8 +141,9 @@ describe('CrossfirePage', () => { mocks.crossfire.mockResolvedValueOnce({ type: 'answer', text: 'A bounded answer.', - signal: { answerTurnId: 'T2', answerQuality: 'direct', answerQualityConfidence: 0.91, concession: 'implicit', concessionConfidence: 0.74, followUpOpportunity: 0.82 }, + signalToken: 'token', }); + mocks.crossfireSignal.mockResolvedValueOnce({ answerTurnId: 'T2', answerQuality: 'direct', answerQualityConfidence: 0.91, concession: 'implicit', concessionConfidence: 0.74, followUpOpportunity: 0.82 }); renderPage(); fireEvent.change(screen.getByTestId('crossfire-resolution'), { target: { value: 'Resolved: test' } }); fireEvent.change(screen.getByTestId('crossfire-argument'), { target: { value: 'Case text' } }); diff --git a/supabase/functions/_shared/auto-model-router.test.ts b/supabase/functions/_shared/auto-model-router.test.ts index 8ac417f13..0a48b5747 100644 --- a/supabase/functions/_shared/auto-model-router.test.ts +++ b/supabase/functions/_shared/auto-model-router.test.ts @@ -79,9 +79,10 @@ Deno.test("Auto Fast falls back to GLM when Jev is uncertain or unavailable", as feature: "ask", input: { message: "Help me" }, canAccessProModels: false, - callJevImpl: () => Promise.reject(new Error("provider unavailable")), + callJevImpl: () => Promise.reject(Object.assign(new Error("provider unavailable"), { usage: { inputTokens: 9, outputTokens: 0 } })), }); if (unavailable.resolvedModel !== "z-ai/glm-5.3-flash" || !unavailable.usedFallback) throw new Error("failed route did not fall back"); + if (unavailable.jevUsage?.inputTokens !== 9) throw new Error("failed route usage was discarded"); }); Deno.test("Auto Fast ignores a Jev label outside its candidate set", async () => { diff --git a/supabase/functions/_shared/auto-model-router.ts b/supabase/functions/_shared/auto-model-router.ts index 3edd0c124..1cb0358c4 100644 --- a/supabase/functions/_shared/auto-model-router.ts +++ b/supabase/functions/_shared/auto-model-router.ts @@ -8,9 +8,11 @@ import { } from "./model-catalog.ts"; import { callJev, + getJevErrorUsage, type CallJevArgs, type JevDecisionResult, } from "./jev.ts"; +import type { NormalizedUsage } from "./cost.ts"; const ROUTING_CONFIDENCE = 0.55; const MAX_ROUTING_STATE_CHARS = 8_000; @@ -49,6 +51,7 @@ export interface ResolvedModelSelection { modelEntry: ModelCatalogEntry; usedFallback: boolean; jev?: JevDecisionResult; + jevUsage?: NormalizedUsage; } function safeStringify(value: unknown): string { @@ -106,7 +109,8 @@ export async function resolveModelSelection(args: ResolveModelSelectionArgs): Pr if (isAbort(error, args.signal)) throw error; const modelEntry = getModelCatalogEntry(fallback); if (!modelEntry) throw new Error(`Missing fallback model: ${fallback}`); - return { requestedModel, resolvedModel: fallback, modelEntry, usedFallback: true }; + const jevUsage = getJevErrorUsage(error); + return { requestedModel, resolvedModel: fallback, modelEntry, usedFallback: true, ...(jevUsage ? { jevUsage } : {}) }; } const answer = jev.answers.model; diff --git a/supabase/functions/_shared/crossfire-signals.test.ts b/supabase/functions/_shared/crossfire-signals.test.ts index f56d0e596..edd4a389e 100644 --- a/supabase/functions/_shared/crossfire-signals.test.ts +++ b/supabase/functions/_shared/crossfire-signals.test.ts @@ -25,12 +25,17 @@ function signalResult(answerQuality = "direct", concession = "implicit"): JevDec } Deno.test("Crossfire signal evaluates an AI answer added after a user question", async () => { + let timeoutMs: number | undefined; const result = await evaluateCrossfireExchange({ transcript: [turn(1, "user", "question", "What evidence supports that?")], reply: { type: "answer", text: "A national survey supports it." }, - callJevImpl: () => Promise.resolve(signalResult("direct", "none")), + callJevImpl: (args) => { + timeoutMs = args.timeoutMs; + return Promise.resolve(signalResult("direct", "none")); + }, }); if (result.signal?.answerTurnId !== "T2" || result.signal.answerQuality !== "direct") throw new Error("AI answer signal was not attached to T2"); + if (timeoutMs !== 1_500) throw new Error(`Crossfire signal timeout was ${timeoutMs}`); }); Deno.test("Crossfire signal evaluates the user answer before the next AI question", async () => { @@ -91,9 +96,10 @@ Deno.test("Crossfire signal skips missing exchanges and invalid or failed decisi const failed = await evaluateCrossfireExchange({ transcript: [turn(1, "user", "question", "Question")], reply: { type: "answer", text: "Answer" }, - callJevImpl: () => Promise.reject(new Error("Jev 500")), + callJevImpl: () => Promise.reject(Object.assign(new Error("Jev 500"), { usage: { inputTokens: 12, outputTokens: 0 } })), }); if (failed.signal) throw new Error("failed decision discarded the optional-signal contract"); + if (failed.jevUsage?.inputTokens !== 12) throw new Error("failed signal usage was discarded"); }); Deno.test("Crossfire signal does not start work after caller cancellation", async () => { diff --git a/supabase/functions/_shared/crossfire-signals.ts b/supabase/functions/_shared/crossfire-signals.ts index 4f7e2ff8d..12b89eb22 100644 --- a/supabase/functions/_shared/crossfire-signals.ts +++ b/supabase/functions/_shared/crossfire-signals.ts @@ -1,10 +1,14 @@ import { callJev, + getJevErrorUsage, type CallJevArgs, type JevChoiceAnswer, type JevDecisionResult, type JevNoulAnswer, } from "./jev.ts"; +import type { NormalizedUsage } from "./cost.ts"; + +const CROSSFIRE_SIGNAL_TIMEOUT_MS = 1_500; export interface CrossfireSignalTurn { id: string; @@ -71,7 +75,7 @@ export async function evaluateCrossfireExchange({ reply: { type: "question" | "answer"; text: string }; signal?: AbortSignal; callJevImpl?: CrossfireSignalJev; -}): Promise<{ signal?: CrossfireTurnSignal; jev?: JevDecisionResult }> { +}): Promise<{ signal?: CrossfireTurnSignal; jev?: JevDecisionResult; jevUsage?: NormalizedUsage }> { if (signal?.aborted) throw signal.reason ?? new DOMException("The operation was aborted", "AbortError"); const exchange = newestCompletedExchange(transcript, reply); if (!exchange) return {}; @@ -99,12 +103,13 @@ export async function evaluateCrossfireExchange({ }, }, signal, + timeoutMs: CROSSFIRE_SIGNAL_TIMEOUT_MS, }); if (signal?.aborted) throw signal.reason ?? new DOMException("The operation was aborted", "AbortError"); const quality = choice(jev.answers.answer_quality, ["direct", "partial", "evasive"]); const concession = choice(jev.answers.concession, ["none", "implicit", "explicit"]); const followUp = noul(jev.answers.follow_up); - if (!quality || !concession || !followUp) return {}; + if (!quality || !concession || !followUp) return { jev }; return { signal: { answerTurnId: exchange.answer.id, @@ -118,6 +123,7 @@ export async function evaluateCrossfireExchange({ }; } catch (error) { if (isAbort(error, signal)) throw error; - return {}; + const jevUsage = getJevErrorUsage(error); + return jevUsage ? { jevUsage } : {}; } } diff --git a/supabase/functions/_shared/jev.test.ts b/supabase/functions/_shared/jev.test.ts index c7a7626ea..30f7ae897 100644 --- a/supabase/functions/_shared/jev.test.ts +++ b/supabase/functions/_shared/jev.test.ts @@ -1,5 +1,6 @@ import { callJev, + getJevErrorUsage, type JevQuestion, } from "./jev.ts"; @@ -146,6 +147,26 @@ Deno.test("Jev rejects a response missing a submitted answer", async () => { }), "concession"); }); +Deno.test("Jev preserves billed usage when a successful response is malformed", async () => { + const response = validResponse(); + delete (response.answers as Record).concession; + try { + await callJev({ + state: "state", + questions: QUESTIONS, + apiKey: "test-key", + fetchImpl: () => Promise.resolve(jsonResponse(response)), + }); + } catch (error) { + const usage = getJevErrorUsage(error); + if (usage?.inputTokens !== 41 || usage.reportedCostUsd !== 0.00000172 || usage.incomplete !== true) { + throw new Error(`malformed response usage was lost: ${JSON.stringify(usage)}`); + } + return; + } + throw new Error("malformed Jev response was accepted"); +}); + Deno.test("Jev bounds successful response bodies", async () => { const oversized = "x".repeat(1_048_577); await rejectsWith(() => callJev({ diff --git a/supabase/functions/_shared/jev.ts b/supabase/functions/_shared/jev.ts index a147e9fba..431ad6023 100644 --- a/supabase/functions/_shared/jev.ts +++ b/supabase/functions/_shared/jev.ts @@ -1,4 +1,5 @@ import { + markIncompleteUsage, normalizeOpenRouterUsage, type NormalizedUsage, } from "./cost.ts"; @@ -157,6 +158,18 @@ function normalizeJevUsage(value: unknown): NormalizedUsage | undefined { return { ...normalized, raw: usage }; } +export function getJevErrorUsage(error: unknown): NormalizedUsage | undefined { + if (!error || typeof error !== "object" || !("usage" in error)) return undefined; + const usage = (error as { usage?: unknown }).usage; + return usage && typeof usage === "object" ? markIncompleteUsage(usage as NormalizedUsage) : undefined; +} + +function attachObservedUsage(error: unknown, usage?: NormalizedUsage): unknown { + if (!usage || !error || typeof error !== "object") return error; + Object.assign(error, { usage: markIncompleteUsage(usage) }); + return error; +} + export async function callJev({ state, questions, @@ -176,6 +189,7 @@ export async function callJev({ const timeoutId = setTimeout(() => { controller.abort(new DOMException("Jev request timed out", "TimeoutError")); }, timeoutMs); + let observedUsage: NormalizedUsage | undefined; try { const response = await fetchImpl(JEV_DECISIONS_URL, { @@ -196,6 +210,12 @@ export async function callJev({ controller.signal, true, ); + try { + const errorPayload = asRecord(JSON.parse(body.text)); + observedUsage = normalizeJevUsage(errorPayload?.usage); + } catch { + // Provider error bodies are not guaranteed to be JSON. + } throw new Error(`OpenRouter Jev error ${response.status}: ${body.text}`); } @@ -207,6 +227,7 @@ export async function callJev({ false, ); const payload = asRecord(JSON.parse(body.text)); + observedUsage = normalizeJevUsage(payload?.usage); const rawAnswers = asRecord(payload?.answers); const model = typeof payload?.model === "string" && payload.model ? payload.model : null; if (!payload || !rawAnswers || !model) { @@ -219,10 +240,10 @@ export async function callJev({ const answers: Record = {}; for (const id of questionIds) answers[id] = parseAnswer(id, questions[id], rawAnswers[id]); - return { model, answers, usage: normalizeJevUsage(payload.usage) }; + return { model, answers, usage: observedUsage }; } catch (error) { if (controller.signal.aborted) throw controller.signal.reason; - throw error; + throw attachObservedUsage(error, observedUsage); } finally { clearTimeout(timeoutId); signal?.removeEventListener("abort", abortFromCaller); diff --git a/supabase/functions/_shared/source-decisions.test.ts b/supabase/functions/_shared/source-decisions.test.ts index a51bca6d2..1b50a9a88 100644 --- a/supabase/functions/_shared/source-decisions.test.ts +++ b/supabase/functions/_shared/source-decisions.test.ts @@ -53,6 +53,16 @@ Deno.test("all low-confidence semantic scores preserve deterministic source orde if (result.sources.map((source) => source.key).join(",") !== "official,study,forum") throw new Error("low-confidence batch reordered sources"); }); +Deno.test("all irrelevant semantic scores preserve deterministic source order", async () => { + const callJevImpl: SourceDecisionJev = (args) => Promise.resolve(answersFor(args, [ + { relevance: 0.1, importance: 1, strength: 1 }, + { relevance: 0.2, importance: 3, strength: 3 }, + { relevance: 0.3, importance: 3, strength: 3 }, + ])); + const result = await rankSourcesForRequest({ feature: "generate", requestText: "motion", sources, callJevImpl }); + if (result.sources.map((source) => source.key).join(",") !== "official,study,forum") throw new Error("all-irrelevant batch reordered sources"); +}); + Deno.test("failed or malformed Jev source decisions preserve deterministic source order", async () => { for (const callJevImpl of [ () => Promise.reject(new Error("unavailable")), @@ -61,6 +71,13 @@ Deno.test("failed or malformed Jev source decisions preserve deterministic sourc const result = await rankSourcesForRequest({ feature: "generate", requestText: "motion", sources, callJevImpl }); if (result.sources.map((source) => source.key).join(",") !== "official,study,forum") throw new Error("failed batch reordered sources"); } + const failedWithUsage = await rankSourcesForRequest({ + feature: "generate", + requestText: "motion", + sources, + callJevImpl: () => Promise.reject(Object.assign(new Error("malformed"), { usage: { inputTokens: 7, outputTokens: 0 } })), + }); + if (failedWithUsage.jevUsage?.inputTokens !== 7) throw new Error("failed source-decision usage was discarded"); }); Deno.test("case use requires probability above 0.70 and keeps the middle band uncertain", async () => { diff --git a/supabase/functions/_shared/source-decisions.ts b/supabase/functions/_shared/source-decisions.ts index 64adc3e93..3b06d2cae 100644 --- a/supabase/functions/_shared/source-decisions.ts +++ b/supabase/functions/_shared/source-decisions.ts @@ -1,10 +1,12 @@ import { callJev, + getJevErrorUsage, type CallJevArgs, type JevDecisionResult, type JevNoulAnswer, type JevScoreAnswer, } from "./jev.ts"; +import type { NormalizedUsage } from "./cost.ts"; const MAX_SOURCES = 10; const MAX_REQUEST_CHARS = 8_000; @@ -91,7 +93,7 @@ export async function rankSourcesForRequest({ sources: SourceDecisionCandidate[]; signal?: AbortSignal; callJevImpl?: SourceDecisionJev; -}): Promise<{ sources: SourceDecisionCandidate[]; assessments: SourceAssessment[]; jev?: JevDecisionResult }> { +}): Promise<{ sources: SourceDecisionCandidate[]; assessments: SourceAssessment[]; jev?: JevDecisionResult; jevUsage?: NormalizedUsage }> { const candidates = boundedSources(sources); if (candidates.length === 0) return { sources, assessments: [] }; const questions: CallJevArgs["questions"] = {}; @@ -135,14 +137,16 @@ export async function rankSourcesForRequest({ retained: sources[index].qualityLabel !== "weak" && relevance.noul >= RETAIN_RELEVANCE, }; }); - if (assessments.some((assessment) => assessment === null)) return { sources, assessments: [] }; + if (assessments.some((assessment) => assessment === null)) return { sources, assessments: [], jev }; const valid = assessments as SourceAssessment[]; const hasConfidentAssessment = candidates.some((_, index) => { const importance = scoreAnswer(jev.answers[`source_${index}_importance`]); const strength = scoreAnswer(jev.answers[`source_${index}_strength`]); return Boolean(importance && strength && importance.confidence >= SCORE_CONFIDENCE && strength.confidence >= SCORE_CONFIDENCE); }); - if (!hasConfidentAssessment) return { sources, assessments: valid, jev }; + if (!hasConfidentAssessment || valid.every((assessment) => assessment.relevance < RETAIN_RELEVANCE)) { + return { sources, assessments: valid, jev }; + } const assessmentByKey = new Map(valid.map((assessment) => [assessment.key, assessment])); const ranked = sources.map((source, index) => ({ source, index })).sort((a, b) => { @@ -160,7 +164,8 @@ export async function rankSourcesForRequest({ return { sources: ranked, assessments: valid, jev }; } catch (error) { if (isAbort(error, signal)) throw error; - return { sources, assessments: [] }; + const jevUsage = getJevErrorUsage(error); + return { sources, assessments: [], ...(jevUsage ? { jevUsage } : {}) }; } } @@ -174,7 +179,7 @@ export async function identifyCaseSourceUsage({ sources: Array>>; signal?: AbortSignal; callJevImpl?: SourceDecisionJev; -}): Promise<{ sourceUsage: CaseSourceUsage[]; jev?: JevDecisionResult }> { +}): Promise<{ sourceUsage: CaseSourceUsage[]; jev?: JevDecisionResult; jevUsage?: NormalizedUsage }> { const candidates = boundedSources(sources); if (candidates.length === 0) return { sourceUsage: [] }; const questions: CallJevArgs["questions"] = {}; @@ -202,9 +207,10 @@ export async function identifyCaseSourceUsage({ judgment: answer.noul > 0.70 ? "used" : answer.noul >= 0.30 ? "uncertain" : "not_used", }]; }); - return sourceUsage.length === candidates.length ? { sourceUsage, jev } : { sourceUsage: [] }; + return sourceUsage.length === candidates.length ? { sourceUsage, jev } : { sourceUsage: [], jev }; } catch (error) { if (isAbort(error, signal)) throw error; - return { sourceUsage: [] }; + const jevUsage = getJevErrorUsage(error); + return { sourceUsage: [], ...(jevUsage ? { jevUsage } : {}) }; } } diff --git a/supabase/functions/ai-stream/index.ts b/supabase/functions/ai-stream/index.ts index 322b957e8..7de16a2bc 100644 --- a/supabase/functions/ai-stream/index.ts +++ b/supabase/functions/ai-stream/index.ts @@ -381,15 +381,16 @@ export async function handleRequest(req: Request): Promise { }); model = resolution.resolvedModel; modelEntry = resolution.modelEntry; - if (resolution.jev?.usage) { + const routingUsage = resolution.jev?.usage ?? resolution.jevUsage; + if (routingUsage) { await logUsage( user.id, OPENROUTER_PROVIDER, JEV_MODEL, - resolution.jev.usage, + routingUsage, Date.now() - routingStartedAt, - "success", - undefined, + resolution.jevUsage ? "error" : "success", + resolution.jevUsage ? "jev_incomplete" : undefined, "jev_model_router", ); } diff --git a/supabase/functions/ai/handler-research.test.ts b/supabase/functions/ai/handler-research.test.ts index 7d0dbda5a..af3b8b951 100644 --- a/supabase/functions/ai/handler-research.test.ts +++ b/supabase/functions/ai/handler-research.test.ts @@ -451,6 +451,7 @@ Deno.test("AI handler ranks stable library sources and returns bounded assessmen Deno.test("Crossfire returns newest-exchange signals without making them a reply dependency", async () => { const originalFetch = globalThis.fetch; let decisionFails = false; + let decisionCalls = 0; globalThis.fetch = (async (input: Request | URL | string, init?: RequestInit) => { const url = new URL(input instanceof Request ? input.url : String(input)); const supabaseResponse = supabaseTestResponse(url); @@ -464,6 +465,7 @@ Deno.test("Crossfire returns newest-exchange signals without making them a reply return jsonResponse({ choices: [{ message: { content: JSON.stringify(output) } }], usage: { prompt_tokens: 10, completion_tokens: 5, cost: 0.000001 } }); } if (url.href === "https://openrouter.ai/api/alpha/decisions") { + decisionCalls += 1; if (decisionFails) return jsonResponse({ error: "unavailable" }, 500); return jsonResponse({ model: "typesafe/jev-1.13", @@ -478,6 +480,7 @@ Deno.test("Crossfire returns newest-exchange signals without making them a reply throw new Error(`Unhandled request: ${url}`); }) as typeof fetch; + const config = (mode: "question-ai" | "ai-questions-user") => ({ resolution: "Cities should ban cars.", argument: "Cars cause congestion.", mode, style: "cooperative", durationSeconds: 180, useSourceLibrary: false, seedQuestions: [] }); const request = (mode: "question-ai" | "ai-questions-user", transcript: unknown[], expectedReply: "question" | "answer") => handleRequest(new Request("http://127.0.0.1/functions/v1/ai", { method: "POST", headers: { Authorization: "Bearer test-user-token", "Content-Type": "application/json" }, @@ -486,35 +489,47 @@ Deno.test("Crossfire returns newest-exchange signals without making them a reply model: "openai/gpt-5.6-luna", input: { operation: "next-turn", - config: { resolution: "Cities should ban cars.", argument: "Cars cause congestion.", mode, style: "cooperative", durationSeconds: 180, useSourceLibrary: false, seedQuestions: [] }, + config: config(mode), transcript, expectedReply, }, }), })); + const requestSignal = (mode: "question-ai" | "ai-questions-user", transcript: unknown[], reply: { type: "question" | "answer"; text: string }, signalToken: string) => handleRequest(new Request("http://127.0.0.1/functions/v1/ai", { + method: "POST", + headers: { Authorization: "Bearer test-user-token", "Content-Type": "application/json" }, + body: JSON.stringify({ feature: "crossfire", model: "openai/gpt-5.6-luna", input: { operation: "signal", config: config(mode), transcript, reply, signalToken } }), + })); try { - const questionAi = await request("question-ai", [{ id: "T1", sequence: 1, speaker: "user", kind: "question", text: "What evidence supports that?", createdAt: 1 }], "answer"); - const questionAiPayload = await questionAi.json() as { signal?: { answerTurnId?: string; answerQuality?: string } }; - if (questionAi.status !== 200 || questionAiPayload.signal?.answerTurnId !== "T2" || questionAiPayload.signal.answerQuality !== "direct") { - throw new Error(`question-ai signal missing: ${JSON.stringify(questionAiPayload)}`); - } - - const aiQuestionsUser = await request("ai-questions-user", [ + const questionTranscript = [{ id: "T1", sequence: 1, speaker: "user", kind: "question", text: "What evidence supports that?", createdAt: 1 }]; + const questionAi = await request("question-ai", questionTranscript, "answer"); + const questionAiPayload = await questionAi.json() as { output?: { type?: string; text?: string }; signalToken?: string; signal?: unknown }; + if (questionAi.status !== 200 || questionAiPayload.output?.type !== "answer" || !questionAiPayload.signalToken || questionAiPayload.signal !== undefined) { + throw new Error(`question-ai reply was delayed by signal work: ${JSON.stringify(questionAiPayload)}`); + } + if (decisionCalls !== 0) throw new Error("main Crossfire reply waited for Jev"); + const questionSignal = await requestSignal("question-ai", questionTranscript, questionAiPayload.output as { type: "answer"; text: string }, questionAiPayload.signalToken); + const questionSignalPayload = await questionSignal.json() as { output?: { signal?: { answerTurnId?: string; answerQuality?: string } } }; + if (questionSignalPayload.output?.signal?.answerTurnId !== "T2" || questionSignalPayload.output.signal.answerQuality !== "direct") throw new Error(`question-ai signal missing: ${JSON.stringify(questionSignalPayload)}`); + const forgedSignal = await requestSignal("question-ai", questionTranscript, questionAiPayload.output as { type: "answer"; text: string }, "forged-token"); + if (forgedSignal.status !== 403) throw new Error("forged Crossfire signal token was accepted"); + + const answerTranscript = [ { id: "T1", sequence: 1, speaker: "ai", kind: "question", text: "Would costs rise?", createdAt: 1 }, { id: "T2", sequence: 2, speaker: "user", kind: "answer", text: "Some costs may rise.", createdAt: 2 }, - ], "question"); - const aiQuestionsPayload = await aiQuestionsUser.json() as { signal?: { answerTurnId?: string; concession?: string } }; - if (aiQuestionsUser.status !== 200 || aiQuestionsPayload.signal?.answerTurnId !== "T2" || aiQuestionsPayload.signal.concession !== "implicit") { - throw new Error(`ai-questions-user signal missing: ${JSON.stringify(aiQuestionsPayload)}`); - } + ]; + const aiQuestionsUser = await request("ai-questions-user", answerTranscript, "question"); + const aiQuestionsPayload = await aiQuestionsUser.json() as { output?: { type?: string; text?: string }; signalToken?: string }; + if (aiQuestionsUser.status !== 200 || aiQuestionsPayload.output?.type !== "question" || !aiQuestionsPayload.signalToken) throw new Error(`ai-questions-user reply missing: ${JSON.stringify(aiQuestionsPayload)}`); + const answerSignal = await requestSignal("ai-questions-user", answerTranscript, aiQuestionsPayload.output as { type: "question"; text: string }, aiQuestionsPayload.signalToken); + const answerSignalPayload = await answerSignal.json() as { output?: { signal?: { answerTurnId?: string; concession?: string } } }; + if (answerSignalPayload.output?.signal?.answerTurnId !== "T2" || answerSignalPayload.output.signal.concession !== "implicit") throw new Error(`ai-questions-user signal missing: ${JSON.stringify(answerSignalPayload)}`); decisionFails = true; - const fallback = await request("question-ai", [{ id: "T1", sequence: 1, speaker: "user", kind: "question", text: "What evidence supports that?", createdAt: 1 }], "answer"); - const fallbackPayload = await fallback.json() as { output?: { type?: string }; signal?: unknown }; - if (fallback.status !== 200 || fallbackPayload.output?.type !== "answer" || fallbackPayload.signal !== undefined) { - throw new Error(`failed optional signal discarded the reply: ${JSON.stringify(fallbackPayload)}`); - } + const fallback = await requestSignal("question-ai", questionTranscript, questionAiPayload.output as { type: "answer"; text: string }, questionAiPayload.signalToken); + const fallbackPayload = await fallback.json() as { output?: { signal?: unknown } }; + if (fallback.status !== 200 || fallbackPayload.output?.signal !== undefined) throw new Error(`failed optional signal returned invalid output: ${JSON.stringify(fallbackPayload)}`); } finally { globalThis.fetch = originalFetch; } diff --git a/supabase/functions/ai/index.ts b/supabase/functions/ai/index.ts index 4b5232379..b5f7284db 100644 --- a/supabase/functions/ai/index.ts +++ b/supabase/functions/ai/index.ts @@ -654,15 +654,25 @@ function crossfireTranscript(value: unknown, config: CrossfirePromptConfig): { v type CrossfireNextTurnInput = CrossfireNextTurnPromptInput & { operation: "next-turn" }; type CrossfireDebriefInput = CrossfireDebriefPromptInput & { operation: "debrief" }; -type CrossfireInput = CrossfireNextTurnInput | CrossfireDebriefInput; +type CrossfireSignalInput = { + operation: "signal"; + config: CrossfirePromptConfig; + transcript: CrossfirePromptTurn[]; + reply: { type: "question" | "answer"; text: string }; + signalToken: string; + language: string; +}; +type CrossfireInput = CrossfireNextTurnInput | CrossfireDebriefInput | CrossfireSignalInput; function validateCrossfireInput(value: unknown): { value: CrossfireInput } | { error: string } { - if (!isRecord(value) || (value.operation !== "next-turn" && value.operation !== "debrief")) { - return { error: "Crossfire operation must be next-turn or debrief" }; + if (!isRecord(value) || (value.operation !== "next-turn" && value.operation !== "debrief" && value.operation !== "signal")) { + return { error: "Crossfire operation must be next-turn, debrief, or signal" }; } const allowedKeys = value.operation === "next-turn" ? ["operation", "config", "transcript", "expectedReply", "language"] - : ["operation", "config", "transcript", "language"]; + : value.operation === "signal" + ? ["operation", "config", "transcript", "reply", "signalToken", "language"] + : ["operation", "config", "transcript", "language"]; if (!hasOnlyKeys(value, allowedKeys)) return { error: "Crossfire input contains unsupported fields" }; const rawLanguage = value.language; const languageError = validateControlInput("crossfire", rawLanguage === undefined ? {} : { language: rawLanguage }); @@ -672,6 +682,24 @@ function validateCrossfireInput(value: unknown): { value: CrossfireInput } | { e if ("error" in config) return config; const transcript = crossfireTranscript(value.transcript, config.value); if ("error" in transcript) return transcript; + if (value.operation === "signal") { + if (!isRecord(value.reply) || !hasOnlyKeys(value.reply, ["type", "text"]) || + (value.reply.type !== "question" && value.reply.type !== "answer")) return { error: "Crossfire signal reply is invalid" }; + const replyText = crossfireText(value.reply.text, "reply.text", MAX_CROSSFIRE_REPLY_CHARS); + if ("error" in replyText) return replyText; + const signalToken = crossfireText(value.signalToken, "signalToken", 128); + if ("error" in signalToken) return signalToken; + return { + value: { + operation: "signal", + config: config.value, + transcript: transcript.value, + reply: { type: value.reply.type, text: replyText.value }, + signalToken: signalToken.value, + language, + }, + }; + } if (value.operation === "debrief") { if (transcript.value.length === 0) return { error: "Crossfire debrief transcript must not be empty" }; return { value: { operation: "debrief", config: config.value, transcript: transcript.value, language } }; @@ -698,6 +726,49 @@ function validateCrossfireInput(value: unknown): { value: CrossfireInput } | { e }; } +const CROSSFIRE_SIGNAL_TOKEN_TTL_MS = 120_000; + +function crossfireSignalTokenPayload(userId: string, input: Pick, issuedAt: number): string { + return JSON.stringify({ + domain: "debatelab-crossfire-signal-v1", + userId, + issuedAt, + mode: input.config.mode, + transcript: input.transcript, + reply: input.reply, + }); +} + +async function crossfireSignalMac(payload: string): Promise { + const encoder = new TextEncoder(); + const key = await crypto.subtle.importKey( + "raw", + encoder.encode(sparringIntegritySecret), + { name: "HMAC", hash: "SHA-256" }, + false, + ["sign"], + ); + const signature = new Uint8Array(await crypto.subtle.sign("HMAC", key, encoder.encode(payload))); + return Array.from(signature, (byte) => byte.toString(16).padStart(2, "0")).join(""); +} + +async function createCrossfireSignalToken(userId: string, input: Pick): Promise { + const issuedAt = Date.now(); + return `${issuedAt}.${await crossfireSignalMac(crossfireSignalTokenPayload(userId, input, issuedAt))}`; +} + +async function verifyCrossfireSignalToken(userId: string, input: CrossfireSignalInput): Promise { + const [rawIssuedAt, suppliedMac, ...extra] = input.signalToken.split("."); + const issuedAt = Number(rawIssuedAt); + if (extra.length > 0 || !Number.isInteger(issuedAt) || !/^[a-f0-9]{64}$/.test(suppliedMac ?? "")) return false; + const age = Date.now() - issuedAt; + if (age < 0 || age > CROSSFIRE_SIGNAL_TOKEN_TTL_MS) return false; + const expectedMac = await crossfireSignalMac(crossfireSignalTokenPayload(userId, input, issuedAt)); + let mismatch = expectedMac.length ^ suppliedMac.length; + for (let index = 0; index < expectedMac.length; index += 1) mismatch |= expectedMac.charCodeAt(index) ^ suppliedMac.charCodeAt(index); + return mismatch === 0; +} + function boundCrossfireSources(value: unknown, enabled: boolean): { value: PromptSource[]; truncated: boolean } | { error: string } { if (!enabled || value === undefined || value === null) return { value: [], truncated: false }; if (!Array.isArray(value)) return { error: "sources must be an array when Source Library is enabled" }; @@ -1148,6 +1219,22 @@ export async function handleRequest(req: Request): Promise { const userId: string = user.id; log(requestId, "INFO", "User authenticated", { userId }); + + if (crossfireInput?.operation === "signal") { + if (await checkUserBanned(userId)) return badRequest("Your account has been suspended. Please contact support.", corsHeaders, 403); + if (!(await verifyCrossfireSignalToken(userId, crossfireInput))) return badRequest("Invalid or expired Crossfire signal token", corsHeaders, 403); + const evaluated = await evaluateCrossfireExchange({ + transcript: crossfireInput.transcript, + reply: crossfireInput.reply, + signal: req.signal, + }); + const signalUsage = evaluated.jev?.usage ?? evaluated.jevUsage; + if (signalUsage) await logUsage(userId, "jev_crossfire_signal", provider, JEV_MODEL, signalUsage, undefined, evaluated.jevUsage ? "error" : "success", evaluated.jevUsage ? "jev_incomplete" : undefined); + return new Response(JSON.stringify(successBody({ output: evaluated.signal ? { signal: evaluated.signal } : {} })), { + headers: { ...corsHeaders, "Content-Type": "application/json" }, + }); + } + const sparringDatabase = narrowSparringDatabase(sparringSupabase); let sparringScoreClaimed = false; let sparringScoreClaimToken: string | undefined; @@ -1249,8 +1336,9 @@ export async function handleRequest(req: Request): Promise { model = resolution.resolvedModel; modelEntry = resolution.modelEntry; resolvedModelMeta = model; - if (resolution.jev?.usage) { - await logUsage(userId, "jev_model_router", provider, JEV_MODEL, resolution.jev.usage, Date.now() - routingStartedAt); + const routingUsage = resolution.jev?.usage ?? resolution.jevUsage; + if (routingUsage) { + await logUsage(userId, "jev_model_router", provider, JEV_MODEL, routingUsage, Date.now() - routingStartedAt, resolution.jevUsage ? "error" : "success", resolution.jevUsage ? "jev_incomplete" : undefined); } } catch (error) { if (sparringScoreClaimed && sparringScoreClaimToken && validatedSparringRequest) { @@ -1295,7 +1383,8 @@ export async function handleRequest(req: Request): Promise { }); sourceAssessments = ranked.assessments; sources = ranked.sources.map(({ title, content, origin, type }) => ({ title, content, ...(origin ? { origin } : {}), type })); - if (ranked.jev?.usage) await logUsage(userId, "jev_source_ranking", provider, JEV_MODEL, ranked.jev.usage); + const rankingUsage = ranked.jev?.usage ?? ranked.jevUsage; + if (rankingUsage) await logUsage(userId, "jev_source_ranking", provider, JEV_MODEL, rankingUsage, undefined, ranked.jevUsage ? "error" : "success", ranked.jevUsage ? "jev_incomplete" : undefined); } const reasoningEffort = effectiveReasoningEffort(model, requestedReasoningEffort ?? undefined, Boolean(rateLimit.canAccessProModels)); @@ -1452,20 +1541,19 @@ export async function handleRequest(req: Request): Promise { : validateCrossfireDebrief(cleanedOutput, crossfireInput.transcript); if ("error" in output) throw new Error(output.error); - let signal; - if (crossfireInput.operation === "next-turn") { - const evaluated = await evaluateCrossfireExchange({ + let signalToken: string | undefined; + if (crossfireInput.operation === "next-turn" && + ((output.value as { type?: unknown }).type === "answer" || crossfireInput.transcript.at(-1)?.kind === "answer")) { + signalToken = await createCrossfireSignalToken(userId, { + config: crossfireInput.config, transcript: crossfireInput.transcript, reply: output.value as { type: "question" | "answer"; text: string }, - signal: req.signal, }); - signal = evaluated.signal; - if (evaluated.jev?.usage) await logUsage(userId, "jev_crossfire_signal", provider, JEV_MODEL, evaluated.jev.usage); } const latencyMs = Date.now() - startTime; await logUsage(userId, feature, provider, model, aiResult.usage, latencyMs); - return new Response(JSON.stringify(successBody({ output: output.value, ...(signal ? { signal } : {}) })), { headers: { ...corsHeaders, "Content-Type": "application/json" } }); + return new Response(JSON.stringify(successBody({ output: output.value, ...(signalToken ? { signalToken } : {}) })), { headers: { ...corsHeaders, "Content-Type": "application/json" } }); } // FACTOIDS: Use deep research with multiple searches @@ -1690,7 +1778,8 @@ export async function handleRequest(req: Request): Promise { sources: discoveredCandidates, signal: req.signal, }); - if (discoveredRanking.jev?.usage) await logUsage(userId, "jev_source_ranking", provider, JEV_MODEL, discoveredRanking.jev.usage); + const discoveredRankingUsage = discoveredRanking.jev?.usage ?? discoveredRanking.jevUsage; + if (discoveredRankingUsage) await logUsage(userId, "jev_source_ranking", provider, JEV_MODEL, discoveredRankingUsage, undefined, discoveredRanking.jevUsage ? "error" : "success", discoveredRanking.jevUsage ? "jev_incomplete" : undefined); const discoveredAssessmentByKey = new Map(discoveredRanking.assessments.map((assessment) => [assessment.key, assessment])); const rankedDiscoveredSources = discoveredRanking.sources.map((candidate) => { const source = discoveredSources.find((item) => item.url === candidate.key)!; @@ -1766,7 +1855,8 @@ export async function handleRequest(req: Request): Promise { sources: usageCandidates, signal: req.signal, }); - if (caseUsage.jev?.usage) await logUsage(userId, "jev_source_usage", provider, JEV_MODEL, caseUsage.jev.usage); + const caseUsageCost = caseUsage.jev?.usage ?? caseUsage.jevUsage; + if (caseUsageCost) await logUsage(userId, "jev_source_usage", provider, JEV_MODEL, caseUsageCost, undefined, caseUsage.jevUsage ? "error" : "success", caseUsage.jevUsage ? "jev_incomplete" : undefined); const usedByKey = new Map(caseUsage.sourceUsage.map((usage) => [usage.key, usage])); for (const assessment of sourceAssessments) { (assessment as SourceAssessment & { usedInCase?: boolean }).usedInCase = usedByKey.get(assessment.key)?.judgment === "used"; From 249648dd97d4f9ef8be8bba1a217a7d1cc2fa395 Mon Sep 17 00:00:00 2001 From: Adam Date: Mon, 21 Sep 2026 18:00:51 +0000 Subject: [PATCH 9/9] fix(crossfire): bound Jev signal sidecars --- docs/architecture.md | 2 +- src/pages/CrossfirePage.tsx | 24 +++++++++++--- src/test/pages/CrossfirePage.test.tsx | 32 ++++++++++++++++++- .../functions/ai/handler-research.test.ts | 26 +++++++++++++-- supabase/functions/ai/index.ts | 21 ++++++++++++ 5 files changed, 95 insertions(+), 10 deletions(-) diff --git a/docs/architecture.md b/docs/architecture.md index 0a28f4026..e710e0369 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -58,7 +58,7 @@ Client-side environment selection is a developer convenience, not authorization. [`supabase/functions/ai/index.ts`](../supabase/functions/ai/index.ts) serves non-streaming structured tools and practice modes. [`supabase/functions/ai-stream/index.ts`](../supabase/functions/ai-stream/index.ts) serves Ask over server-sent events. -For a validated Crossfire next turn, the server returns the reply first with a short-lived signed sidecar token. The client then requests a Jev classification of only the newest completed question-answer exchange, without reserving a second daily quota unit. The optional directness, concession, and follow-up signal is attached by turn ID to the answer; a missing, slow, forged, or failed sidecar never delays or invalidates the main reply. Draft version 1 remains compatible because turn signals are optional. +For a validated Crossfire next turn, the server returns the reply first with a short-lived signed sidecar token. The client then requests a Jev classification of only the newest completed question-answer exchange. The token is atomically claimed through its own one-use counter, separate from the product's daily AI quota. The optional directness, concession, and follow-up signal is attached by turn ID to the answer; a missing, slow, replayed, forged, or failed sidecar never delays or invalidates the main reply. Draft version 1 remains compatible because turn signals are optional. A configured mock provider bypasses Supabase for local and test behavior. It is not authenticated backend evidence. diff --git a/src/pages/CrossfirePage.tsx b/src/pages/CrossfirePage.tsx index 68036dcea..b29e322ed 100644 --- a/src/pages/CrossfirePage.tsx +++ b/src/pages/CrossfirePage.tsx @@ -263,21 +263,31 @@ export default function CrossfirePage() { const [storageError, setStorageError] = useState(false); const [stateOwnerId, setStateOwnerId] = useState(user?.id); const requestRef = useRef(null); + const signalRequestsRef = useRef(new Set()); const shouldReduceMotion = useReducedMotion(); const activeSourceCount = useSessionStore((session) => session.sourceLibrary.filter((source) => source.isActive).length); const { getFeatureConfig } = useSettingsStore(); const modelConfig = getFeatureConfig('ask'); - useEffect(() => () => requestRef.current?.abort(), []); + const abortSignalRequests = useCallback(() => { + signalRequestsRef.current.forEach((controller) => controller.abort()); + signalRequestsRef.current.clear(); + }, []); + + useEffect(() => () => { + requestRef.current?.abort(); + abortSignalRequests(); + }, [abortSignalRequests]); useEffect(() => { if (stateOwnerId === user?.id) return; requestRef.current?.abort(); + abortSignalRequests(); dispatch({ type: 'state-restored', state: restoreState(user?.id, undefined) }); setStateOwnerId(user?.id); setDraftTurn(''); setFormError(''); - }, [stateOwnerId, user?.id]); + }, [abortSignalRequests, stateOwnerId, user?.id]); useEffect(() => { if (!user?.id || stateOwnerId !== user.id) return; @@ -322,16 +332,19 @@ export default function CrossfirePage() { turn: nextTurn(snapshot.turns.length + 1, 'ai', result.type, result.text), }); if (result.signalToken) { + const signalController = new AbortController(); + signalRequestsRef.current.add(signalController); void crossfireSignal({ config: snapshot.config, transcript: snapshot.turns, reply, signalToken: result.signalToken, - }, modelConfig, { signal: controller.signal }) + }, modelConfig, { signal: signalController.signal }) .then((signal) => { - if (signal && !controller.signal.aborted) dispatch({ type: 'signal-received', signal }); + if (signal && !signalController.signal.aborted) dispatch({ type: 'signal-received', signal }); }) - .catch((): void => {}); + .catch((): void => {}) + .finally(() => signalRequestsRef.current.delete(signalController)); } } catch (error) { if (controller.signal.aborted) return; @@ -445,6 +458,7 @@ export default function CrossfirePage() { const restart = () => { requestRef.current?.abort(); + abortSignalRequests(); if (user?.id) clearCrossfireDraft(user.id); dispatch({ type: 'session-reset' }); setDraftTurn(''); diff --git a/src/test/pages/CrossfirePage.test.tsx b/src/test/pages/CrossfirePage.test.tsx index 1a80e95c2..c7d6dc7ea 100644 --- a/src/test/pages/CrossfirePage.test.tsx +++ b/src/test/pages/CrossfirePage.test.tsx @@ -5,7 +5,7 @@ import { beforeEach, describe, expect, it, vi } from 'vitest'; const mocks = vi.hoisted(() => ({ userId: 'user-a', crossfire: vi.fn<() => Promise>(() => new Promise(() => undefined)), - crossfireSignal: vi.fn<() => Promise>(() => Promise.resolve(undefined)), + crossfireSignal: vi.fn<(input: unknown, config: unknown, options: { signal?: AbortSignal }) => Promise>(() => Promise.resolve(undefined)), })); vi.mock('@/lib/supabase/AuthProvider', () => ({ @@ -158,4 +158,34 @@ describe('CrossfirePage', () => { expect(screen.getByText('What is the mechanism?').closest('article')).not.toHaveTextContent('Direct answer'); expect(answer.closest('article')?.querySelector('[aria-label*="82%"]')).not.toBeNull(); }); + + it('keeps a delayed coaching signal alive while the next turn runs', async () => { + let resolveSignal: (value: unknown) => void = () => undefined; + let sidecarSignal: AbortSignal | undefined; + mocks.crossfire + .mockResolvedValueOnce({ type: 'answer', text: 'First bounded answer.', signalToken: 'token-1' }) + .mockResolvedValueOnce({ type: 'answer', text: 'Second bounded answer.' }); + mocks.crossfireSignal.mockImplementationOnce((_input, _config, options) => { + sidecarSignal = options.signal; + return new Promise((resolve) => { + resolveSignal = resolve; + }); + }); + renderPage(); + fireEvent.change(screen.getByTestId('crossfire-resolution'), { target: { value: 'Resolved: test' } }); + fireEvent.change(screen.getByTestId('crossfire-argument'), { target: { value: 'Case text' } }); + fireEvent.click(screen.getByTestId('start-crossfire')); + + fireEvent.change(screen.getByTestId('crossfire-input'), { target: { value: 'First question?' } }); + fireEvent.click(screen.getByTestId('submit-turn')); + const firstAnswer = await screen.findByText('First bounded answer.'); + + fireEvent.change(screen.getByTestId('crossfire-input'), { target: { value: 'Second question?' } }); + fireEvent.click(screen.getByTestId('submit-turn')); + expect(await screen.findByText('Second bounded answer.')).toBeInTheDocument(); + expect(sidecarSignal?.aborted).toBe(false); + + resolveSignal({ answerTurnId: 'T2', answerQuality: 'direct', answerQualityConfidence: 0.91, concession: 'none', concessionConfidence: 0.88, followUpOpportunity: 0.76 }); + await waitFor(() => expect(firstAnswer.closest('article')).toHaveTextContent('Direct answer')); + }); }); diff --git a/supabase/functions/ai/handler-research.test.ts b/supabase/functions/ai/handler-research.test.ts index af3b8b951..b07d7b577 100644 --- a/supabase/functions/ai/handler-research.test.ts +++ b/supabase/functions/ai/handler-research.test.ts @@ -452,8 +452,17 @@ Deno.test("Crossfire returns newest-exchange signals without making them a reply const originalFetch = globalThis.fetch; let decisionFails = false; let decisionCalls = 0; + const claimedSignalKeys = new Set(); globalThis.fetch = (async (input: Request | URL | string, init?: RequestInit) => { const url = new URL(input instanceof Request ? input.url : String(input)); + if (url.origin === "https://supabase.test" && url.pathname.includes("/rest/v1/rpc/consume_daily_quota")) { + const body = JSON.parse(String(init?.body ?? "{}")) as { p_quota_key?: string }; + if (body.p_quota_key?.startsWith("crossfire_signal:")) { + const allowed = !claimedSignalKeys.has(body.p_quota_key); + claimedSignalKeys.add(body.p_quota_key); + return jsonResponse([{ allowed, current_count: 1 }]); + } + } const supabaseResponse = supabaseTestResponse(url); if (supabaseResponse) return supabaseResponse; if (url.href === "https://openrouter.ai/api/v1/chat/completions") { @@ -512,6 +521,8 @@ Deno.test("Crossfire returns newest-exchange signals without making them a reply const questionSignal = await requestSignal("question-ai", questionTranscript, questionAiPayload.output as { type: "answer"; text: string }, questionAiPayload.signalToken); const questionSignalPayload = await questionSignal.json() as { output?: { signal?: { answerTurnId?: string; answerQuality?: string } } }; if (questionSignalPayload.output?.signal?.answerTurnId !== "T2" || questionSignalPayload.output.signal.answerQuality !== "direct") throw new Error(`question-ai signal missing: ${JSON.stringify(questionSignalPayload)}`); + const replayedSignal = await requestSignal("question-ai", questionTranscript, questionAiPayload.output as { type: "answer"; text: string }, questionAiPayload.signalToken); + if (replayedSignal.status !== 409 || Number(decisionCalls) !== 1) throw new Error("replayed Crossfire signal token reached Jev"); const forgedSignal = await requestSignal("question-ai", questionTranscript, questionAiPayload.output as { type: "answer"; text: string }, "forged-token"); if (forgedSignal.status !== 403) throw new Error("forged Crossfire signal token was accepted"); @@ -522,12 +533,21 @@ Deno.test("Crossfire returns newest-exchange signals without making them a reply const aiQuestionsUser = await request("ai-questions-user", answerTranscript, "question"); const aiQuestionsPayload = await aiQuestionsUser.json() as { output?: { type?: string; text?: string }; signalToken?: string }; if (aiQuestionsUser.status !== 200 || aiQuestionsPayload.output?.type !== "question" || !aiQuestionsPayload.signalToken) throw new Error(`ai-questions-user reply missing: ${JSON.stringify(aiQuestionsPayload)}`); - const answerSignal = await requestSignal("ai-questions-user", answerTranscript, aiQuestionsPayload.output as { type: "question"; text: string }, aiQuestionsPayload.signalToken); - const answerSignalPayload = await answerSignal.json() as { output?: { signal?: { answerTurnId?: string; concession?: string } } }; + const decisionCallsBeforeReplayRace = decisionCalls; + const racedSignals = await Promise.all(Array.from({ length: 4 }, () => requestSignal("ai-questions-user", answerTranscript, aiQuestionsPayload.output as { type: "question"; text: string }, aiQuestionsPayload.signalToken!))); + if (racedSignals.filter((response) => response.status === 200).length !== 1 || racedSignals.filter((response) => response.status === 409).length !== 3) { + throw new Error(`concurrent signal token claim was not atomic: ${racedSignals.map((response) => response.status).join(",")}`); + } + if (decisionCalls !== decisionCallsBeforeReplayRace + 1) throw new Error("concurrent Crossfire signal replays reached Jev"); + const answerSignalPayload = await racedSignals.find((response) => response.status === 200)!.json() as { output?: { signal?: { answerTurnId?: string; concession?: string } } }; if (answerSignalPayload.output?.signal?.answerTurnId !== "T2" || answerSignalPayload.output.signal.concession !== "implicit") throw new Error(`ai-questions-user signal missing: ${JSON.stringify(answerSignalPayload)}`); decisionFails = true; - const fallback = await requestSignal("question-ai", questionTranscript, questionAiPayload.output as { type: "answer"; text: string }, questionAiPayload.signalToken); + const fallbackTranscript = [{ id: "T1", sequence: 1, speaker: "user", kind: "question", text: "What is the fallback evidence?", createdAt: 3 }]; + const fallbackReply = await request("question-ai", fallbackTranscript, "answer"); + const fallbackReplyPayload = await fallbackReply.json() as { output?: { type: "answer"; text: string }; signalToken?: string }; + if (!fallbackReplyPayload.output || !fallbackReplyPayload.signalToken) throw new Error("fallback Crossfire reply token missing"); + const fallback = await requestSignal("question-ai", fallbackTranscript, fallbackReplyPayload.output, fallbackReplyPayload.signalToken); const fallbackPayload = await fallback.json() as { output?: { signal?: unknown } }; if (fallback.status !== 200 || fallbackPayload.output?.signal !== undefined) throw new Error(`failed optional signal returned invalid output: ${JSON.stringify(fallbackPayload)}`); } finally { diff --git a/supabase/functions/ai/index.ts b/supabase/functions/ai/index.ts index b5f7284db..7090c0113 100644 --- a/supabase/functions/ai/index.ts +++ b/supabase/functions/ai/index.ts @@ -769,6 +769,18 @@ async function verifyCrossfireSignalToken(userId: string, input: CrossfireSignal return mismatch === 0; } +async function claimCrossfireSignalToken(userId: string, input: CrossfireSignalInput): Promise { + const tokenMac = input.signalToken.split(".")[1]; + const { data, error } = await supabase.rpc("consume_daily_quota", { + p_user_id: userId, + p_limit: 1, + p_quota_key: `crossfire_signal:${tokenMac}`, + }); + if (error) throw error; + const row = Array.isArray(data) ? data[0] : data; + return Boolean(row?.allowed); +} + function boundCrossfireSources(value: unknown, enabled: boolean): { value: PromptSource[]; truncated: boolean } | { error: string } { if (!enabled || value === undefined || value === null) return { value: [], truncated: false }; if (!Array.isArray(value)) return { error: "sources must be an array when Source Library is enabled" }; @@ -1223,6 +1235,15 @@ export async function handleRequest(req: Request): Promise { if (crossfireInput?.operation === "signal") { if (await checkUserBanned(userId)) return badRequest("Your account has been suspended. Please contact support.", corsHeaders, 403); if (!(await verifyCrossfireSignalToken(userId, crossfireInput))) return badRequest("Invalid or expired Crossfire signal token", corsHeaders, 403); + try { + // A token-specific counter atomically bounds replays without consuming the product's AI quota. + if (!(await claimCrossfireSignalToken(userId, crossfireInput))) { + return badRequest("Crossfire signal token has already been used", corsHeaders, 409); + } + } catch (error) { + log(requestId, "ERROR", "Crossfire signal token claim failed", { error: error instanceof Error ? error.message : "Unknown error" }); + return badRequest("Unable to verify Crossfire signal token", corsHeaders, 500); + } const evaluated = await evaluateCrossfireExchange({ transcript: crossfireInput.transcript, reply: crossfireInput.reply,