From de6d375d78a9d193a9ba50e8d8b1a678b77a71ec Mon Sep 17 00:00:00 2001 From: Adam Date: Mon, 31 Aug 2026 15:54:28 +0000 Subject: [PATCH 1/4] feat: complete search quality plan --- e2e/search-quality.spec.ts | 90 +++++++++ plans/search-quality.md | 8 +- .../features/factoids/FactoidsResults.tsx | 20 +- src/lib/ai/client.ts | 20 +- src/pages/FactoidsPage.tsx | 9 +- src/test/ai/client-branches.test.ts | 47 ++++- src/test/components/FactoidsResults.test.tsx | 16 ++ src/test/pages/ConversationPages.test.tsx | 32 +++- supabase/functions/_shared/research.test.ts | 105 +++++++++++ supabase/functions/_shared/research.ts | 36 +++- supabase/functions/_shared/source-quality.ts | 176 ++++++++++++++++-- supabase/functions/ai/control-matrix.test.ts | 8 +- .../functions/ai/handler-research.test.ts | 162 ++++++++++++++++ supabase/functions/ai/index.ts | 8 +- supabase/functions/ai/source-quality.test.ts | 38 +++- vitest.config.ts | 6 +- 16 files changed, 724 insertions(+), 57 deletions(-) create mode 100644 e2e/search-quality.spec.ts create mode 100644 supabase/functions/ai/handler-research.test.ts diff --git a/e2e/search-quality.spec.ts b/e2e/search-quality.spec.ts new file mode 100644 index 000000000..ba833cf53 --- /dev/null +++ b/e2e/search-quality.spec.ts @@ -0,0 +1,90 @@ +import { expect, test } from '@playwright/test'; +import { setLocale, stubSupabaseSession } from './helpers/auth'; + +const strongSourceUrl = 'https://energy.gov/reports/solar-costs-2024'; +const weakSourceUrl = 'https://example.com/broad-commentary'; + +test('authenticated Factoids shows the consulted-source quality panel', async ({ page }) => { + await setLocale(page, 'en'); + await stubSupabaseSession(page); + + await page.route('**/functions/v1/ai', async (route) => { + await route.fulfill({ + status: 200, + contentType: 'application/json', + body: JSON.stringify({ + output: { + factoids: [{ + id: 'fact-1', + shortClaim: 'Solar costs fell by 42 percent', + keyFigure: '42%', + context: 'The official report compares the relevant period.', + limitations: 'The result depends on the measured market segment.', + citation: { + title: 'Official energy statistics', + publisher: 'Department of Energy', + date: '2024', + url: strongSourceUrl, + verified: 'verified', + }, + howToUse: 'Use this as a bounded cost trend.', + relevanceScore: 92, + }], + searchSuggestions: [], + warnings: [], + verificationStatus: 'complete', + }, + discoveredSources: [{ + title: 'Official energy statistics', + url: strongSourceUrl, + content: 'Official government statistics.', + sourceType: 'government', + qualityScore: 91, + qualityLabel: 'strong', + query: 'solar energy costs government evidence', + }, { + title: 'Broad commentary', + url: weakSourceUrl, + content: 'General commentary.', + sourceType: 'generic', + qualityScore: 30, + qualityLabel: 'weak', + query: 'solar energy costs commentary', + }], + }), + }); + }); + + await page.route('**/rest/v1/source_folders*', async (route) => { + await route.fulfill({ status: 201, contentType: 'application/json', body: '{}' }); + }); + await page.route('**/rest/v1/sources*', async (route) => { + await route.fulfill({ + status: 201, + contentType: 'application/json', + body: JSON.stringify({ + id: 'source-1', + title: 'Consulted source', + content: 'Evidence', + type: 'url', + origin: strongSourceUrl, + folder: 'Auto-saved', + metadata: {}, + is_active: true, + created_at: new Date().toISOString(), + }), + }); + }); + + await page.goto('/app/factoids'); + await page.getByRole('textbox', { name: /statement/i }).fill('Solar energy costs are falling'); + await page.getByRole('button', { name: /find factoids/i }).click(); + + await expect(page.getByRole('heading', { name: /researched sources/i })).toBeVisible(); + await expect(page.getByRole('link', { name: /official energy statistics/i }).first()).toHaveAttribute('href', strongSourceUrl); + await expect(page.getByRole('link', { name: /broad commentary/i })).toHaveAttribute('href', weakSourceUrl); + await expect(page.getByText('strong', { exact: true })).toBeVisible(); + await expect(page.getByText('weak', { exact: true })).toBeVisible(); + await expect(page.getByText('fallback context only', { exact: true })).toBeVisible(); + await expect(page.getByText('Solar costs fell by 42 percent')).toBeVisible(); +}); diff --git a/plans/search-quality.md b/plans/search-quality.md index e11af3e59..6331e9083 100644 --- a/plans/search-quality.md +++ b/plans/search-quality.md @@ -51,6 +51,8 @@ Tests: `_shared/research.test.ts`, extended `source-quality.test.ts`, frontend ` Deferred: -- Replace inline `site:gov` query text with Exa's dedicated domain-filter request field (needs a live Exa parameter-verification pass). -- Handler-level integration test that mocks Exa end-to-end through `Deno.serve`; current coverage is at the extracted-function level plus frontend mocks. -- Factoids UI panel listing all consulted sources with quality chips. +- None. The remaining search-quality scope was completed on `t3code/search-quality-completion`: + - Exa search requests use the documented camelCase fields (`includeDomains`, `excludeDomains`, `category`, `startPublishedDate`, `endPublishedDate`, `numResults`, and `contents.text.maxCharacters`) instead of inline `site:` operators. Domain/category filters retry without the restrictive filter when they return no results. + - The exported Edge handler has a Deno boundary test that mocks authentication, quota, Exa search/content, and the model response, then verifies citation quality and consulted-source provenance in the HTTP response. + - Factoids returns `discoveredSources`, autosaves their provenance, and renders the shared consulted-source quality list, including the weak-source fallback warning. + - Authenticated Playwright coverage exercises the Factoids flow on isolated desktop and mobile Chromium projects. diff --git a/src/components/features/factoids/FactoidsResults.tsx b/src/components/features/factoids/FactoidsResults.tsx index ff622a5a9..8e2bce65e 100644 --- a/src/components/features/factoids/FactoidsResults.tsx +++ b/src/components/features/factoids/FactoidsResults.tsx @@ -16,10 +16,13 @@ import { toast } from 'sonner'; import { ExportToolbar } from '@/components/features/ExportToolbar'; import { GoogleDriveAuthModal } from '@/components/features/export/GoogleDriveAuthModal'; import { openExternalUrl } from '@/lib/native'; +import { SourceQualityList } from '@/components/features/shared/SourceQualityList'; +import type { DiscoveredSource } from '@/lib/ai/client'; interface FactoidsResultsProps { result: FactoidsOutput; + discoveredSources?: DiscoveredSource[]; onDiscuss?: () => void; onSaveSource?: (title: string, content: string, url?: string) => void; onUseSuggestion?: (query: string) => void; @@ -137,7 +140,7 @@ function FactoidCard({ factoid, onSave }: { factoid: Factoid, onSave?: FactoidsR ); } -export function FactoidsResults({ result, onDiscuss, onSaveSource, onUseSuggestion }: FactoidsResultsProps) { +export function FactoidsResults({ result, discoveredSources, onDiscuss, onSaveSource, onUseSuggestion }: FactoidsResultsProps) { const { t } = useTranslation(); const [isExportingGDocs, setIsExportingGDocs] = useState(false); const [showDriveAuth, setShowDriveAuth] = useState(false); @@ -195,6 +198,21 @@ export function FactoidsResults({ result, onDiscuss, onSaveSource, onUseSuggesti discussLabel={t('factoidsResults.discuss')} /> + {discoveredSources && discoveredSources.length > 0 && ( +
+

+ {t('sourceQuality.researchedSources')} +

+ +

+ {t('sourceQuality.fallbackNote')} +

+
+ )} + {/* Verification Status */} { if (sources.length === 0) return; @@ -1021,8 +1021,8 @@ export async function brainstorm(input: BrainstormInput, config: ProviderConfig) }; } -export async function factoids(input: FactoidsInput, config: ProviderConfig): Promise { - if (config.provider === 'mock') return mockFactoids(input.claim); +export async function factoids(input: FactoidsInput, config: ProviderConfig): Promise> { + if (config.provider === 'mock') return { result: await mockFactoids(input.claim), discoveredSources: [] }; // Inject global language into input for the edge function const language = useSettingsStore.getState().outputLanguage; const inputWithLanguage = { @@ -1031,7 +1031,7 @@ export async function factoids(input: FactoidsInput, config: ProviderConfig): Pr sourceCount: input.sourceCount || 5, language, } as Record; - const data = await callSupabaseAI('factoids', inputWithLanguage, config) as { output: unknown }; + const data = await callSupabaseAI('factoids', inputWithLanguage, config) as { output: unknown; discoveredSources?: unknown }; const parsed = FactoidsOutputSchema.safeParse(data.output); if (!parsed.success) { throw new AIResponseError('AI response did not match the expected factoids schema.', { @@ -1042,10 +1042,16 @@ export async function factoids(input: FactoidsInput, config: ProviderConfig): Pr const allFactoids = parsed.data.factoids; const verifiedFactoids = allFactoids.filter((factoid) => factoid.citation.verified === 'verified'); const returnedFactoids = verifiedFactoids.slice(0, input.sourceCount || 5); + const discovered = parseDiscoveredSources(data.discoveredSources); + await autosaveDiscoveredSources(discovered, 'factoids'); + return { - ...parsed.data, - factoids: returnedFactoids, - filteredOutCount: Math.max(0, allFactoids.length - returnedFactoids.length), + result: { + ...parsed.data, + factoids: returnedFactoids, + filteredOutCount: Math.max(0, allFactoids.length - returnedFactoids.length), + }, + discoveredSources: discovered, }; } diff --git a/src/pages/FactoidsPage.tsx b/src/pages/FactoidsPage.tsx index dd5408816..2b2d214e4 100644 --- a/src/pages/FactoidsPage.tsx +++ b/src/pages/FactoidsPage.tsx @@ -1,10 +1,11 @@ +import { useState } from 'react'; import { useNavigate } from 'react-router-dom'; import { useTranslation } from 'react-i18next'; import { useSessionStore } from '@/lib/stores/session-store'; import { useSettingsStore } from '@/lib/stores/settings-store'; import { FactoidsForm } from '@/components/features/factoids/FactoidsForm'; import { FactoidsResults } from '@/components/features/factoids/FactoidsResults'; -import { factoids, askChat } from '@/lib/ai/client'; +import { factoids, askChat, type DiscoveredSource } from '@/lib/ai/client'; import { useSubscriptionError } from '@/hooks/use-subscription-error'; import { getUserFacingAIError } from '@/lib/ai/error-display'; import { LimitReachedModal, ModelRestrictedModal, PricingModal } from '@/components/features/subscription'; @@ -22,11 +23,14 @@ export default function FactoidsPage() { const navigate = useNavigate(); const subError = useSubscriptionError(); const { t } = useTranslation(); + const [discoveredSources, setDiscoveredSources] = useState([]); const handleSubmit = async (data: FactoidsInput) => { + setDiscoveredSources([]); setFactoidsLoading(true); try { - const result = await factoids(data, config); + const { result, discoveredSources: sources } = await factoids(data, config); + setDiscoveredSources(sources); setFactoidsResult(result); toast({ title: t('factoids.complete', 'Research complete'), description: t('factoids.completeDesc', 'Verified evidence is ready.'), variant: 'success' }); } catch (error) { @@ -130,6 +134,7 @@ export default function FactoidsPage() {
{ const result = await factoids({ claim: 'Claim', direction: 'for', sourceType: 'any', sourceCount: 3 }, config); - expect(result.factoids).toHaveLength(3); - expect(result.filteredOutCount).toBe(1); + expect(result.result.factoids).toHaveLength(3); + expect(result.result.filteredOutCount).toBe(1); + }); + + it('returns and autosaves discovered sources for factoids responses', async () => { + invokeMock.mockResolvedValueOnce({ + data: { + output: { + factoids: [], + searchSuggestions: [], + warnings: [], + verificationStatus: 'complete', + }, + discoveredSources: [{ + title: 'Factoids source', + url: 'https://example.com/factoids', + content: 'Evidence content', + sourceType: 'government', + qualityLabel: 'moderate', + qualityScore: 70, + query: 'factoids query', + }], + }, + error: null, + }); + + const outcome = await factoids({ claim: 'Claim', direction: 'for', sourceType: 'any', sourceCount: 3 }, config); + + expect(outcome.discoveredSources).toEqual([{ + title: 'Factoids source', + url: 'https://example.com/factoids', + content: 'Evidence content', + sourceType: 'government', + qualityLabel: 'moderate', + qualityScore: 70, + query: 'factoids query', + }]); + expect(addSourceMock).toHaveBeenCalledWith(expect.objectContaining({ + title: 'Factoids source', + origin: 'https://example.com/factoids', + isActive: true, + metadata: expect.objectContaining({ capturedVia: 'factoids', qualityLabel: 'moderate' }), + })); }); it('maps invoke errors with 401 status to AIAuthError', async () => { @@ -846,7 +887,7 @@ describe('AI client branch coverage', () => { expect(attack.overallAssessment.score).toBeTypeOf('number'); expect(ideas.nextSteps).toBeInstanceOf(Array); - expect(facts.factoids).toBeInstanceOf(Array); + expect(facts.result.factoids).toBeInstanceOf(Array); expect(chat.response).toContain('hello'); expect(invokeMock).not.toHaveBeenCalled(); }, 15000); diff --git a/src/test/components/FactoidsResults.test.tsx b/src/test/components/FactoidsResults.test.tsx index da05479a5..70fb97a22 100644 --- a/src/test/components/FactoidsResults.test.tsx +++ b/src/test/components/FactoidsResults.test.tsx @@ -2,6 +2,7 @@ import { render, screen } from '@testing-library/react'; import { vi } from 'vitest'; import { FactoidsResults } from '@/components/features/factoids/FactoidsResults'; import type { FactoidsOutput } from '@/lib/ai/schemas/factoids'; +import type { DiscoveredSource } from '@/lib/ai/client'; vi.mock('@/components/features/ExportToolbar', () => ({ ExportToolbar: (): null => null, @@ -35,3 +36,18 @@ test('does not expose source verification while no verification behavior exists' expect(screen.queryByRole('button', { name: /verify all sources/i })).not.toBeInTheDocument(); }); + +test('renders consulted sources with a fallback label for weak sources', () => { + const discoveredSources: DiscoveredSource[] = [{ + title: 'Broad fallback article', + url: 'https://example.com/fallback', + qualityLabel: 'weak', + qualityScore: 20, + }]; + + render(); + + expect(screen.getByRole('heading', { name: /researched sources/i })).toBeInTheDocument(); + expect(screen.getByRole('link', { name: /broad fallback article/i })).toHaveAttribute('href', discoveredSources[0].url); + expect(screen.getByText('fallback context only')).toBeInTheDocument(); +}); diff --git a/src/test/pages/ConversationPages.test.tsx b/src/test/pages/ConversationPages.test.tsx index 2eb43c047..3611c71c7 100644 --- a/src/test/pages/ConversationPages.test.tsx +++ b/src/test/pages/ConversationPages.test.tsx @@ -2,6 +2,7 @@ import { beforeEach, describe, expect, it, vi } from 'vitest'; import { act, fireEvent, render, screen, waitFor } from '@testing-library/react'; import { BrowserRouter } from 'react-router-dom'; +import type { FactoidsOutput } from '@/lib/ai/schemas/factoids'; vi.mock('@/hooks/use-subscription', () => ({ useSubscription: () => ({ isPro: false, tier: 'trial', loading: false }), @@ -140,8 +141,9 @@ vi.mock('@/components/features/factoids/FactoidsForm', () => ({ })); vi.mock('@/components/features/factoids/FactoidsResults', () => ({ - FactoidsResults: ({ onDiscuss, onSaveSource }: { onDiscuss: () => void, onSaveSource: (t: string, c: string, u?: string) => void }) => ( + FactoidsResults: ({ onDiscuss, onSaveSource, discoveredSources }: { onDiscuss: () => void, onSaveSource: (t: string, c: string, u?: string) => void, discoveredSources?: unknown[] }) => (
+ {discoveredSources?.length ?? 0}