diff --git a/e2e/search-quality.spec.ts b/e2e/search-quality.spec.ts new file mode 100644 index 000000000..ba833cf53 --- /dev/null +++ b/e2e/search-quality.spec.ts @@ -0,0 +1,90 @@ +import { expect, test } from '@playwright/test'; +import { setLocale, stubSupabaseSession } from './helpers/auth'; + +const strongSourceUrl = 'https://energy.gov/reports/solar-costs-2024'; +const weakSourceUrl = 'https://example.com/broad-commentary'; + +test('authenticated Factoids shows the consulted-source quality panel', async ({ page }) => { + await setLocale(page, 'en'); + await stubSupabaseSession(page); + + await page.route('**/functions/v1/ai', async (route) => { + await route.fulfill({ + status: 200, + contentType: 'application/json', + body: JSON.stringify({ + output: { + factoids: [{ + id: 'fact-1', + shortClaim: 'Solar costs fell by 42 percent', + keyFigure: '42%', + context: 'The official report compares the relevant period.', + limitations: 'The result depends on the measured market segment.', + citation: { + title: 'Official energy statistics', + publisher: 'Department of Energy', + date: '2024', + url: strongSourceUrl, + verified: 'verified', + }, + howToUse: 'Use this as a bounded cost trend.', + relevanceScore: 92, + }], + searchSuggestions: [], + warnings: [], + verificationStatus: 'complete', + }, + discoveredSources: [{ + title: 'Official energy statistics', + url: strongSourceUrl, + content: 'Official government statistics.', + sourceType: 'government', + qualityScore: 91, + qualityLabel: 'strong', + query: 'solar energy costs government evidence', + }, { + title: 'Broad commentary', + url: weakSourceUrl, + content: 'General commentary.', + sourceType: 'generic', + qualityScore: 30, + qualityLabel: 'weak', + query: 'solar energy costs commentary', + }], + }), + }); + }); + + await page.route('**/rest/v1/source_folders*', async (route) => { + await route.fulfill({ status: 201, contentType: 'application/json', body: '{}' }); + }); + await page.route('**/rest/v1/sources*', async (route) => { + await route.fulfill({ + status: 201, + contentType: 'application/json', + body: JSON.stringify({ + id: 'source-1', + title: 'Consulted source', + content: 'Evidence', + type: 'url', + origin: strongSourceUrl, + folder: 'Auto-saved', + metadata: {}, + is_active: true, + created_at: new Date().toISOString(), + }), + }); + }); + + await page.goto('/app/factoids'); + await page.getByRole('textbox', { name: /statement/i }).fill('Solar energy costs are falling'); + await page.getByRole('button', { name: /find factoids/i }).click(); + + await expect(page.getByRole('heading', { name: /researched sources/i })).toBeVisible(); + await expect(page.getByRole('link', { name: /official energy statistics/i }).first()).toHaveAttribute('href', strongSourceUrl); + await expect(page.getByRole('link', { name: /broad commentary/i })).toHaveAttribute('href', weakSourceUrl); + await expect(page.getByText('strong', { exact: true })).toBeVisible(); + await expect(page.getByText('weak', { exact: true })).toBeVisible(); + await expect(page.getByText('fallback context only', { exact: true })).toBeVisible(); + await expect(page.getByText('Solar costs fell by 42 percent')).toBeVisible(); +}); diff --git a/plans/search-quality.md b/plans/search-quality.md index e11af3e59..90cf0f395 100644 --- a/plans/search-quality.md +++ b/plans/search-quality.md @@ -43,7 +43,7 @@ Implemented: 1. Research pipeline extracted to `supabase/functions/_shared/research.ts` (Exa search, follow-ups, ranking context, citation sources) with an injected model caller so it is testable; ranking/query builders moved to `_shared/source-quality.ts`. 2. Topic-specific query builders (tournament, policy/statistics, legal/current events) were already present and are now covered by tests in the shared module. 3. All discovered-source returns carry quality metadata; Factoids now also returns `discoveredSources`. -4. Source display: shared `SourceQualityList` renders strong/moderate/weak labels plus "fallback context only" for weak sources in Generate, Attack, and Brainstorm results. Factoids shows only verified citations (quality-labeled consulted-list display deferred). +4. Source display: shared `SourceQualityList` renders strong/moderate/weak labels plus "fallback context only" for weak sources in Generate, Attack, Brainstorm, and Factoids results. Factoids still limits evidence cards to verified citations while listing every consulted source separately. 5. Provenance persists to saved sources: `sources.metadata` jsonb (migration `20260822000001`, prepared but NOT applied) stores url, sourceType, qualityScore, qualityLabel, originating query, capturedVia. 6. Source-quality floor completed: factoid citations verify only against fetched non-weak research content (`applyFactoidsVerification`); weak or unknown URLs are downgraded and filtered. Ask fallback already excluded weak sources. @@ -51,6 +51,8 @@ Tests: `_shared/research.test.ts`, extended `source-quality.test.ts`, frontend ` Deferred: -- Replace inline `site:gov` query text with Exa's dedicated domain-filter request field (needs a live Exa parameter-verification pass). -- Handler-level integration test that mocks Exa end-to-end through `Deno.serve`; current coverage is at the extracted-function level plus frontend mocks. -- Factoids UI panel listing all consulted sources with quality chips. +- None. The remaining search-quality scope was completed on `t3code/search-quality-completion`: + - Exa search requests use the documented camelCase fields (`includeDomains`, `excludeDomains`, `category`, `startPublishedDate`, `endPublishedDate`, `numResults`, and `contents.text.maxCharacters`) instead of inline `site:` operators. Domain/category filters retry without the restrictive filter when they return no results. + - The exported Edge handler has a Deno boundary test that mocks authentication, quota, Exa search/content, and the model response, then verifies citation quality and consulted-source provenance in the HTTP response. + - Factoids returns `discoveredSources`, autosaves their provenance, and renders the shared consulted-source quality list, including the weak-source fallback warning. + - Authenticated Playwright coverage exercises the Factoids flow on isolated desktop and mobile Chromium projects. diff --git a/src/components/features/factoids/FactoidsResults.tsx b/src/components/features/factoids/FactoidsResults.tsx index ff622a5a9..8e2bce65e 100644 --- a/src/components/features/factoids/FactoidsResults.tsx +++ b/src/components/features/factoids/FactoidsResults.tsx @@ -16,10 +16,13 @@ import { toast } from 'sonner'; import { ExportToolbar } from '@/components/features/ExportToolbar'; import { GoogleDriveAuthModal } from '@/components/features/export/GoogleDriveAuthModal'; import { openExternalUrl } from '@/lib/native'; +import { SourceQualityList } from '@/components/features/shared/SourceQualityList'; +import type { DiscoveredSource } from '@/lib/ai/client'; interface FactoidsResultsProps { result: FactoidsOutput; + discoveredSources?: DiscoveredSource[]; onDiscuss?: () => void; onSaveSource?: (title: string, content: string, url?: string) => void; onUseSuggestion?: (query: string) => void; @@ -137,7 +140,7 @@ function FactoidCard({ factoid, onSave }: { factoid: Factoid, onSave?: FactoidsR ); } -export function FactoidsResults({ result, onDiscuss, onSaveSource, onUseSuggestion }: FactoidsResultsProps) { +export function FactoidsResults({ result, discoveredSources, onDiscuss, onSaveSource, onUseSuggestion }: FactoidsResultsProps) { const { t } = useTranslation(); const [isExportingGDocs, setIsExportingGDocs] = useState(false); const [showDriveAuth, setShowDriveAuth] = useState(false); @@ -195,6 +198,21 @@ export function FactoidsResults({ result, onDiscuss, onSaveSource, onUseSuggesti discussLabel={t('factoidsResults.discuss')} /> + {discoveredSources && discoveredSources.length > 0 && ( +
+

+ {t('sourceQuality.researchedSources')} +

+ +

+ {t('sourceQuality.fallbackNote')} +

+
+ )} + {/* Verification Status */} { if (sources.length === 0) return; @@ -1122,8 +1124,8 @@ export async function brainstorm(input: BrainstormInput, config: ProviderConfig) return response; } -export async function factoids(input: FactoidsInput, config: ProviderConfig): Promise { - if (config.provider === 'mock') return mockFactoids(input.claim); +export async function factoids(input: FactoidsInput, config: ProviderConfig): Promise> { + if (config.provider === 'mock') return { result: await mockFactoids(input.claim), discoveredSources: [] }; // Inject global language into input for the edge function const language = useSettingsStore.getState().outputLanguage; const inputWithLanguage = { @@ -1132,9 +1134,9 @@ export async function factoids(input: FactoidsInput, config: ProviderConfig): Pr sourceCount: input.sourceCount || 5, language, } as Record; - return await callSupabaseAI('factoids', inputWithLanguage, config, { + const response = await callSupabaseAI>('factoids', inputWithLanguage, config, { transformResponse: (rawData) => { - const data = rawData as { output: unknown }; + const data = rawData as { output: unknown; discoveredSources?: unknown }; const parsed = FactoidsOutputSchema.safeParse(data.output); if (!parsed.success) { throw new AIResponseError('AI response did not match the expected factoids schema.', { @@ -1146,12 +1148,17 @@ export async function factoids(input: FactoidsInput, config: ProviderConfig): Pr const verifiedFactoids = allFactoids.filter((factoid) => factoid.citation.verified === 'verified'); const returnedFactoids = verifiedFactoids.slice(0, input.sourceCount || 5); return { - ...parsed.data, - factoids: returnedFactoids, - filteredOutCount: Math.max(0, allFactoids.length - returnedFactoids.length), + result: { + ...parsed.data, + factoids: returnedFactoids, + filteredOutCount: Math.max(0, allFactoids.length - returnedFactoids.length), + }, + discoveredSources: parseDiscoveredSources(data.discoveredSources), }; }, }); + await autosaveDiscoveredSources(response.discoveredSources, 'factoids'); + return response; } export interface AskChatOptions { diff --git a/src/pages/FactoidsPage.tsx b/src/pages/FactoidsPage.tsx index 320bb4cf9..dfe200d1c 100644 --- a/src/pages/FactoidsPage.tsx +++ b/src/pages/FactoidsPage.tsx @@ -1,10 +1,11 @@ +import { useState } from 'react'; import { useNavigate } from 'react-router-dom'; import { useTranslation } from 'react-i18next'; import { useSessionStore } from '@/lib/stores/session-store'; import { useSettingsStore } from '@/lib/stores/settings-store'; import { FactoidsForm } from '@/components/features/factoids/FactoidsForm'; import { FactoidsResults } from '@/components/features/factoids/FactoidsResults'; -import { factoids, askChat } from '@/lib/ai/client'; +import { factoids, askChat, type DiscoveredSource } from '@/lib/ai/client'; import { useSubscriptionError } from '@/hooks/use-subscription-error'; import { getUserFacingAIError } from '@/lib/ai/error-display'; import { LimitReachedModal, ModelRestrictedModal, PricingModal } from '@/components/features/subscription'; @@ -22,11 +23,14 @@ export default function FactoidsPage() { const navigate = useNavigate(); const subError = useSubscriptionError(); const { t } = useTranslation(); + const [discoveredSources, setDiscoveredSources] = useState([]); const handleSubmit = async (data: FactoidsInput) => { + setDiscoveredSources([]); setFactoidsLoading(true); try { - const result = await factoids(data, config); + const { result, discoveredSources: sources } = await factoids(data, config); + setDiscoveredSources(sources); setFactoidsResult(result); toast({ title: t('factoids.complete', 'Research complete'), description: t('factoids.completeDesc', 'Verified evidence is ready.'), variant: 'success' }); } catch (error) { @@ -130,6 +134,7 @@ export default function FactoidsPage() {
{ title: 'Source A', content: 'A content', type: 'url', - origin: 'https://a.com', + origin: 'https://a.com/', folder: 'Auto-saved', isActive: true, - metadata: expect.objectContaining({ capturedVia: 'generate', url: 'https://a.com' }), + metadata: expect.objectContaining({ capturedVia: 'generate', url: 'https://a.com/' }), }); }); @@ -311,7 +311,9 @@ describe('AI client branch coverage', () => { qualityLabel: 'unknown', query: 42, }, + { title: 'Protocol-less source', url: 'example.com/source-two' }, { title: '', url: 'https://example.com/rejected' }, + { title: 'Unsafe protocol', url: 'httpx://example.com/rejected' }, ], }, error: null, @@ -321,6 +323,7 @@ describe('AI client branch coverage', () => { expect(outcome.discoveredSources).toEqual([ { title: 'Source with malformed metadata', url: 'https://example.com/source' }, + { title: 'Protocol-less source', url: 'https://example.com/source-two' }, ]); expect(addSourceMock).toHaveBeenCalledWith(expect.objectContaining({ title: 'Source with malformed metadata', @@ -427,10 +430,10 @@ describe('AI client branch coverage', () => { title: 'Source C', content: 'C content', type: 'url', - origin: 'https://c.com', + origin: 'https://c.com/', folder: 'Auto-saved', isActive: true, - metadata: expect.objectContaining({ capturedVia: 'brainstorm', url: 'https://c.com' }), + metadata: expect.objectContaining({ capturedVia: 'brainstorm', url: 'https://c.com/' }), }); }); @@ -501,8 +504,49 @@ describe('AI client branch coverage', () => { const result = await factoids({ claim: 'Claim', direction: 'for', sourceType: 'any', sourceCount: 3 }, config); - expect(result.factoids).toHaveLength(3); - expect(result.filteredOutCount).toBe(1); + expect(result.result.factoids).toHaveLength(3); + expect(result.result.filteredOutCount).toBe(1); + }); + + it('returns and autosaves discovered sources for factoids responses', async () => { + invokeMock.mockResolvedValueOnce({ + data: { + output: { + factoids: [], + searchSuggestions: [], + warnings: [], + verificationStatus: 'complete', + }, + discoveredSources: [{ + title: 'Factoids source', + url: 'https://example.com/factoids', + content: 'Evidence content', + sourceType: 'government', + qualityLabel: 'moderate', + qualityScore: 70, + query: 'factoids query', + }], + }, + error: null, + }); + + const outcome = await factoids({ claim: 'Claim', direction: 'for', sourceType: 'any', sourceCount: 3 }, config); + + expect(outcome.discoveredSources).toEqual([{ + title: 'Factoids source', + url: 'https://example.com/factoids', + content: 'Evidence content', + sourceType: 'government', + qualityLabel: 'moderate', + qualityScore: 70, + query: 'factoids query', + }]); + expect(addSourceMock).toHaveBeenCalledWith(expect.objectContaining({ + title: 'Factoids source', + origin: 'https://example.com/factoids', + isActive: true, + metadata: expect.objectContaining({ capturedVia: 'factoids', qualityLabel: 'moderate' }), + })); }); it('maps invoke errors with 401 status to AIAuthError', async () => { @@ -846,7 +890,7 @@ describe('AI client branch coverage', () => { expect(attack.overallAssessment.score).toBeTypeOf('number'); expect(ideas.nextSteps).toBeInstanceOf(Array); - expect(facts.factoids).toBeInstanceOf(Array); + expect(facts.result.factoids).toBeInstanceOf(Array); expect(chat.response).toContain('hello'); expect(invokeMock).not.toHaveBeenCalled(); }, 15000); diff --git a/src/test/components/FactoidsResults.test.tsx b/src/test/components/FactoidsResults.test.tsx index da05479a5..70fb97a22 100644 --- a/src/test/components/FactoidsResults.test.tsx +++ b/src/test/components/FactoidsResults.test.tsx @@ -2,6 +2,7 @@ import { render, screen } from '@testing-library/react'; import { vi } from 'vitest'; import { FactoidsResults } from '@/components/features/factoids/FactoidsResults'; import type { FactoidsOutput } from '@/lib/ai/schemas/factoids'; +import type { DiscoveredSource } from '@/lib/ai/client'; vi.mock('@/components/features/ExportToolbar', () => ({ ExportToolbar: (): null => null, @@ -35,3 +36,18 @@ test('does not expose source verification while no verification behavior exists' expect(screen.queryByRole('button', { name: /verify all sources/i })).not.toBeInTheDocument(); }); + +test('renders consulted sources with a fallback label for weak sources', () => { + const discoveredSources: DiscoveredSource[] = [{ + title: 'Broad fallback article', + url: 'https://example.com/fallback', + qualityLabel: 'weak', + qualityScore: 20, + }]; + + render(); + + expect(screen.getByRole('heading', { name: /researched sources/i })).toBeInTheDocument(); + expect(screen.getByRole('link', { name: /broad fallback article/i })).toHaveAttribute('href', discoveredSources[0].url); + expect(screen.getByText('fallback context only')).toBeInTheDocument(); +}); diff --git a/src/test/pages/ConversationPages.test.tsx b/src/test/pages/ConversationPages.test.tsx index 2eb43c047..3611c71c7 100644 --- a/src/test/pages/ConversationPages.test.tsx +++ b/src/test/pages/ConversationPages.test.tsx @@ -2,6 +2,7 @@ import { beforeEach, describe, expect, it, vi } from 'vitest'; import { act, fireEvent, render, screen, waitFor } from '@testing-library/react'; import { BrowserRouter } from 'react-router-dom'; +import type { FactoidsOutput } from '@/lib/ai/schemas/factoids'; vi.mock('@/hooks/use-subscription', () => ({ useSubscription: () => ({ isPro: false, tier: 'trial', loading: false }), @@ -140,8 +141,9 @@ vi.mock('@/components/features/factoids/FactoidsForm', () => ({ })); vi.mock('@/components/features/factoids/FactoidsResults', () => ({ - FactoidsResults: ({ onDiscuss, onSaveSource }: { onDiscuss: () => void, onSaveSource: (t: string, c: string, u?: string) => void }) => ( + FactoidsResults: ({ onDiscuss, onSaveSource, discoveredSources }: { onDiscuss: () => void, onSaveSource: (t: string, c: string, u?: string) => void, discoveredSources?: unknown[] }) => (
+ {discoveredSources?.length ?? 0}