Skip to content

Commit 3d27bac

Browse files
authored
fix: enforce AI controls and preserve source provenance (#9)
* fix: enforce AI controls and preserve source provenance - Validate AI output and sanitize discovered-source metadata - Persist source quality and activation state across sessions - Report feature costs honestly and expand control regression coverage * chore: keep verification plan out of AI fixes * fix: address CodeRabbit AI flow findings * test: strengthen AI lifecycle verification
1 parent 0b25014 commit 3d27bac

26 files changed

Lines changed: 1579 additions & 268 deletions

‎plans/release-checklist.md‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -17,7 +17,7 @@ Run every step before merging this branch. Nothing here deploys anything; deploy
1717
- [ ] `npm test` — full vitest suite green
1818
- [ ] `npm run build` — production build succeeds
1919
- [ ] `deno check supabase/functions/ai/index.ts supabase/functions/ai-stream/index.ts supabase/functions/admin/index.ts`
20-
- [ ] `deno test --allow-read supabase/functions/` — includes cost calculator, research/quality-floor, prompt-layer, and control-matrix suites
20+
- [ ] `deno test --allow-read --allow-env=EXA_API_KEY --allow-net=127.0.0.1 supabase/functions/` — includes cost calculator, research/quality-floor, prompt-layer, and control-matrix suites
2121

2222
## 2. Migration review (before any deploy)
2323

‎src/lib/ai/client.ts‎

Lines changed: 110 additions & 108 deletions
Original file line numberDiff line numberDiff line change
@@ -1,28 +1,24 @@
11
import { mockGenerateCase, mockAttackCase, mockBrainstorm, mockFactoids, mockAsk } from './providers/mock-provider';
22
import type { ProviderConfig } from './providers/types';
3-
import type { GenerateInput, GenerateOutput } from './schemas/generate';
3+
import { GenerateOutputSchema, type GenerateInput, type GenerateOutput } from './schemas/generate';
44
import { AttackOutputSchema, type AttackOutput, type AttackInput } from './schemas/attack';
55
import type { BrainstormOutput, BrainstormInput } from './schemas/brainstorm';
66
import { FactoidsOutputSchema, type FactoidsOutput, type FactoidsInput } from './schemas/factoids';
77
import { AskOutputSchema, type AskOutput } from './schemas/ask';
88
import { useSettingsStore } from '../stores/settings-store';
99
import { useSessionStore } from '../stores/session-store';
10+
import type { SourceProvenance } from '../stores/session-store';
1011
import { supabase } from '@/lib/supabase/client';
1112
import { toast } from '@/hooks/use-toast';
1213

1314
// Request timeout in milliseconds (200 seconds)
1415
export const REQUEST_TIMEOUT_MS = 200 * 1000;
1516

1617
/** A web source discovered during research, with auditable ranking metadata from the edge function. */
17-
export interface DiscoveredSource {
18+
export interface DiscoveredSource extends Omit<SourceProvenance, 'url'> {
1819
title: string;
1920
url: string;
2021
content?: string;
21-
sourceType?: string;
22-
qualityScore?: number;
23-
qualityLabel?: 'strong' | 'moderate' | 'weak';
24-
/** The research query that surfaced this source. */
25-
query?: string;
2622
}
2723

2824
/** Feature result plus the researched sources that informed it. */
@@ -37,11 +33,38 @@ export interface ResultWithSources<T> {
3733
*/
3834
function parseDiscoveredSources(value: unknown): DiscoveredSource[] {
3935
if (!Array.isArray(value)) return [];
40-
return value.filter((item): item is DiscoveredSource => {
41-
if (!item || typeof item !== 'object') return false;
42-
const candidate = item as Partial<DiscoveredSource>;
43-
return typeof candidate.title === 'string' && candidate.title.length > 0 &&
44-
typeof candidate.url === 'string' && candidate.url.length > 0;
36+
37+
return value.flatMap((item): DiscoveredSource[] => {
38+
if (!item || typeof item !== 'object' || Array.isArray(item)) return [];
39+
40+
const candidate: Record<string, unknown> = {};
41+
for (const [key, fieldValue] of Object.entries(item)) candidate[key] = fieldValue;
42+
43+
const title = typeof candidate.title === 'string' ? candidate.title.trim() : '';
44+
const url = typeof candidate.url === 'string' ? candidate.url.trim() : '';
45+
if (!title || !url) return [];
46+
47+
const source: DiscoveredSource = { title, url };
48+
if (typeof candidate.content === 'string') source.content = candidate.content;
49+
50+
if (typeof candidate.sourceType === 'string' && candidate.sourceType.trim()) {
51+
source.sourceType = candidate.sourceType.trim();
52+
}
53+
54+
if (typeof candidate.qualityScore === 'number' && Number.isFinite(candidate.qualityScore) &&
55+
candidate.qualityScore >= 0 && candidate.qualityScore <= 100) {
56+
source.qualityScore = candidate.qualityScore;
57+
}
58+
59+
if (candidate.qualityLabel === 'strong' || candidate.qualityLabel === 'moderate' || candidate.qualityLabel === 'weak') {
60+
source.qualityLabel = candidate.qualityLabel;
61+
}
62+
63+
if (typeof candidate.query === 'string' && candidate.query.trim()) {
64+
source.query = candidate.query.trim();
65+
}
66+
67+
return [source];
4568
});
4669
}
4770

@@ -718,89 +741,98 @@ function validateAttackOutput(output: unknown): AttackOutput {
718741
return parsed.data;
719742
}
720743

744+
/** Validates the transformed generate response before it can be rendered or persisted alongside sources. */
745+
function validateGenerateOutput(output: unknown): GenerateOutput {
746+
const parsed = GenerateOutputSchema.safeParse(output);
747+
if (!parsed.success) {
748+
throw new AIResponseError('AI response did not match the expected generate schema.', {
749+
details: { issues: parsed.error.issues },
750+
});
751+
}
752+
return parsed.data;
753+
}
754+
755+
type AutosaveFeature = 'generate' | 'attack' | 'brainstorm';
756+
757+
async function autosaveDiscoveredSources(sources: DiscoveredSource[], capturedVia: AutosaveFeature): Promise<void> {
758+
if (sources.length === 0) return;
759+
760+
const session = useSessionStore.getState();
761+
for (const source of sources) {
762+
try {
763+
await session.addSource({
764+
title: source.title,
765+
content: source.content ?? '',
766+
type: 'url',
767+
origin: source.url,
768+
folder: 'Auto-saved',
769+
isActive: source.qualityLabel === 'strong' || source.qualityLabel === 'moderate',
770+
metadata: {
771+
url: source.url,
772+
...(source.sourceType !== undefined ? { sourceType: source.sourceType } : {}),
773+
...(source.qualityScore !== undefined ? { qualityScore: source.qualityScore } : {}),
774+
...(source.qualityLabel !== undefined ? { qualityLabel: source.qualityLabel } : {}),
775+
...(source.query !== undefined ? { query: source.query } : {}),
776+
capturedVia,
777+
},
778+
});
779+
} catch {
780+
toast({
781+
title: 'Source save failed',
782+
description: `Could not save "${source.title}" to your library.`,
783+
variant: 'destructive',
784+
});
785+
}
786+
}
787+
console.log(`[AI Client] Saved ${sources.length} sources to library`);
788+
}
789+
790+
function expectedContentionCount(length: GenerateInput['length']): number {
791+
switch (length) {
792+
case 'standard':
793+
return 3;
794+
case 'comprehensive':
795+
return 4;
796+
case 'brief':
797+
default:
798+
return 2;
799+
}
800+
}
801+
721802
export async function generateCase(input: GenerateInput, config: ProviderConfig): Promise<ResultWithSources<GenerateOutput>> {
722-
if (config.provider === 'mock') return { result: await mockGenerateCase(input.motion, input.side), discoveredSources: [] };
803+
if (config.provider === 'mock') {
804+
return { result: await mockGenerateCase(input.motion, input.side, input.length), discoveredSources: [] };
805+
}
723806
// Inject global language into input for the edge function
724807
const language = useSettingsStore.getState().outputLanguage;
725808
const inputWithLanguage = { ...input, language } as Record<string, unknown>;
726809
const data = await callSupabaseAI('generate', inputWithLanguage, config) as { output: unknown; discoveredSources?: unknown };
727-
const discovered = parseDiscoveredSources(data.discoveredSources);
728-
729-
// Auto-save discovered sources to library (keeping provenance metadata)
730-
if (discovered.length > 0) {
731-
const session = useSessionStore.getState();
732-
for (const source of discovered) {
733-
try {
734-
await session.addSource({
735-
title: source.title,
736-
content: source.content ?? '',
737-
type: 'url',
738-
origin: source.url,
739-
folder: 'Auto-saved',
740-
metadata: {
741-
url: source.url,
742-
sourceType: source.sourceType,
743-
qualityScore: source.qualityScore,
744-
qualityLabel: source.qualityLabel,
745-
query: source.query,
746-
capturedVia: 'generate',
747-
},
748-
});
749-
} catch {
750-
toast({
751-
title: 'Source save failed',
752-
description: `Could not save "${source.title}" to your library.`,
753-
variant: 'destructive',
754-
});
755-
}
756-
}
757-
console.log(`[AI Client] Saved ${discovered.length} sources to library`);
810+
const result = validateGenerateOutput(transformGenerateResponse(data.output as Record<string, unknown>));
811+
const expectedCount = expectedContentionCount(input.length);
812+
if (result.contentions.length !== expectedCount) {
813+
throw new AIResponseError('AI response did not contain the expected number of contentions.', {
814+
details: { expectedCount, actualCount: result.contentions.length },
815+
});
758816
}
759817

818+
const discovered = parseDiscoveredSources(data.discoveredSources);
819+
await autosaveDiscoveredSources(discovered, 'generate');
820+
760821
return {
761-
result: transformGenerateResponse(data.output as Record<string, unknown>),
822+
result,
762823
discoveredSources: discovered,
763824
};
764825
}
765826

766827
export async function attackCase(input: AttackInput, config: ProviderConfig): Promise<ResultWithSources<AttackOutput>> {
767828
if (config.provider === 'mock') return { result: await mockAttackCase(input.caseText), discoveredSources: [] };
768829
const data = await callSupabaseAI('attack', input as Record<string, unknown>, config) as { output: unknown; discoveredSources?: unknown };
830+
const result = validateAttackOutput(data.output);
769831
const discovered = parseDiscoveredSources(data.discoveredSources);
770-
771-
// Auto-save discovered sources to library (keeping provenance metadata)
772-
if (discovered.length > 0) {
773-
const session = useSessionStore.getState();
774-
for (const source of discovered) {
775-
try {
776-
await session.addSource({
777-
title: source.title,
778-
content: source.content ?? '',
779-
type: 'url',
780-
origin: source.url,
781-
folder: 'Auto-saved',
782-
metadata: {
783-
url: source.url,
784-
sourceType: source.sourceType,
785-
qualityScore: source.qualityScore,
786-
qualityLabel: source.qualityLabel,
787-
query: source.query,
788-
capturedVia: 'attack',
789-
},
790-
});
791-
} catch {
792-
toast({
793-
title: 'Source save failed',
794-
description: `Could not save "${source.title}" to your library.`,
795-
variant: 'destructive',
796-
});
797-
}
798-
}
799-
console.log(`[AI Client] Saved ${discovered.length} sources to library`);
800-
}
832+
await autosaveDiscoveredSources(discovered, 'attack');
801833

802834
return {
803-
result: validateAttackOutput(data.output),
835+
result,
804836
discoveredSources: discovered,
805837
};
806838
}
@@ -809,37 +841,7 @@ export async function brainstorm(input: BrainstormInput, config: ProviderConfig)
809841
if (config.provider === 'mock') return { result: await mockBrainstorm(input.motion, input.outputMode), discoveredSources: [] };
810842
const data = await callSupabaseAI('brainstorm', input as Record<string, unknown>, config) as { output: unknown; discoveredSources?: unknown };
811843
const discovered = parseDiscoveredSources(data.discoveredSources);
812-
813-
// Auto-save discovered sources to library (keeping provenance metadata)
814-
if (discovered.length > 0) {
815-
const session = useSessionStore.getState();
816-
for (const source of discovered) {
817-
try {
818-
await session.addSource({
819-
title: source.title,
820-
content: source.content ?? '',
821-
type: 'url',
822-
origin: source.url,
823-
folder: 'Auto-saved',
824-
metadata: {
825-
url: source.url,
826-
sourceType: source.sourceType,
827-
qualityScore: source.qualityScore,
828-
qualityLabel: source.qualityLabel,
829-
query: source.query,
830-
capturedVia: 'brainstorm',
831-
},
832-
});
833-
} catch {
834-
toast({
835-
title: 'Source save failed',
836-
description: `Could not save "${source.title}" to your library.`,
837-
variant: 'destructive',
838-
});
839-
}
840-
}
841-
console.log(`[AI Client] Saved ${discovered.length} sources to library`);
842-
}
844+
await autosaveDiscoveredSources(discovered, 'brainstorm');
843845

844846
return {
845847
result: transformBrainstormResponse(data.output as Record<string, unknown>),

‎src/lib/ai/providers/mock-provider.ts‎

Lines changed: 46 additions & 30 deletions
Original file line numberDiff line numberDiff line change
@@ -1,15 +1,58 @@
11
// src/lib/ai/providers/mock-provider.ts
2-
import type { GenerateOutput } from '../schemas/generate';
2+
import type { GenerateInput, GenerateOutput } from '../schemas/generate';
33
import type { AttackOutput } from '../schemas/attack';
44
import type { BrainstormOutput } from '../schemas/brainstorm';
55
import type { FactoidsOutput } from '../schemas/factoids';
66
import type { AskOutput } from '../schemas/ask';
77

88
const delay = (ms: number) => new Promise(resolve => setTimeout(resolve, ms));
99

10-
export async function mockGenerateCase(motion: string, side: string): Promise<GenerateOutput> {
10+
export async function mockGenerateCase(
11+
motion: string,
12+
side: string,
13+
length: GenerateInput['length'] = 'standard',
14+
): Promise<GenerateOutput> {
1115
await delay(1500);
1216
const actualSide = side === 'auto' ? (Math.random() > 0.5 ? 'pro' : 'con') : side as 'pro' | 'con';
17+
const contentionCount = length === 'brief' ? 2 : length === 'comprehensive' ? 4 : 3;
18+
19+
const contentions: GenerateOutput['contentions'] = [
20+
{
21+
id: '1',
22+
claim: 'Economic Impact Analysis',
23+
reasoning: 'Historical precedent shows that similar policies have led to measurable economic outcomes.',
24+
evidence: 'Studies from comparable jurisdictions demonstrate consistent patterns.',
25+
needsSource: true,
26+
sourceQuery: 'economic impact studies policy implementation',
27+
impact: 'This directly affects the livelihoods of millions and determines policy viability.',
28+
},
29+
{
30+
id: '2',
31+
claim: 'Rights-Based Considerations',
32+
reasoning: 'Fundamental rights must be balanced against collective interests.',
33+
evidence: 'Constitutional frameworks in democratic nations provide guidance.',
34+
needsSource: false,
35+
impact: 'The legitimacy of government action depends on rights protection.',
36+
},
37+
{
38+
id: '3',
39+
claim: 'Practical Implementation',
40+
reasoning: 'Policy effectiveness depends on realistic implementation pathways.',
41+
evidence: 'Case studies from similar implementations inform feasibility.',
42+
needsSource: true,
43+
sourceQuery: 'policy implementation case studies success factors',
44+
impact: 'Without practical viability, the best intentions fail.',
45+
},
46+
{
47+
id: '4',
48+
claim: 'Long-Term Social Effects',
49+
reasoning: 'The policy should be assessed against its effects on social cohesion over time.',
50+
evidence: 'Longitudinal comparisons reveal how similar policies affect communities.',
51+
needsSource: true,
52+
sourceQuery: 'long-term social effects policy outcomes',
53+
impact: 'Durable social outcomes determine whether the policy succeeds beyond its first effects.',
54+
},
55+
];
1356

1457
return {
1558
caseStructure: {
@@ -21,34 +64,7 @@ export async function mockGenerateCase(motion: string, side: string): Promise<Ge
2164
],
2265
burdenOfProof: `The ${actualSide === 'pro' ? 'proposition' : 'opposition'} must demonstrate that the ${actualSide === 'pro' ? 'benefits outweigh costs' : 'status quo is preferable'}.`,
2366
},
24-
contentions: [
25-
{
26-
id: '1',
27-
claim: 'Economic Impact Analysis',
28-
reasoning: 'Historical precedent shows that similar policies have led to measurable economic outcomes.',
29-
evidence: 'Studies from comparable jurisdictions demonstrate consistent patterns.',
30-
needsSource: true,
31-
sourceQuery: 'economic impact studies policy implementation',
32-
impact: 'This directly affects the livelihoods of millions and determines policy viability.',
33-
},
34-
{
35-
id: '2',
36-
claim: 'Rights-Based Considerations',
37-
reasoning: 'Fundamental rights must be balanced against collective interests.',
38-
evidence: 'Constitutional frameworks in democratic nations provide guidance.',
39-
needsSource: false,
40-
impact: 'The legitimacy of government action depends on rights protection.',
41-
},
42-
{
43-
id: '3',
44-
claim: 'Practical Implementation',
45-
reasoning: 'Policy effectiveness depends on realistic implementation pathways.',
46-
evidence: 'Case studies from similar implementations inform feasibility.',
47-
needsSource: true,
48-
sourceQuery: 'policy implementation case studies success factors',
49-
impact: 'Without practical viability, the best intentions fail.',
50-
},
51-
],
67+
contentions: contentions.slice(0, contentionCount),
5268
predictedOpponentCase: {
5369
likelyArguments: [
5470
'The opposing side will likely argue about unintended consequences',

0 commit comments

Comments
 (0)