research.ts

// Shared research pipeline for AI features: deterministic Exa search,
// AI follow-up queries, source ranking, quality floors, and provenance.
// Model calls are injected via ResearchDeps so this module stays testable
// and free of provider-specific code.

import {
  buildFollowUpSearchRequest,
  buildSearchRequests,
  formatRankedSearchResults,
  nationalSourceHint,
  rankSearchResults,
  sourceTitleFromUrl,
  type ExaResult,
  type ExaSearchRequest,
  type RankedSearchResult,
  type ResearchLocale,
  type ResearchSource,
} from "./source-quality.ts";

export type { ExaResult, RankedSearchResult, ResearchSource };

import type { ReasoningEffort } from "./model-catalog.ts";
import { normalizeExaUsage, type NormalizedUsage } from "./cost.ts";
import { MAX_FETCHED_PAGE_CHARS, MAX_RESEARCH_SNIPPETS } from "./prompt-budget.ts";

export type Provider = "openrouter";

// Local browser gates point this at a fake provider; production leaves it unset.
export const OPENROUTER_API_URL = Deno.env.get("OPENROUTER_API_URL") || "https://openrouter.ai/api";
export const MAX_PROVIDER_RESPONSE_BYTES = 1_048_576;
export const MAX_PROVIDER_ERROR_BODY_BYTES = 65_536;
export const MAX_PROVIDER_ERROR_BODY_CHARS = 16_000;

export type Source = {
  title: string;
  content: string;
  origin?: string;
  type: string;
};

export interface ResearchContext {
  searches: Array<{ query: string; results: RankedSearchResult[] }>;
  rankedResults: RankedSearchResult[];
  readContent: Array<{ url: string; title: string; content: string; result?: RankedSearchResult }>;
  verifiedUrls: Set<string>;
}

export interface ResearchOptions {
  maxFollowUpQueries: number; // 0 = skip AI follow-up entirely
  maxUrlsToRead: number;      // How many pages to read in full
  maxSeedQueries: number;     // How many deterministic searches before AI follow-up
  overallTimeoutMs: number;   // Hard timeout for entire research
  preferredSourceType?: string;
  timeframe?: string;
  /** Output language code: a Hungarian or Spanish request prefers that country's sources. */
  language?: string;
  /** ISO country code whose sources to prefer; derived from language when absent. */
  country?: string;
}

export const DEEP_RESEARCH: ResearchOptions = {
  maxFollowUpQueries: 2,
  maxUrlsToRead: 5,
  maxSeedQueries: 4,
  overallTimeoutMs: 35000,
};

export const ASK_RESEARCH: ResearchOptions = {
  maxFollowUpQueries: 2,
  maxUrlsToRead: 4,
  maxSeedQueries: 4,
  overallTimeoutMs: 30000,
};

export interface ResearchDeps {
  provider: Provider;
  model: string;
  reasoningEffort: ReasoningEffort;
  /** Request-scoped cancellation signal, if the caller has one. */
  signal?: AbortSignal;
  /** Called synchronously when the overall research timeout fires. */
  onTimeout?: () => void;
  /** Receives the charge of every paid Exa call made for the request. */
  onExaUsage?: ExaUsageSink;
  /** Injected model caller; returns the parsed JSON output of a provider call. */
  callModel: (args: {
    provider: Provider;
    model: string;
    systemPrompt: string;
    messages: Array<{ role: string; content: string }>;
    reasoningEffort: ReasoningEffort;
    signal?: AbortSignal;
    outputSchema?: JsonOutputSchema;
  }) => Promise<{ output: unknown }>;
}

/**
 * A JSON Schema the provider must hold the reply to. Only for outputs the
 * server already validates in code: with `strict` every property is required,
 * so the schema has to match that validator exactly. Keep to keywords every
 * routed provider accepts: type, properties, required, items, enum, anyOf,
 * minItems, maxItems, and additionalProperties false.
 */
export interface JsonOutputSchema {
  name: string;
  schema: Record<string, unknown>;
}

const QUERIES_SCHEMA: JsonOutputSchema = {
  name: "follow_up_queries",
  schema: {
    type: "object",
    properties: { queries: { type: "array", items: { type: "string" } } },
    required: ["queries"],
    additionalProperties: false,
  },
};

const SEARCH_DECISION_SCHEMA: JsonOutputSchema = {
  name: "search_decision",
  schema: {
    type: "object",
    properties: { shouldSearch: { type: "boolean" }, query: { type: "string" } },
    required: ["shouldSearch", "query"],
    additionalProperties: false,
  },
};

export function emptyResearchContext(): ResearchContext {
  return { searches: [], rankedResults: [], readContent: [], verifiedUrls: new Set() };
}

function throwIfAborted(signal?: AbortSignal): void {
  if (signal?.aborted) {
    throw signal.reason ?? new DOMException("The operation was aborted", "AbortError");
  }
}

function linkAbortSignals(signals: Array<AbortSignal | null | undefined>) {
  const controller = new AbortController();
  const cleanup: Array<() => void> = [];
  const abort = (reason?: unknown) => {
    if (!controller.signal.aborted) controller.abort(reason);
  };

  for (const signal of signals) {
    if (!signal) continue;
    if (signal.aborted) {
      abort(signal.reason);
      break;
    }
    const onAbort = () => abort(signal.reason);
    signal.addEventListener("abort", onAbort, { once: true });
    cleanup.push(() => signal.removeEventListener("abort", onAbort));
  }

  return {
    signal: controller.signal,
    abort,
    cleanup: () => cleanup.forEach((remove) => remove()),
  };
}

export async function fetchWithTimeoutAndConsume<T>(
  url: string,
  options: RequestInit,
  timeoutMs: number,
  signal: AbortSignal | undefined,
  consume: (response: Response, signal: AbortSignal) => Promise<T>,
): Promise<T> {
  const controller = new AbortController();
  const linked = linkAbortSignals([options.signal, signal]);
  const onLinkedAbort = () => controller.abort(linked.signal.reason);
  if (linked.signal.aborted) onLinkedAbort();
  else linked.signal.addEventListener("abort", onLinkedAbort, { once: true });
  const timeoutId = setTimeout(() => controller.abort(), timeoutMs);
  try {
    const response = await fetch(url, { ...options, signal: controller.signal });
    return await consume(response, controller.signal);
  } finally {
    clearTimeout(timeoutId);
    linked.signal.removeEventListener("abort", onLinkedAbort);
    linked.cleanup();
  }
}

async function readResponseBytes(
  response: Response,
  maxBytes: number,
  signal: AbortSignal,
  truncate: boolean,
): Promise<{ bytes: Uint8Array; truncated: boolean }> {
  if (!response.body) {
    throw new Error("Provider response has no body");
  }
  const reader = response.body.getReader();
  const chunks: Uint8Array[] = [];
  let totalBytes = 0;
  try {
    while (true) {
      if (signal.aborted) throw signal.reason ?? new DOMException("The operation was aborted", "AbortError");
      const { done, value } = await reader.read();
      if (done) break;
      const remaining = maxBytes - totalBytes;
      if (value.byteLength > remaining) {
        if (truncate && remaining > 0) chunks.push(value.slice(0, remaining));
        await reader.cancel(new Error("Response body exceeded its byte limit"));
        if (!truncate) throw new Error(`Provider response exceeds ${maxBytes} bytes`);
        totalBytes = maxBytes;
        return { bytes: joinBytes(chunks, totalBytes), truncated: true };
      }
      chunks.push(value);
      totalBytes += value.byteLength;
    }
    return { bytes: joinBytes(chunks, totalBytes), truncated: false };
  } finally {
    reader.releaseLock();
  }
}

function joinBytes(chunks: Uint8Array[], totalBytes: number): Uint8Array {
  const bytes = new Uint8Array(totalBytes);
  let offset = 0;
  for (const chunk of chunks) {
    bytes.set(chunk, offset);
    offset += chunk.byteLength;
  }
  return bytes;
}

export async function readResponseText(
  response: Response,
  maxBytes: number,
  maxChars: number,
  signal: AbortSignal,
  truncate: boolean,
): Promise<{ text: string; truncated: boolean }> {
  if (response.body) {
    const result = await readResponseBytes(response, maxBytes, signal, truncate);
    const text = new TextDecoder().decode(result.bytes);
    if (text.length > maxChars) return { text: text.slice(0, maxChars), truncated: true };
    return { text, truncated: result.truncated };
  }

  // A few injected test doubles expose json/text without a ReadableStream.
  // Keep those calls signal-aware through the fetch wrapper and apply the same
  // post-read bound before returning the parsed result.
  if (typeof response.text === "function") {
    const text = await response.text();
    if (new TextEncoder().encode(text).byteLength > maxBytes && !truncate) {
      throw new Error(`Provider response exceeds ${maxBytes} bytes`);
    }
    return { text: text.slice(0, maxChars), truncated: text.length > maxChars || new TextEncoder().encode(text).byteLength > maxBytes };
  }
  throw new Error("Provider response body cannot be consumed");
}

export async function fetchJsonWithTimeout<T = unknown>(
  url: string,
  options: RequestInit,
  timeoutMs: number,
  signal?: AbortSignal,
): Promise<{ response: Response; data?: T; errorBody?: string }> {
  return fetchWithTimeoutAndConsume(url, options, timeoutMs, signal, async (response, bodySignal) => {
    if (!response.ok) {
      const body = await readResponseText(
        response,
        MAX_PROVIDER_ERROR_BODY_BYTES,
        MAX_PROVIDER_ERROR_BODY_CHARS,
        bodySignal,
        true,
      );
      return { response, errorBody: body.text };
    }

    if (response.body) {
      const body = await readResponseText(
        response,
        MAX_PROVIDER_RESPONSE_BYTES,
        MAX_PROVIDER_RESPONSE_BYTES,
        bodySignal,
        false,
      );
      return { response, data: JSON.parse(body.text) as T };
    }

    if (typeof response.json !== "function") throw new Error("Provider response body cannot be parsed as JSON");
    const data = await response.json() as T;
    let serialized: string;
    try {
      serialized = JSON.stringify(data);
    } catch {
      throw new Error("Provider response could not be serialized for size validation");
    }
    if (new TextEncoder().encode(serialized).byteLength > MAX_PROVIDER_RESPONSE_BYTES) {
      throw new Error(`Provider response exceeds ${MAX_PROVIDER_RESPONSE_BYTES} bytes`);
    }
    return { response, data };
  });
}

// ============== EXA FUNCTIONS ==============

export type ExaUsageSink = (usage: NormalizedUsage) => void;

// Reports each sent Exa call once. A successful response was billed at the
// charge it reports; an error status was not. A call that was sent but never
// settled (cancelled, timed out, unreadable body) may still have been billed,
// so it makes the request's cost unknown instead of quietly lowering it.
function exaUsageReporter(endpoint: "search" | "contents", onUsage: ExaUsageSink | undefined) {
  let sent = false;
  let reported = false;
  const report = (usage: NormalizedUsage) => {
    if (reported || !onUsage) return;
    reported = true;
    onUsage(usage);
  };
  return {
    sent: () => { sent = true; },
    response: (ok: boolean, data: unknown) => {
      if (!ok) {
        reported = true;
        return;
      }
      report(normalizeExaUsage(endpoint, data));
    },
    unsettled: () => {
      if (sent) report({ incomplete: true, raw: { provider: "exa", endpoint, unsettled: true } });
    },
  };
}

// exaSearch: web search using Exa's neural search (type "auto" = balanced relevance & speed)
export async function exaSearch(
  search: string | ExaSearchRequest,
  signal?: AbortSignal,
  onUsage?: ExaUsageSink,
): Promise<ExaResult[]> {
  const request = typeof search === "string" ? { query: search } : search;
  const query = request.query;
  const apiKey = Deno.env.get("EXA_API_KEY");
  if (!apiKey) {
    console.warn("[EXA] EXA_API_KEY not configured");
    return [];
  }
  const usage = exaUsageReporter("search", onUsage);
  try {
    throwIfAborted(signal);
    usage.sent();
    const { response, data } = await fetchJsonWithTimeout<{ results?: unknown }>("https://api.exa.ai/search", {
      method: "POST",
      headers: { "Content-Type": "application/json", "x-api-key": apiKey },
      body: JSON.stringify({
        ...request,
        type: "auto",
        numResults: 10,
        contents: { text: { maxCharacters: 10000 } },
      }),
    }, 30000, signal);
    usage.response(response.ok, data);
    throwIfAborted(signal);
    if (!response.ok) {
      console.warn(`[EXA] Search failed (${response.status}) queryLength=${query.length}`);
      return [];
    }
    if (!data?.results || !Array.isArray(data.results)) return [];
    return data.results.map((r: { title?: string; url?: string; text?: string; publishedDate?: string }) => ({
      title: r.title,
      url: r.url,
      content: r.text,
      publishedDate: r.publishedDate,
    }));
  } catch (err) {
    usage.unsettled();
    if (signal?.aborted) throw err;
    console.warn(`[EXA] Search failed queryLength=${query.length}`);
    return [];
  }
}

// exaGetContents: extract a page's text, up to maxChars, using Exa's /contents endpoint
export async function exaGetContents(
  url: string,
  signal?: AbortSignal,
  onUsage?: ExaUsageSink,
  maxChars = MAX_FETCHED_PAGE_CHARS,
): Promise<string> {
  const apiKey = Deno.env.get("EXA_API_KEY");
  if (!apiKey) {
    console.warn("[EXA] EXA_API_KEY not configured");
    return "";
  }
  const usage = exaUsageReporter("contents", onUsage);
  try {
    throwIfAborted(signal);
    usage.sent();
    const { response, data } = await fetchJsonWithTimeout<{ results?: unknown }>("https://api.exa.ai/contents", {
      method: "POST",
      headers: { "Content-Type": "application/json", "x-api-key": apiKey },
      body: JSON.stringify({ urls: [url], text: { maxCharacters: maxChars } }),
    }, 30000, signal);
    usage.response(response.ok, data);
    throwIfAborted(signal);
    if (!response.ok) {
      console.warn(`[EXA] Contents fetch failed (${response.status}) urlLength=${url.length}`);
      return "";
    }
    if (!data?.results || !Array.isArray(data.results) || data.results.length === 0) return "";
    return (data.results[0]?.text || "").slice(0, maxChars);
  } catch (err) {
    usage.unsettled();
    if (signal?.aborted) throw err;
    console.warn(`[EXA] Contents fetch failed urlLength=${url.length}`);
    return "";
  }
}

export function verifyUrl(url: string): boolean {
  // Do not fetch model-provided citation URLs from the Edge Function. A URL
  // that is syntactically public can still resolve to a private service after
  // validation, and the deployed runtime cannot safely pin DNS resolution.
  // Citations remain visible but are explicitly marked unverified.
  void url;
  return false;
}

// ============== DEEP RESEARCH ==============

const QUERY_GENERATION_PROMPT = `You are a research assistant. Based on the initial claim and search results so far, generate 2-3 follow-up search queries to find more specific evidence.

Return JSON: { "queries": ["query1", "query2", "query3"] }

Keep queries specific, factual, and focused on statistics, studies, or official sources.`;

const ASK_SEARCH_DECISION_PROMPT = `You decide whether a debate coaching answer needs live web research.

Return strict JSON:
{
  "shouldSearch": boolean,
  "query": "string"
}

Set shouldSearch=true when the user asks for current events, recent stats, real-world examples, laws/policies, or anything time-sensitive.
Set shouldSearch=false for timeless coaching (strategy, structure, delivery, framing).
When shouldSearch=true, provide one concise search query.
Do not add any fields.`;

function likelyNeedsFreshWebData(text: string): boolean {
  return /\b(latest|today|current|recent|new|newest|update|updated|news|202[0-9]|law|policy|gdp|inflation|ceo|president|prime minister|market|stock|price|war|conflict)\b/i.test(text);
}

export async function decideAskSearch(
  message: string,
  history: Array<{ role: string; content: string }>,
  deps: ResearchDeps
): Promise<{ shouldSearch: boolean; query: string }> {
  throwIfAborted(deps.signal);
  const context = history.slice(-4).map((m) => `${m.role}: ${m.content}`).join("\n");
  const prompt = `${context ? `Recent conversation:\n${context}\n\n` : ""}User message:\n${message}`;

  try {
    const decisionResult = await deps.callModel({
      provider: deps.provider,
      model: deps.model,
      systemPrompt: ASK_SEARCH_DECISION_PROMPT,
      messages: [{ role: "user", content: prompt }],
      reasoningEffort: deps.reasoningEffort,
      signal: deps.signal,
      outputSchema: SEARCH_DECISION_SCHEMA,
    });
    const decision = decisionResult.output;
    if (decision && typeof decision === "object") {
      const parsed = decision as { shouldSearch?: unknown; query?: unknown };
      const shouldSearch = Boolean(parsed.shouldSearch);
      const query = typeof parsed.query === "string" ? parsed.query.trim().slice(0, 240) : "";
      if (shouldSearch) {
        return { shouldSearch: true, query: query || message.slice(0, 240) };
      }
      return { shouldSearch: false, query: "" };
    }
  } catch (err) {
    if (deps.signal?.aborted) throw err;
    console.warn("[ASK] Search decision failed:", err);
  }

  const fallbackShouldSearch = likelyNeedsFreshWebData(message);
  return {
    shouldSearch: fallbackShouldSearch,
    query: fallbackShouldSearch ? message.slice(0, 240) : "",
  };
}

async function generateFollowUpQueries(
  claim: string,
  currentResults: string,
  deps: ResearchDeps,
  locale?: ResearchLocale,
): Promise<string[]> {
  throwIfAborted(deps.signal);
  const hint = nationalSourceHint(locale);
  const prompt = `Claim: "${claim}"\n\nCurrent search results:\n${currentResults.slice(0, 3000)}\n\nGenerate 2-3 specific follow-up queries to find more evidence.${hint ? ` ${hint}` : ""}`;
  try {
    const result = await deps.callModel({
      provider: deps.provider,
      model: deps.model,
      systemPrompt: QUERY_GENERATION_PROMPT,
      messages: [{ role: "user", content: prompt }],
      reasoningEffort: deps.reasoningEffort,
      signal: deps.signal,
      outputSchema: QUERIES_SCHEMA,
    });
    const parsed = typeof result.output === "object" && result.output !== null ? result.output : {};
    if (Array.isArray((parsed as { queries?: string[] }).queries)) {
      return (parsed as { queries: string[] }).queries.slice(0, 3);
    }
    return [];
  } catch (err) {
    if (deps.signal?.aborted) throw err;
    console.warn("[DEEP RESEARCH] Failed to generate follow-up queries:", err);
    return [];
  }
}

async function deepResearchInner(
  claim: string,
  region: string,
  deps: ResearchDeps,
  options: ResearchOptions,
  context: ResearchContext = emptyResearchContext(),
  signal?: AbortSignal
): Promise<ResearchContext> {
  throwIfAborted(signal);
  console.log(`[RESEARCH] Starting (mode: ${options.maxFollowUpQueries > 0 ? "deep" : "light"}) claimLength=${claim.length}`);

  const preferredSourceType = options.preferredSourceType || "any";
  const locale: ResearchLocale = { language: options.language, country: options.country };
  const seedRequests = buildSearchRequests(claim, region, preferredSourceType, options.timeframe, locale).slice(0, options.maxSeedQueries);
  const allResults: Array<ExaResult & { query: string }> = [];
  const matchesPreferredSourceType = (result: RankedSearchResult) => (
    preferredSourceType === "any" ||
    result.sourceType === preferredSourceType
  );

  const searchWithFallback = async (request: ExaSearchRequest) => {
    const filtered = await exaSearch(request, signal, deps.onExaUsage);
    if (filtered.length > 0 || (!request.includeDomains && !request.category)) return filtered;
    const {
      includeDomains: _includeDomains,
      category: _category,
      ...fallbackRequest
    } = request;
    return exaSearch(fallbackRequest, signal, deps.onExaUsage);
  };

  const seedResults = await Promise.all(seedRequests.map(async (request) => {
    const { query } = request;
    const results = await searchWithFallback(request);
    throwIfAborted(signal);
    const ranked = rankSearchResults(results.map((result) => ({ ...result, query })), claim, preferredSourceType, locale);
    const visibleRanked = preferredSourceType === "any" ? ranked : ranked.filter(matchesPreferredSourceType);
    context.searches.push({ query, results: visibleRanked });
    console.log(`[RESEARCH] Seed queryLength=${query.length}: ${results.length} results`);
    return results.map((result) => ({ ...result, query }));
  }));
  seedResults.forEach((results) => allResults.push(...results));

  const initialRanked = rankSearchResults(allResults, claim, preferredSourceType, locale);
  const initialText = formatRankedSearchResults(initialRanked.slice(0, 12));

  // Round 2: AI generates follow-up queries (skipped in light mode)
  if (options.maxFollowUpQueries > 0) {
    const followUpQueries = await generateFollowUpQueries(claim, initialText, deps, locale);
    throwIfAborted(signal);
    console.log(`[RESEARCH] Follow-up query count: ${followUpQueries.length}`);

    const followUpPromises = followUpQueries.slice(0, options.maxFollowUpQueries).map(async (query) => {
      const results = await searchWithFallback(buildFollowUpSearchRequest(query, preferredSourceType, options.timeframe));
      throwIfAborted(signal);
      const ranked = rankSearchResults(results.map((result) => ({ ...result, query })), claim, preferredSourceType, locale);
      const visibleRanked = preferredSourceType === "any" ? ranked : ranked.filter(matchesPreferredSourceType);
      context.searches.push({ query, results: visibleRanked });
      allResults.push(...results.map((result) => ({ ...result, query })));
      return results.length;
    });
    await Promise.all(followUpPromises);
    console.log(`[RESEARCH] Round 2 complete, ${context.searches.length} total searches`);
  }

  context.rankedResults = rankSearchResults(allResults, claim, preferredSourceType, locale);
  const readCandidates = preferredSourceType === "any"
    ? context.rankedResults
    : context.rankedResults.filter(matchesPreferredSourceType);
  const nonWeakCount = context.rankedResults.filter((r) => r.qualityLabel !== "weak").length;
  const urlsToRead = readCandidates
    .filter((result) => result.qualityLabel !== "weak" || nonWeakCount < 2)
    .slice(0, options.maxUrlsToRead);

  const readPromises = urlsToRead.map(async (result) => {
    const url = result.url || "";
    const content = await exaGetContents(url, signal, deps.onExaUsage);
    throwIfAborted(signal);
    if (content) {
      context.readContent.push({ url, title: result.title || sourceTitleFromUrl(url), content, result });
      context.verifiedUrls.add(url);
    }
    return content.length > 0;
  });
  await Promise.all(readPromises);
  console.log(`[RESEARCH] Read ${context.readContent.length} pages`);

  return context;
}

// Wrapper with overall timeout - returns whatever completed if the limit hits,
// so a slow search round does not throw away finished rounds.
export async function deepResearch(
  claim: string,
  region: string,
  deps: ResearchDeps,
  options: ResearchOptions = DEEP_RESEARCH
): Promise<ResearchContext> {
  const partial = emptyResearchContext();
  const linked = linkAbortSignals([deps.signal]);
  let timeoutId: ReturnType<typeof setTimeout> | undefined;
  const work = deepResearchInner(
    claim,
    region,
    { ...deps, signal: linked.signal },
    options,
    partial,
    linked.signal,
  );
  try {
    const result = await Promise.race([
      work,
      new Promise<never>((_, reject) => {
        timeoutId = setTimeout(() => {
          try {
            deps.onTimeout?.();
          } catch (callbackError) {
            console.warn("[RESEARCH] Timeout accounting callback failed:", callbackError);
          }
          linked.abort(new Error("Research timeout"));
          reject(new Error("Research timeout"));
        }, options.overallTimeoutMs);
      }),
    ]);
    return result;
  } catch (err) {
    console.warn(`[RESEARCH] Timed out or failed after ${options.overallTimeoutMs}ms:`, err);
    // Return promptly after aborting. Production providers honor the signal;
    // any non-cooperative late work is detached and must not affect usage.
    return {
      searches: [...partial.searches],
      rankedResults: [...partial.rankedResults],
      readContent: [...partial.readContent],
      verifiedUrls: new Set(partial.verifiedUrls),
    };
  } finally {
    if (timeoutId !== undefined) clearTimeout(timeoutId);
    linked.cleanup();
  }
}

export function buildResearchContext(research: ResearchContext): string {
  if (research.rankedResults.length === 0 && research.readContent.length === 0) return "";

  let ctx = "=== WEB RESEARCH RESULTS ===\n\n";
  ctx += "IMPORTANT: Only cite sources marked 'strong' or 'moderate'. Sources marked 'weak' are fallback context only, do NOT cite them as evidence. If only weak sources are available, state that evidence is limited.\n\n";

  // Each source appears once. A page read in full is not repeated as a search
  // snippet, and a URL several queries found is listed a single time.
  const listed = new Set(research.readContent.map((page) => page.url));
  const snippets = research.searches
    .flatMap((search) => search.results)
    .sort((a, b) => b.qualityScore - a.qualityScore)
    .filter((result) => {
      if (!result.url || listed.has(result.url)) return false;
      listed.add(result.url);
      return true;
    })
    .slice(0, MAX_RESEARCH_SNIPPETS);

  if (research.searches.length > 0) {
    ctx += `Searches run: ${research.searches.map((search) => `"${search.query}"`).join("; ")}\n\n`;
  }
  if (snippets.length > 0) {
    ctx += `--- Search results ---\n${formatRankedSearchResults(snippets)}\n\n`;
  }

  // Add full page content
  if (research.readContent.length > 0) {
    ctx += "\n=== FULL PAGE CONTENT (verified sources) ===\n\n";
    for (const page of research.readContent) {
      const quality = page.result ? `${page.result.sourceType}, ${page.result.qualityLabel}, ${page.result.qualityScore}/100` : "verified";
      const warning = page.result?.qualityLabel === "weak" ? " [WEAK, do not cite as evidence]" : "";
      ctx += `--- From: ${page.title} (${page.url}) ---\nSource quality: ${quality}${warning}\n${page.content.slice(0, MAX_FETCHED_PAGE_CHARS)}\n\n`;
    }
  }

  return ctx;
}

export function getResearchSources(research: ResearchContext): ResearchSource[] {
  const seen = new Set<string>();
  const sources: ResearchSource[] = [];

  for (const page of research.readContent) {
    if (!page.url || seen.has(page.url)) continue;
    if (page.result?.qualityLabel === "weak") continue; // Quality floor: weak sources can't be cited
    if (!page.result) continue; // Fail closed: unranked pages stay fallback context
    seen.add(page.url);
    const result = page.result;
    sources.push({
      title: page.title || result?.title || sourceTitleFromUrl(page.url),
      url: page.url,
      verified: research.verifiedUrls.has(page.url),
      sourceType: result?.sourceType ?? "generic",
      qualityScore: result?.qualityScore ?? 50,
      qualityLabel: result?.qualityLabel ?? "moderate",
    });
  }

  if (sources.length > 0) return sources.slice(0, 5);

  // Fallback: if no strong/moderate read content, use ranked results (excluding weak)
  for (const result of research.rankedResults) {
    const url = result.url || "";
    if (!url || seen.has(url)) continue;
    if (result.qualityLabel === "weak") continue;
    seen.add(url);
    sources.push({
      title: result.title || sourceTitleFromUrl(url),
      url,
      verified: false,
      sourceType: result.sourceType,
      qualityScore: result.qualityScore,
      qualityLabel: result.qualityLabel,
    });
    if (sources.length >= 5) return sources;
  }

  return sources;
}

export function withResearchPreferences(base: ResearchOptions, preferences: Partial<ResearchOptions>): ResearchOptions {
  return { ...base, ...preferences };
}

export function buildDiscoveredSources(research: ResearchContext) {
  return research.readContent.map((item) => ({
    title: item.title || sourceTitleFromUrl(item.url),
    url: item.url,
    content: item.content.slice(0, 1000),
    qualityScore: item.result?.qualityScore,
    sourceType: item.result?.sourceType,
    qualityLabel: item.result?.qualityLabel,
    // Provenance: which research query surfaced this source.
    query: item.result?.query,
  }));
}

/**
 * Source-quality floor for factoids: a citation may only be marked "verified"
 * when its URL matches research content that was actually fetched AND that
 * content carries an explicit strong or moderate label. Anything else,
 * including fetched pages without ranking metadata, fails closed to
 * unverified. Weak sources may appear as fallback context, never as evidence.
 */
export function applyFactoidsVerification(
  output: unknown,
  research: ResearchContext
): void {
  if (!output || typeof output !== "object" || !("factoids" in output)) return;
  const factoids = (output as { factoids: Array<{ citation?: { url?: string; verified?: string } }> }).factoids;
  if (!Array.isArray(factoids)) return;

  for (const factoid of factoids) {
    if (!factoid || typeof factoid !== "object" || Array.isArray(factoid)) continue;
    const citation = (factoid as { citation?: unknown }).citation;
    if (!citation || typeof citation !== "object" || Array.isArray(citation)) continue;
    const typedCitation = citation as { url?: unknown; verified?: string };
    const url = typeof typedCitation.url === "string" ? typedCitation.url : "";
    const fetched = url && research.verifiedUrls.has(url)
      ? research.readContent.find((page) => page.url === url)
      : undefined;
    const quality = fetched?.result?.qualityLabel;
    typedCitation.verified = quality === "strong" || quality === "moderate" ? "verified" : "unverified";
  }
}