diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 774f495..f753ada 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -53,6 +53,9 @@ jobs: print('all entry points import cleanly') " + - name: Network-free test suite + run: uv run pytest -q + # Exercises extractors, slicers, prompt templates and the programmatic # checkers end to end. --dry-run makes no API calls, so this needs no key # and costs nothing. diff --git a/.gitignore b/.gitignore index 4f3a6fb..60f54c4 100644 --- a/.gitignore +++ b/.gitignore @@ -212,5 +212,27 @@ output/ # CI smoke-test output ci-output/ +# Private model-evaluation inputs and derived artifacts +/Pristine Accessibility Defect Report.xlsx +/.model-evaluation/ +/vision_aid/evaluation/human_finding_mappings/ + +# Node.js / Next.js generated files +node_modules/ +.next/ +next-env.d.ts +*.tsbuildinfo +web/prisma/generated/ + # Stray git worktrees — never belong in the index .claude/worktrees/ + +# Local-only issue tracker and detailed evaluator tests +/.scratch/ +/.local-tests/ + +# Local architecture decision records +/docs/adr/ + +# Local agent context +CONTEXT.md diff --git a/.vscode/settings.json b/.vscode/settings.json new file mode 100644 index 0000000..1e26040 --- /dev/null +++ b/.vscode/settings.json @@ -0,0 +1,3 @@ +{ + "python.terminal.useEnvFile": true +} \ No newline at end of file diff --git a/AGENTS.md b/AGENTS.md new file mode 100644 index 0000000..4d00ed3 --- /dev/null +++ b/AGENTS.md @@ -0,0 +1,13 @@ +## Agent skills + +### Issue tracker + +Issues are tracked as local Markdown files under `.scratch/`. See `docs/agents/issue-tracker.md`. + +### Triage labels + +The default five-role triage vocabulary is used. See `docs/agents/triage-labels.md`. + +### Domain docs + +This repository uses a single-context domain-doc layout. See `docs/agents/domain.md`. diff --git a/CLAUDE.md b/CLAUDE.md index 220212a..5a9d342 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -5,6 +5,7 @@ This project builds LLM-powered tools that analyze websites for WCAG accessibility issues and generate structured remediation reports. It is a Computing for Good course project at Georgia Tech (OMSCS) partnered with the Vision Aid Digital Accessibility Testing Team. The system has three planned stages: + 1. Automated webpage accessibility analysis → structured CSV reports 2. Code remediation suggestions based on those reports 3. (Stretch) Chrome extension for on-the-fly accessibility fixes @@ -116,3 +117,17 @@ Docker image → GHCR (`ghcr.io/c4g/va-dat`) → Coolify at `https://va-dat.c4g. - HTML test files can be enormous (500K+ tokens) — do not read them fully into context. - Some files may have non-UTF-8 encoding; always use `errors='replace'`. - Model defaults differ by entry point: `run_pipeline.py` uses `claude-sonnet-5`, `api_server.py` uses `claude-haiku-4-5-20251001`. + +## Agent skills + +### Issue tracker + +Issues are tracked as local Markdown files under `.scratch/`. See `docs/agents/issue-tracker.md`. + +### Triage labels + +The default five-role triage vocabulary is used. See `docs/agents/triage-labels.md`. + +### Domain docs + +This repository uses a single-context domain-doc layout. See `docs/agents/domain.md`. diff --git a/README.md b/README.md index 909446b..eac1383 100644 --- a/README.md +++ b/README.md @@ -57,10 +57,9 @@ HTML file (e.g. 1.9 MB) │ │ ├── forms_checklist_02.txt # 6 prompts for form accessibility │ │ └── nontext_checklist_03.txt # 8 prompts for non-text content │ │ -│ ├── llm_client/ # Standalone Claude API client (Andrew) -│ │ ├── client.py # API wrapper -│ │ ├── prompt_loader.py # Prompt loading utilities -│ │ └── runner.py # End-to-end audit runner +│ ├── llm_client/ # Provider-independent audit requests +│ │ ├── audit.py # API requests and response metadata +│ │ └── client.py # Model-provider capability helpers │ │ │ ├── llm_preprocessing/ # HTML → structured JSON extractors │ │ ├── semantic_checklist_01.py # Headings, links, landmarks, tables, iframes @@ -74,9 +73,29 @@ HTML file (e.g. 1.9 MB) │ │ │ └── docs/pipeline.md # Pipeline architecture documentation │ -├── vision_aid/ingestion/ -│ ├── file_crawler.py # fetch_page / fetch_pages_nested (used by the server) -│ └── pull_html.py # Standalone HTML download helper +├── vision_aid/ +│ ├── evaluation/ # Evaluation workflows, review, and shared schemas +│ │ ├── README.md # Detailed evaluator guide and command reference +│ │ ├── cli.py # CLI commands and operator interaction +│ │ ├── full.py # Full evaluation across all audited pages +│ │ ├── page.py # Shared page preparation and review validation +│ │ ├── review.py # Review CSV validation and detection counts +│ │ ├── schemas.py # Shared Human audit and finding types +│ │ ├── inputs/ +│ │ │ └── human_audit.py # Import Human findings and audited URLs +│ │ ├── execution/ +│ │ │ ├── llm_audit.py # LLM requests, saved responses, and cost guardrails +│ │ │ ├── normalization.py # Convert tool responses into findings +│ │ │ ├── pricing.py # Load price schedules and estimate request cost +│ │ │ └── pricing.json # Versioned public model rates +│ │ └── reporting/ +│ │ ├── html_report.py # Prepare report data and render standalone HTML +│ │ ├── templates/ # Report HTML templates +│ │ ├── css/ # Report styles +│ │ └── js/ # Report filters and printing +│ └── ingestion/ +│ ├── file_crawler.py # fetch_page / fetch_pages_nested (used by the server) +│ └── pull_html.py # Standalone HTML download helper │ ├── Dockerfile # Multi-stage uv build → runtime image ├── docker-compose.yml # Local run (Coolify deploys the image directly) @@ -100,7 +119,7 @@ HTML file (e.g. 1.9 MB) ├── docs/ # Architecture documentation │ └── modular-prompts-plan.md # Full architectural plan │ -├── reports/ # Raw JSON output from standalone llm_client runs +├── reports/ # Historical raw JSON audit output │ ├── test_results/ │ ├── chatgpt/ # Legacy ChatGPT testing results @@ -276,10 +295,16 @@ everything it does is free: - `uv.lock` is in sync with `pyproject.toml`, and `requirements.txt` matches the lock - entry points import +- the tracked synthetic pytest suite (`uv run pytest -q`), with no private data or provider calls - a full pipeline dry run, asserting prompts generated, findings found, and zero tokens consumed - `index.html`'s inline JavaScript parses - the Docker image builds, becomes healthy, serves the site, and returns a valid NDJSON audit +Default pytest discovery is limited to `tests/`, where `tests/test_evaluation.py` +covers the evaluator end to end through its CLI with a fake provider. Optional +detailed evaluator cases live in the ignored `.local-tests/evaluation/` folder; +run them with `uv run pytest -q .local-tests/evaluation`. CI never needs them. + ## Running the Pipeline ### Dry run (no API calls, no cost) @@ -384,3 +409,143 @@ The pipeline skips prompts with empty payloads (e.g., no forms on the page = no | Andrew Yin | CL02 + CL03 extractors, CL02 + CL03 prompts, LLM client, pipeline docs | `processing_scripts/llm_preprocessing/`, `processing_scripts/llm_client/`, `processing_scripts/llm/*.txt` | | nfulton99 | HTML ingestion, packaging | `vision_aid/ingestion/pull_html.py`, `pyproject.toml` | | ColeANiblett | Pipeline orchestration, prompt system, report generator | `processing_scripts/llm/{registry,slicers,templates}.py`, `entry_points/`, `docs/` | + +## Model evaluation + +The evaluator measures how many findings from an existing human accessibility +audit va-dat catches. It runs programmatic checks and one selected LLM against +saved page HTML, then uses reviewed matches to produce an offline HTML report +with LLM, Programmatic, and Overall detection rates. + +There are two reviews: approve the audit's classifications and programmatic +matches once, then review LLM matches after each model run. The evaluator +validates the saved evidence and calculates scores; the reviewer decides +whether findings describe the same defect. + +The [evaluator guide](vision_aid/evaluation/README.md) covers input formats, +matching rules, scoring, model settings, saved files, and troubleshooting. + +### Evaluator quick start + +Complete [setup](#setup) first. + +**Run all commands in this guide from the repository root**, the va-dat/ directory containing pyproject.toml and the root README. +All relative paths in the examples are relative to that directory. + +The human audit Excel file, `Pristine Accessibility Defect Report.xlsx`, +belongs directly in the repository root, alongside this README. The examples +expect it at `./Pristine Accessibility Defect Report.xlsx`. The file and +approved mappings are private local inputs, excluded from Git. If the file +is missing, obtain it from your project team and place it there, or supply your +own audit in the [supported format](vision_aid/evaluation/README.md#human-audit-excel-file-format) +and update `AUDIT` below. + +This walkthrough evaluates every URL listed in the audit. Keep the same saved +HTML when comparing models. If you already have snapshots and an approved +mapping for this audit, set their paths below and skip steps 1 and 2. + +```bash +AUDIT='./Pristine Accessibility Defect Report.xlsx' +SNAPSHOTS='.model-evaluation/pristine-snapshots/snapshots.json' +REVIEW_DIR='.model-evaluation/pristine-mapping-review' +MAPPING="$REVIEW_DIR/human-finding-mapping.json" +MODEL='claude-haiku-5-5' +MAX_COST_USD=5.00 # Choose your total allowance for all pages. +``` + +1. **Capture the pages.** The output directory must be new. Capture downloads + server HTML without rendering JavaScript; inspect the saved pages before review. + It requires no model API key. + + ```bash + uv run visionaid-evaluate capture \ + --human-audit "$AUDIT" \ + --output-dir .model-evaluation/pristine-snapshots + ``` + +2. **Prepare and approve the reusable mapping.** This makes no model requests. + The review directory must be new. + + ```bash + uv run visionaid-evaluate human-finding-mapping-prepare \ + --human-audit "$AUDIT" --snapshots "$SNAPSHOTS" \ + --output-dir "$REVIEW_DIR" + ``` + + Follow the generated README: fill `classifications.csv` and + `programmatic-review.csv`, using `programmatic-findings.csv` as the checker + catalog. The [mapping review guide](vision_aid/evaluation/README.md#approve-a-human-finding-mapping) + explains the classification values and complete-match requirements. Then approve: + + ```bash + uv run visionaid-evaluate human-finding-mapping-approve \ + --human-audit "$AUDIT" --snapshots "$SNAPSHOTS" \ + --review-dir "$REVIEW_DIR" --reviewer "Your name and review method" + ``` + +3. **Preview the evaluation.** Inspect the pages, prompt count, model settings, + and prices without an API key or paid requests. A preview cannot produce a + final report. + + ```bash + uv run visionaid-evaluate run-all \ + --human-audit "$AUDIT" --snapshots "$SNAPSHOTS" \ + --human-finding-mapping "$MAPPING" \ + --model "$MODEL" --max-cost-usd "$MAX_COST_USD" + ``` + +4. **Run the model.** Put `ANTHROPIC_API_KEY` for Claude or `OPENAI_API_KEY` for + GPT in your environment or the repository-root `.env`. Only the selected + provider's key is required. Repeat the command with `--live`: + + ```bash + uv run visionaid-evaluate run-all \ + --human-audit "$AUDIT" --snapshots "$SNAPSHOTS" \ + --human-finding-mapping "$MAPPING" \ + --model "$MODEL" --max-cost-usd "$MAX_COST_USD" --live + ``` + + Type `yes` when prompted. The allowance is checked between requests; a request + underway can exceed it. This creates a fresh run instead of executing the + preview directory. Requests run sequentially without retries. Restarting after + failure repeats all requests, including previously completed pages. + +5. **Review the LLM matches.** Use the new directory printed by the live command: + + ```bash + RUN_DIR='.model-evaluation/full-runs/' + ``` + + Confirm `"complete": true` in `full.json`, and check each child's `run.json` + for `format_failures`. Follow each page's README under `$RUN_DIR/pages/`. + Compare `normalized-llm-findings.json`, `review.csv`, and `snapshot.html`. + Edit only `llm_finding_id`, `match_reason`, and `review_notes`. Leave an LLM ID + blank for a miss; record IDs only when they fully cover the human finding. + Keep approved classification and programmatic fields unchanged. + See the [matching rules](vision_aid/evaluation/README.md#review-llm-matches). + +6. **Generate the reports.** After saving every page's review CSV: + + ```bash + uv run visionaid-evaluate report-all \ + --run-dir "$RUN_DIR" --reviewer "Your name and review method" + ``` + + Open `$RUN_DIR/report.html` for combined results. Each page also gets a + `report.html`. Reports work offline and can regenerate without model calls. + +Choose `claude-haiku-5-5`, `claude-opus-5-5`, `claude-sonnet-5-5`, +`gpt-6-luna`, or `gpt-6.1-sol` as `MODEL`. Haiku is the default when `--model` +is omitted. Each run uses one model; compare models with separate runs using +the same audit, snapshots, and mapping. + +To evaluate just one page, use `run` with `--html` and `--source-url`, then +`report`. See the [single-page commands](vision_aid/evaluation/README.md#evaluate-one-page). +For command options, use `uv run visionaid-evaluate --help` or add `--help` +after a command name. + +The rates measure coverage of the supplied human audit. Findings that cannot +be tested from saved HTML remain in Overall, so perfect coverage of testable +findings can still produce an Overall rate below 100%. Read the +[scoring guide](vision_aid/evaluation/README.md#how-scoring-works) before interpreting +or comparing results. diff --git a/entry_points/run_pipeline.py b/entry_points/run_pipeline.py index 1334021..a5e33b5 100644 --- a/entry_points/run_pipeline.py +++ b/entry_points/run_pipeline.py @@ -18,7 +18,6 @@ import json import os import sys -import time from datetime import datetime, timezone from pathlib import Path @@ -31,6 +30,11 @@ from processing_scripts.llm.registry import PROMPT_REGISTRY, PromptSpec from processing_scripts.llm.slicers import get_slicer, is_empty_slice from processing_scripts.llm.templates import fill_template +from processing_scripts.llm_client.audit import ( + AuditRequestClient as PipelineClient, + AuditRequestConfig, + resolved_request_metadata, +) from processing_scripts.llm_preprocessing.semantic_checklist_01 import ( extract as cl01_extract, ) @@ -103,130 +107,6 @@ def estimate_tokens(text: str) -> int: return max(1, len(text) // 4) -class PipelineClient: - """Thin wrapper around the Anthropic, OpenAI, or Gemini API. - - Detects the provider from the model ID and uses the appropriate SDK. - Returns the same dict shape regardless of provider so downstream code - (``generate_report.py``, etc.) works unchanged. - """ - - def __init__(self, api_key: str, model: str, max_tokens: int = 8192): - from processing_scripts.llm_client.client import is_openai_model, is_gemini_model - - self.model = model - self.max_tokens = max_tokens - self._is_openai = is_openai_model(model) - self._is_gemini = is_gemini_model(model) - - if self._is_openai: - import openai - self._client = openai.OpenAI(api_key=api_key) - elif self._is_gemini: - from google import genai - self._client = genai.Client(api_key=api_key) - else: - import anthropic - self._client = anthropic.Anthropic(api_key=api_key) - - def call(self, prompt: str) -> dict: - """Send *prompt* to the API and return a result dict. - - Returns a dict with keys ``success``, ``response``, ``model``, - ``usage``, ``stop_reason``, and ``duration_seconds``. - """ - start = time.time() - - try: - if self._is_openai: - return self._call_openai(prompt, start) - if self._is_gemini: - return self._call_gemini(prompt, start) - return self._call_anthropic(prompt, start) - except Exception as e: - return { - "success": False, - "error": str(e), - "duration_seconds": round(time.time() - start, 2), - } - - def _call_anthropic(self, prompt: str, start: float) -> dict: - """Call the Anthropic Messages API.""" - from processing_scripts.llm_client.client import supports_temperature - - request_kwargs = { - "model": self.model, - "max_tokens": self.max_tokens, - "messages": [{"role": "user", "content": prompt}], - } - if supports_temperature(self.model): - request_kwargs["temperature"] = 0.1 - - message = self._client.messages.create(**request_kwargs) - - response_text = "".join( - block.text for block in message.content if block.type == "text" - ) - - return { - "success": True, - "response": response_text, - "model": message.model, - "usage": { - "input_tokens": message.usage.input_tokens, - "output_tokens": message.usage.output_tokens, - }, - "stop_reason": message.stop_reason, - "duration_seconds": round(time.time() - start, 2), - } - - def _call_openai(self, prompt: str, start: float) -> dict: - """Call the OpenAI Chat Completions API.""" - response = self._client.chat.completions.create( - model=self.model, - max_tokens=self.max_tokens, - temperature=0.1, - messages=[{"role": "user", "content": prompt}], - ) - - return { - "success": True, - "response": response.choices[0].message.content, - "model": response.model, - "usage": { - "input_tokens": response.usage.prompt_tokens, - "output_tokens": response.usage.completion_tokens, - }, - "stop_reason": response.choices[0].finish_reason, - "duration_seconds": round(time.time() - start, 2), - } - - def _call_gemini(self, prompt: str, start: float) -> dict: - """Call the Gemini generateContent API.""" - response = self._client.models.generate_content( - model=self.model, - contents=prompt, - config={ - "temperature": 0.1, - "max_output_tokens": self.max_tokens, - }, - ) - - return { - "success": True, - "response": response.text, - "model": self.model, - "usage": { - "input_tokens": response.usage_metadata.prompt_token_count, - "output_tokens": response.usage_metadata.candidates_token_count, - }, - "stop_reason": response.candidates[0].finish_reason.name - if response.candidates - else None, - "duration_seconds": round(time.time() - start, 2), - } - - def save_json(obj: object, path: Path) -> None: """Write an object as formatted JSON to a file.""" path.parent.mkdir(parents=True, exist_ok=True) @@ -241,11 +121,17 @@ def run_pipeline( dry_run: bool, include_summaries: bool, progress_callback=None, + request_config: AuditRequestConfig | None = None, ) -> dict: """Execute the full element-specific accessibility audit pipeline. Returns a summary dict with run metadata and per-prompt results. """ + if request_config is None: + request_config = AuditRequestConfig(model=model) + elif request_config.model != model: + raise ValueError("model and request_config.model must match") + output_dir.mkdir(parents=True, exist_ok=True) html_path_str = str(html_path) @@ -327,7 +213,10 @@ def run_pipeline( # Initialise client once (only needed for live runs) client = None if not dry_run: - client = PipelineClient(api_key=api_key, model=model) + client = PipelineClient( + api_key=api_key, + request_config=request_config, + ) for spec in PROMPT_REGISTRY: # Skip summaries unless requested @@ -436,6 +325,7 @@ def run_pipeline( "run_timestamp": datetime.now(timezone.utc).isoformat(), "html_file": str(html_path), "model": model, + "request_config": resolved_request_metadata(request_config), "dry_run": dry_run, "include_summaries": include_summaries, "programmatic_findings_count": len(programmatic_findings), diff --git a/processing_scripts/llm_client/__init__.py b/processing_scripts/llm_client/__init__.py index 01e8aa9..192290d 100644 --- a/processing_scripts/llm_client/__init__.py +++ b/processing_scripts/llm_client/__init__.py @@ -1,28 +1,8 @@ -""" -llm_client — LLM API clients for the VisionAid accessibility audit pipeline. +"""Provider-independent requests for the accessibility audit pipeline.""" -Supports both Anthropic (Claude) and OpenAI (GPT) models. - -Usage ------ - from llm_client import create_audit_client, load_all_prompts, run_all - - client = create_audit_client(model="claude-sonnet-5") # or "gpt-4o" - prompts = load_all_prompts(PROMPTS_DIR) - report = run_all(client, prompts, all_slices) -""" - -from .client import AuditClient, OpenAIAuditClient, create_audit_client, is_openai_model -from .prompt_loader import load_prompts, load_all_prompts -from .runner import run_checklist, run_all +from .audit import AuditRequestClient, AuditRequestConfig __all__ = [ - "AuditClient", - "OpenAIAuditClient", - "create_audit_client", - "is_openai_model", - "load_prompts", - "load_all_prompts", - "run_checklist", - "run_all", + "AuditRequestClient", + "AuditRequestConfig", ] diff --git a/processing_scripts/llm_client/audit.py b/processing_scripts/llm_client/audit.py new file mode 100644 index 0000000..6f88413 --- /dev/null +++ b/processing_scripts/llm_client/audit.py @@ -0,0 +1,493 @@ +"""Provider-independent request configuration for accessibility audits.""" + +from __future__ import annotations + +import time +from collections.abc import Mapping +from dataclasses import dataclass +from enum import Enum +from types import SimpleNamespace +from typing import Any, Literal + +from .client import is_gemini_model, is_openai_model, supports_temperature + +ReasoningEffort = Literal[ + "none", + "minimal", + "low", + "medium", + "high", + "xhigh", + "max", +] + + +@dataclass(frozen=True) +class AuditRequestConfig: + """Configuration shared by every prompt in one audit run. + + ``temperature=None`` deliberately omits the sampling parameter. Existing + production callers retain the historical 0.1 temperature and 8,192-token + output limit when they do not supply this configuration explicitly. + """ + + model: str + reasoning_effort: ReasoningEffort | None = None + temperature: float | None = 0.1 + max_output_tokens: int = 8192 + thinking_budget_tokens: int | None = None + adaptive_thinking: bool = False + + def __post_init__(self) -> None: + """Reject configuration values that cannot form an audit request.""" + if not self.model: + raise ValueError("model must not be empty") + if self.max_output_tokens <= 0: + raise ValueError("max_output_tokens must be greater than zero") + + if self.adaptive_thinking: + if is_openai_model(self.model) or is_gemini_model(self.model): + raise ValueError("adaptive_thinking requires Anthropic") + if self.thinking_budget_tokens is not None: + raise ValueError("adaptive thinking cannot have a manual budget") + if self.temperature is not None: + raise ValueError("temperature must be omitted with thinking") + if self.reasoning_effort not in (None, "low", "medium", "high", "xhigh", "max"): + raise ValueError("unsupported Anthropic reasoning effort") + + if self.thinking_budget_tokens is not None: + if is_openai_model(self.model) or is_gemini_model(self.model): + raise ValueError("thinking_budget_tokens requires Anthropic") + if not 1024 <= self.thinking_budget_tokens < self.max_output_tokens: + raise ValueError("thinking budget must be >= 1024 and below output cap") + if self.temperature is not None: + raise ValueError("temperature must be omitted with thinking") + + def as_metadata(self) -> dict[str, Any]: + """Return the requested configuration as serializable metadata.""" + omitted = [] if self.temperature is not None else ["temperature"] + return { + **({"thinking": {"type": "adaptive"}} if self.adaptive_thinking else {}), + **( + { + "thinking": { + "type": "enabled", + "budget_tokens": self.thinking_budget_tokens, + } + } + if self.thinking_budget_tokens is not None + else {} + ), + "model": self.model, + "reasoning_effort": self.reasoning_effort, + "max_output_tokens": self.max_output_tokens, + "sampling": { + "temperature": self.temperature, + "omitted_parameters": omitted, + }, + } + + +def resolved_request_metadata(config: AuditRequestConfig) -> dict[str, Any]: + """Describe the effective request after provider capability omissions.""" + metadata = config.as_metadata() + if ( + not is_openai_model(config.model) + and not is_gemini_model(config.model) + and not supports_temperature(config.model) + ): + metadata["sampling"] = { + "temperature": None, + "omitted_parameters": ["temperature"], + } + return metadata + + +def _provider_data(value: Any) -> Any: + """Convert SDK response metadata into JSON-serializable builtins.""" + if value is None or isinstance(value, (str, int, float, bool)): + return value + if isinstance(value, Enum): + return value.value + if isinstance(value, Mapping): + return {str(key): _provider_data(item) for key, item in value.items()} + if isinstance(value, (list, tuple)): + return [_provider_data(item) for item in value] + if hasattr(value, "model_dump"): + return _provider_data(value.model_dump()) + if hasattr(value, "to_dict"): + return _provider_data(value.to_dict()) + if hasattr(value, "__dict__"): + return { + key: _provider_data(item) + for key, item in vars(value).items() + if not key.startswith("_") + } + return str(value) + + +def _usage_result( + provider_usage: Any, + *, + input_tokens: int, + output_tokens: int, + total_tokens: int | None = None, + cached_input_tokens: int = 0, + cache_creation_input_tokens: int = 0, + reasoning_tokens: int | None = 0, +) -> dict[str, Any]: + """Return common token categories plus the lossless provider payload.""" + return { + "input_tokens": input_tokens, + "output_tokens": output_tokens, + "total_tokens": ( + total_tokens if total_tokens is not None else input_tokens + output_tokens + ), + "cached_input_tokens": cached_input_tokens, + "cache_creation_input_tokens": cache_creation_input_tokens, + "reasoning_tokens": reasoning_tokens, + "provider_usage": _provider_data(provider_usage), + } + + +class _StreamFailure(Exception): + """Carry observed usage and final text across a transport failure.""" + + def __init__( + self, + cause: Exception, + usage: dict[str, Any], + stop_reason: str | None, + response_text: str, + ): + """Retain partial evidence without retaining thinking blocks.""" + super().__init__(str(cause)) + self.cause = cause + self.usage = usage + self.stop_reason = stop_reason + self.response_text = response_text + + +class AuditRequestClient: + """Send an audit prompt through Anthropic, OpenAI, or Gemini. + + The returned dict retains the legacy keys consumed by the production + pipeline and adds complete provider, request, usage, timing, stop, and + failure metadata for evaluation runs. + """ + + def __init__( + self, + api_key: str, + model: str | None = None, + max_tokens: int = 8192, + *, + request_config: AuditRequestConfig | None = None, + provider_client: Any | None = None, + max_retries: int | None = None, + ): + """Resolve legacy arguments and initialize the selected SDK client.""" + if request_config is None: + if model is None: + raise TypeError("model or request_config is required") + request_config = AuditRequestConfig( + model=model, + max_output_tokens=max_tokens, + ) + elif model is not None and model != request_config.model: + raise ValueError("model and request_config.model must match") + + self.request_config = request_config + self.model = request_config.model + self.max_tokens = request_config.max_output_tokens + self._is_openai = is_openai_model(self.model) + self._is_gemini = is_gemini_model(self.model) + + if ( + request_config.reasoning_effort is not None + and not self._is_openai + and not request_config.adaptive_thinking + ): + raise ValueError( + "reasoning_effort requires OpenAI or Anthropic adaptive thinking" + ) + + self._client: Any + if provider_client is not None: + self._client = provider_client + elif self._is_openai: + import openai + + options: dict[str, Any] = {"api_key": api_key} + if max_retries is not None: + options["max_retries"] = max_retries + self._client = openai.OpenAI(**options) + elif self._is_gemini: + from google import genai + + self._client = genai.Client(api_key=api_key) + else: + import anthropic + + options: dict[str, Any] = {"api_key": api_key} + if max_retries is not None: + options["max_retries"] = max_retries + self._client = anthropic.Anthropic(**options) + + def call(self, prompt: str) -> dict[str, Any]: + """Send one prompt and return a provider-neutral audit result.""" + start = time.monotonic() + + try: + if self._is_openai: + result = self._call_openai(prompt, start) + elif self._is_gemini: + result = self._call_gemini(prompt, start) + else: + result = self._call_anthropic(prompt, start) + result["request"] = resolved_request_metadata(self.request_config) + return result + except Exception as exc: # Report provider failures as evidence. + partial_usage = {} + stop_reason = None + response_text = None + if isinstance(exc, _StreamFailure): + partial_usage = exc.usage + stop_reason = exc.stop_reason + response_text = exc.response_text + exc = exc.cause + return { + "success": False, + "response": response_text, + "model": self.model, + "provider": self._provider_identity(), + "request": resolved_request_metadata(self.request_config), + "usage": _usage_result( + partial_usage or None, + input_tokens=partial_usage.get("input_tokens", 0), + output_tokens=partial_usage.get("output_tokens", 0), + reasoning_tokens=( + 0 if self._is_openai or self._is_gemini else None + ), + cached_input_tokens=partial_usage.get("cache_read_input_tokens", 0) + or 0, + cache_creation_input_tokens=partial_usage.get( + "cache_creation_input_tokens", 0 + ) + or 0, + ), + "stop_reason": stop_reason, + "error": str(exc), + "duration_seconds": round(time.monotonic() - start, 2), + "failure": { + "type": type(exc).__name__, + "message": str(exc), + "code": _provider_data(getattr(exc, "code", None)), + "status_code": getattr(exc, "status_code", None), + "request_id": getattr(exc, "request_id", None), + }, + } + + def _provider_identity(self) -> dict[str, Any]: + """Return stable provider and endpoint names for failed requests.""" + if self._is_openai: + return {"name": "openai", "endpoint": "chat.completions"} + if self._is_gemini: + return {"name": "google", "endpoint": "models.generate_content"} + return {"name": "anthropic", "endpoint": "messages"} + + def _call_anthropic(self, prompt: str, start: float) -> dict[str, Any]: + """Send one prompt through Anthropic Messages.""" + request_kwargs: dict[str, Any] = { + "model": self.model, + "max_tokens": self.max_tokens, + "messages": [{"role": "user", "content": prompt}], + } + if self.request_config.temperature is not None and supports_temperature( + self.model + ): + request_kwargs["temperature"] = self.request_config.temperature + + if self.request_config.thinking_budget_tokens is not None: + request_kwargs["thinking"] = { + "type": "enabled", + "budget_tokens": self.request_config.thinking_budget_tokens, + } + elif self.request_config.adaptive_thinking: + request_kwargs["thinking"] = {"type": "adaptive"} + if self.request_config.reasoning_effort is not None: + request_kwargs["output_config"] = { + "effort": self.request_config.reasoning_effort, + } + + if ( + self.request_config.thinking_budget_tokens is not None + or self.request_config.adaptive_thinking + ): + message = self._stream_anthropic(request_kwargs) + else: + message = self._client.messages.create(**request_kwargs) + response_text = "".join( + block.text for block in message.content if block.type == "text" + ) + + return { + "success": True, + "response": response_text, + "model": message.model, + "provider": { + "name": "anthropic", + "endpoint": "messages", + "response_id": getattr(message, "id", None), + "response_model": message.model, + "response_type": getattr(message, "type", None), + }, + "usage": _usage_result( + message.usage, + reasoning_tokens=None, + input_tokens=message.usage.input_tokens, + output_tokens=message.usage.output_tokens, + cached_input_tokens=getattr(message.usage, "cache_read_input_tokens", 0) + or 0, + cache_creation_input_tokens=getattr( + message.usage, "cache_creation_input_tokens", 0 + ) + or 0, + ), + "stop_reason": message.stop_reason, + "duration_seconds": round(time.monotonic() - start, 2), + "failure": None, + } + + def _stream_anthropic(self, request_kwargs: dict[str, Any]) -> Any: + """Consume Messages events without accumulating private thinking blocks.""" + text_parts: list[str] = [] + usage: dict[str, Any] = {} + message = None + stopped = False + try: + with self._client.messages.create(**request_kwargs, stream=True) as stream: + for event in stream: + if event.type == "message_start": + source = event.message + message = SimpleNamespace( + id=source.id, + model=source.model, + type=source.type, + stop_reason=None, + ) + usage.update(_provider_data(source.usage)) + elif event.type == "content_block_start": + if event.content_block.type == "text": + text_parts.append(event.content_block.text) + elif event.type == "content_block_delta": + if event.delta.type == "text_delta": + text_parts.append(event.delta.text) + elif event.type == "message_delta" and message is not None: + message.stop_reason = event.delta.stop_reason + usage.update( + { + key: value + for key, value in _provider_data(event.usage).items() + if value is not None + } + ) + elif event.type == "message_stop": + stopped = True + if message is None or not stopped: + raise ValueError("Anthropic stream ended before message_stop") + except Exception as exc: + raise _StreamFailure( + exc, + usage, + message.stop_reason if message is not None else None, + "".join(text_parts), + ) from exc + message.content = [SimpleNamespace(type="text", text="".join(text_parts))] + message.usage = SimpleNamespace(**usage) + return message + + def _call_openai(self, prompt: str, start: float) -> dict[str, Any]: + """Send one prompt through OpenAI Chat Completions.""" + request_kwargs: dict[str, Any] = { + "model": self.model, + "messages": [{"role": "user", "content": prompt}], + } + if self.request_config.reasoning_effort is None: + request_kwargs["max_tokens"] = self.max_tokens + else: + request_kwargs["max_completion_tokens"] = self.max_tokens + request_kwargs["reasoning_effort"] = self.request_config.reasoning_effort + if self.request_config.temperature is not None: + request_kwargs["temperature"] = self.request_config.temperature + + response = self._client.chat.completions.create(**request_kwargs) + prompt_details = getattr(response.usage, "prompt_tokens_details", None) + completion_details = getattr(response.usage, "completion_tokens_details", None) + + return { + "success": True, + "response": response.choices[0].message.content, + "model": response.model, + "provider": { + "name": "openai", + "endpoint": "chat.completions", + "response_id": getattr(response, "id", None), + "response_model": response.model, + "service_tier": getattr(response, "service_tier", None), + "system_fingerprint": getattr(response, "system_fingerprint", None), + }, + "usage": _usage_result( + response.usage, + input_tokens=response.usage.prompt_tokens, + output_tokens=response.usage.completion_tokens, + total_tokens=getattr(response.usage, "total_tokens", None), + cached_input_tokens=getattr(prompt_details, "cached_tokens", 0) or 0, + reasoning_tokens=getattr(completion_details, "reasoning_tokens", 0) + or 0, + ), + "stop_reason": response.choices[0].finish_reason, + "duration_seconds": round(time.monotonic() - start, 2), + "failure": None, + } + + def _call_gemini(self, prompt: str, start: float) -> dict[str, Any]: + """Send one prompt through Gemini generateContent.""" + generation_config: dict[str, Any] = {"max_output_tokens": self.max_tokens} + if self.request_config.temperature is not None: + generation_config["temperature"] = self.request_config.temperature + + response = self._client.models.generate_content( + model=self.model, + contents=prompt, + config=generation_config, + ) + usage = response.usage_metadata + + return { + "success": True, + "response": response.text, + "model": self.model, + "provider": { + "name": "google", + "endpoint": "models.generate_content", + "response_id": getattr(response, "response_id", None), + "response_model": getattr(response, "model_version", self.model), + }, + "usage": _usage_result( + usage, + input_tokens=usage.prompt_token_count, + output_tokens=usage.candidates_token_count, + total_tokens=getattr(usage, "total_token_count", None), + cached_input_tokens=getattr(usage, "cached_content_token_count", 0) + or 0, + reasoning_tokens=(getattr(usage, "thoughts_token_count", 0) or 0), + ), + "stop_reason": ( + response.candidates[0].finish_reason.name + if response.candidates + else None + ), + "duration_seconds": round(time.monotonic() - start, 2), + "failure": None, + } diff --git a/processing_scripts/llm_client/client.py b/processing_scripts/llm_client/client.py index 9f0aae1..d4eecfc 100644 --- a/processing_scripts/llm_client/client.py +++ b/processing_scripts/llm_client/client.py @@ -1,21 +1,4 @@ -""" -AuditClient — thin wrappers around the Anthropic and OpenAI APIs. - -Loads API keys from the project root .env file automatically. -""" - -import json -import os -from pathlib import Path - -import anthropic -import openai -from dotenv import load_dotenv -from google import genai - -# Project root is two levels up from this file (processing_scripts/llm_client/) -_ENV_FILE = Path(__file__).resolve().parents[2] / ".env" -load_dotenv(_ENV_FILE) +"""Model-provider detection and request capability helpers.""" # OpenAI model prefixes used to detect provider from model ID _OPENAI_PREFIXES = ("gpt-", "o1", "o3", "o4") @@ -51,232 +34,3 @@ def is_gemini_model(model: str) -> bool: def supports_temperature(model: str) -> bool: """Return False for Claude models that reject the `temperature` param.""" return model not in _NO_SAMPLING_PARAMS_MODELS - - -class AuditClient: - """ - Sends filled accessibility-audit prompts to the Claude API and returns - parsed JSON responses. - - Parameters - ---------- - model : str - Claude model ID to use (default: claude-sonnet-5). Also accepts - ``claude-opus-5`` and ``claude-fable-5``, Anthropic's most capable - model. Newer Claude models (Opus 4.6+, Sonnet 4.6+, Fable/Mythos 5) - reject a ``temperature`` param entirely (see ``supports_temperature``), - which this client already accounts for. - temperature : float - Sampling temperature (default: 0.1 for consistent structured output). - Silently omitted from the request for models that reject it. - max_tokens : int - Maximum tokens to generate per response (default: 8192). - """ - - def __init__( - self, - model: str = "claude-sonnet-5", - temperature: float = 0.1, - max_tokens: int = 8192, - ): - api_key = os.environ.get("ANTHROPIC_API_KEY") - if not api_key: - raise EnvironmentError( - f"ANTHROPIC_API_KEY not found. Expected in {_ENV_FILE}" - ) - self._client = anthropic.Anthropic(api_key=api_key) - self.model = model - self.temperature = temperature - self.max_tokens = max_tokens - - def call(self, prompt_text: str, payload: dict) -> tuple[dict, dict]: - """ - Fill `{payload}` in *prompt_text* with JSON-serialised *payload*, - send to the API, and return ``(parsed_response, usage)``. - - The response is expected to be a JSON object (possibly wrapped in a - markdown code fence, which is stripped automatically). - - Returns - ------- - response_json : dict - Parsed JSON from the model's reply. - usage : dict - ``{"input_tokens": int, "output_tokens": int}`` - """ - filled = prompt_text.replace("{payload}", json.dumps(payload, separators=(",", ":"))) - - request_kwargs = { - "model": self.model, - "max_tokens": self.max_tokens, - "messages": [{"role": "user", "content": filled}], - } - if supports_temperature(self.model): - request_kwargs["temperature"] = self.temperature - - message = self._client.messages.create(**request_kwargs) - - usage = { - "input_tokens": message.usage.input_tokens, - "output_tokens": message.usage.output_tokens, - } - - raw = message.content[0].text.strip() - - # Strip markdown code fences if present - if raw.startswith("```"): - raw = raw.split("\n", 1)[1] - raw = raw.rsplit("```", 1)[0].strip() - - return json.loads(raw), usage - - -class OpenAIAuditClient: - """ - Sends filled accessibility-audit prompts to the OpenAI API and returns - parsed JSON responses. Same interface as :class:`AuditClient`. - - Parameters - ---------- - model : str - OpenAI model ID to use (default: gpt-4o). - temperature : float - Sampling temperature (default: 0.1 for consistent structured output). - max_tokens : int - Maximum tokens to generate per response (default: 8192). - """ - - def __init__( - self, - model: str = "gpt-4o", - temperature: float = 0.1, - max_tokens: int = 8192, - ): - api_key = os.environ.get("OPENAI_API_KEY") - if not api_key: - raise EnvironmentError( - f"OPENAI_API_KEY not found. Expected in {_ENV_FILE}" - ) - self._client = openai.OpenAI(api_key=api_key) - self.model = model - self.temperature = temperature - self.max_tokens = max_tokens - - def call(self, prompt_text: str, payload: dict) -> tuple[dict, dict]: - """ - Fill `{payload}` in *prompt_text* with JSON-serialised *payload*, - send to the API, and return ``(parsed_response, usage)``. - - Returns - ------- - response_json : dict - Parsed JSON from the model's reply. - usage : dict - ``{"input_tokens": int, "output_tokens": int}`` - """ - filled = prompt_text.replace("{payload}", json.dumps(payload, separators=(",", ":"))) - - response = self._client.chat.completions.create( - model=self.model, - max_tokens=self.max_tokens, - temperature=self.temperature, - messages=[{"role": "user", "content": filled}], - ) - - usage = { - "input_tokens": response.usage.prompt_tokens, - "output_tokens": response.usage.completion_tokens, - } - - raw = response.choices[0].message.content.strip() - - # Strip markdown code fences if present - if raw.startswith("```"): - raw = raw.split("\n", 1)[1] - raw = raw.rsplit("```", 1)[0].strip() - - return json.loads(raw), usage - - -class GeminiAuditClient: - """ - Sends filled accessibility-audit prompts to the Gemini API and returns - parsed JSON responses. Same interface as :class:`AuditClient`. - - Parameters - ---------- - model : str - Gemini model ID to use (default: gemini-flash-latest). Prefer the - ``-latest`` aliases over dated model IDs (e.g. ``gemini-2.5-flash``), - which Google periodically retires for new API keys. - temperature : float - Sampling temperature (default: 0.1 for consistent structured output). - max_tokens : int - Maximum tokens to generate per response (default: 8192). - """ - - def __init__( - self, - model: str = "gemini-flash-latest", - temperature: float = 0.1, - max_tokens: int = 8192, - ): - api_key = os.environ.get("GEMINI_API_KEY") - if not api_key: - raise EnvironmentError( - f"GEMINI_API_KEY not found. Expected in {_ENV_FILE}" - ) - self._client = genai.Client(api_key=api_key) - self.model = model - self.temperature = temperature - self.max_tokens = max_tokens - - def call(self, prompt_text: str, payload: dict) -> tuple[dict, dict]: - """ - Fill `{payload}` in *prompt_text* with JSON-serialised *payload*, - send to the API, and return ``(parsed_response, usage)``. - - Returns - ------- - response_json : dict - Parsed JSON from the model's reply. - usage : dict - ``{"input_tokens": int, "output_tokens": int}`` - """ - filled = prompt_text.replace("{payload}", json.dumps(payload, separators=(",", ":"))) - - response = self._client.models.generate_content( - model=self.model, - contents=filled, - config={ - "temperature": self.temperature, - "max_output_tokens": self.max_tokens, - }, - ) - - usage = { - "input_tokens": response.usage_metadata.prompt_token_count, - "output_tokens": response.usage_metadata.candidates_token_count, - } - - raw = response.text.strip() - - # Strip markdown code fences if present - if raw.startswith("```"): - raw = raw.split("\n", 1)[1] - raw = raw.rsplit("```", 1)[0].strip() - - return json.loads(raw), usage - - -def create_audit_client( - model: str = "claude-sonnet-5", - temperature: float = 0.1, - max_tokens: int = 8192, -) -> AuditClient | OpenAIAuditClient | GeminiAuditClient: - """Factory that returns the right client class based on the model ID.""" - if is_openai_model(model): - return OpenAIAuditClient(model=model, temperature=temperature, max_tokens=max_tokens) - if is_gemini_model(model): - return GeminiAuditClient(model=model, temperature=temperature, max_tokens=max_tokens) - return AuditClient(model=model, temperature=temperature, max_tokens=max_tokens) diff --git a/processing_scripts/llm_client/prompt_loader.py b/processing_scripts/llm_client/prompt_loader.py deleted file mode 100644 index 60d33d6..0000000 --- a/processing_scripts/llm_client/prompt_loader.py +++ /dev/null @@ -1,72 +0,0 @@ -""" -Parses the numbered prompt sections from the LLM prompt-library text files. - -File format (see processing_scripts/llm/*.txt): - --------------------------------- - N) PROMPT TITLE - --------------------------------- - - Data: {payload} - - --------------------------------- - N+1) NEXT PROMPT TITLE - ... -""" - -import re -from pathlib import Path - - -_HEADER_RE = re.compile(r"^\s{2,}(\d+)\)\s+.+", re.MULTILINE) -_DIVIDER_RE = re.compile(r"^\s*-{5,}\s*$", re.MULTILINE) - - -def load_prompts(filepath: str | Path) -> dict[int, str]: - """ - Parse *filepath* and return ``{prompt_number: prompt_text}`` for every - numbered section found. Divider lines (``---...``) are stripped. - - The returned text for each prompt still contains the ``{payload}`` - placeholder so the caller can fill it later. - """ - text = Path(filepath).read_text(encoding="utf-8") - - # Find all header positions and their prompt numbers - headers = list(_HEADER_RE.finditer(text)) - if not headers: - return {} - - prompts: dict[int, str] = {} - - for i, header in enumerate(headers): - num = int(header.group(1)) - start = header.end() # character just after the header line - end = headers[i + 1].start() if i + 1 < len(headers) else len(text) - - body = text[start:end] - - # Remove divider lines - body = _DIVIDER_RE.sub("", body) - - prompts[num] = body.strip() - - return prompts - - -def load_all_prompts(prompts_dir: str | Path) -> dict[str, dict[int, str]]: - """ - Load all ``*.txt`` prompt files in *prompts_dir*. - - Returns ``{stem: {prompt_number: prompt_text}}``, e.g.:: - - { - "semantic_checklist_01": {1: "...", 2: "...", ...}, - "forms_checklist_02": {1: "...", ...}, - "nontext_checklist_03": {1: "...", ...}, - } - """ - prompts_dir = Path(prompts_dir) - result: dict[str, dict[int, str]] = {} - for txt_file in sorted(prompts_dir.glob("*.txt")): - result[txt_file.stem] = load_prompts(txt_file) - return result diff --git a/processing_scripts/llm_client/runner.py b/processing_scripts/llm_client/runner.py deleted file mode 100644 index 97bbee8..0000000 --- a/processing_scripts/llm_client/runner.py +++ /dev/null @@ -1,159 +0,0 @@ -""" -High-level runner that executes one or all checklist prompt sets against -pre-built JSON payloads, collects responses, and aggregates token usage. -""" - -from __future__ import annotations - -from typing import Any - -from .client import AuditClient - - -def run_checklist( - client: AuditClient, - prompts: dict[int, str], - slices: dict[str, Any | None], - *, - verbose: bool = True, -) -> dict[str, Any]: - """ - Run every prompt in *prompts* against the matching payload slice in *slices*. - - Parameters - ---------- - client : AuditClient - Configured API client. - prompts : dict[int, str] - ``{prompt_number: prompt_text}`` from ``load_prompts()``. - slices : dict[str, payload | None] - Mapping of label → payload dict (or ``None`` to skip). - Labels are used as keys in the returned results dict. - verbose : bool - Print progress to stdout while running. - - Returns - ------- - dict with keys: - - ``"results"`` : ``{label: response_json}`` - - ``"skipped"`` : list of skipped labels - - ``"usage"`` : ``{"input_tokens": int, "output_tokens": int}`` - - ``"errors"`` : ``{label: error_message}`` - """ - results: dict[str, Any] = {} - skipped: list[str] = [] - errors: dict[str, str] = {} - total_input = 0 - total_output = 0 - - # Map labels → prompt numbers by position (label order == prompt order) - labels = list(slices.keys()) - - for idx, label in enumerate(labels): - prompt_num = idx + 1 - payload = slices[label] - - if payload is None: - if verbose: - print(f" [{prompt_num}] {label} — SKIPPED (filtered by Pass 1)") - skipped.append(label) - continue - - # Skip if the payload is empty after filtering - if _is_empty_payload(payload): - if verbose: - print(f" [{prompt_num}] {label} — SKIPPED (empty payload after filtering)") - skipped.append(label) - continue - - prompt_text = prompts.get(prompt_num) - if prompt_text is None: - if verbose: - print(f" [{prompt_num}] {label} — SKIPPED (no prompt text found)") - skipped.append(label) - continue - - if verbose: - print(f" [{prompt_num}] {label} ...", end=" ", flush=True) - - try: - response, usage = client.call(prompt_text, payload) - results[label] = response - total_input += usage["input_tokens"] - total_output += usage["output_tokens"] - if verbose: - print(f"✓ ({usage['input_tokens']}in / {usage['output_tokens']}out tokens)") - except Exception as exc: - errors[label] = str(exc) - if verbose: - print(f"✗ ERROR: {exc}") - - return { - "results": results, - "skipped": skipped, - "usage": {"input_tokens": total_input, "output_tokens": total_output}, - "errors": errors, - } - - -def run_all( - client: AuditClient, - all_prompts: dict[str, dict[int, str]], - all_slices: dict[str, dict[str, Any | None]], - *, - verbose: bool = True, -) -> dict[str, Any]: - """ - Run all checklists in *all_slices* and return a consolidated report dict. - - Parameters - ---------- - client : AuditClient - all_prompts : dict[str, dict[int, str]] - ``{checklist_stem: {prompt_num: prompt_text}}`` from ``load_all_prompts()``. - all_slices : dict[str, dict[str, payload|None]] - ``{checklist_stem: {label: payload|None}}``. - verbose : bool - - Returns - ------- - dict with keys: - - ``"checklists"`` : ``{stem: checklist_result}`` - - ``"total_usage"`` : ``{"input_tokens": int, "output_tokens": int}`` - """ - checklists: dict[str, Any] = {} - grand_input = 0 - grand_output = 0 - - for stem, slices in all_slices.items(): - prompts = all_prompts.get(stem, {}) - if verbose: - print(f"\n--- {stem} ---") - - result = run_checklist(client, prompts, slices, verbose=verbose) - checklists[stem] = result - grand_input += result["usage"]["input_tokens"] - grand_output += result["usage"]["output_tokens"] - - return { - "checklists": checklists, - "total_usage": {"input_tokens": grand_input, "output_tokens": grand_output}, - } - - -# --------------------------------------------------------------------------- -# Helpers -# --------------------------------------------------------------------------- - -def _is_empty_payload(payload: Any) -> bool: - """Return True if every list-valued entry in the payload is empty.""" - if not isinstance(payload, dict): - return False - for v in payload.values(): - if isinstance(v, list) and v: - return False - if isinstance(v, dict) and v: - return False - if isinstance(v, str) and v.strip(): - return False - return True diff --git a/pyproject.toml b/pyproject.toml index 29ab6af..9686263 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -12,11 +12,31 @@ dependencies = [ "anthropic~=0.84.0", "beautifulsoup4>=4.9.2", "lxml>=4.9.0", + "jinja2>=3.1.6", "openai>=1.0.0", "google-genai>=1.0.0", + "openpyxl>=3.1.5", "python-dotenv~=1.2.1", "requests>=2.32.2", ] -[tool.setuptools] -packages = ["vision_aid"] \ No newline at end of file +[project.scripts] +visionaid-evaluate = "vision_aid.evaluation.cli:main" + +[dependency-groups] +dev = [ + "pytest>=8.3.5", +] + +[tool.pytest.ini_options] +pythonpath = ["."] +testpaths = ["tests"] + +[tool.setuptools.packages.find] +include = ["vision_aid*", "processing_scripts*", "entry_points*"] +namespaces = true + +[tool.setuptools.package-data] +"vision_aid.evaluation.execution" = ["pricing.json"] +"vision_aid.evaluation.reporting" = ["templates/*.html.jinja", "css/*.css", "js/*.js"] +"processing_scripts.llm" = ["*.txt"] diff --git a/requirements.txt b/requirements.txt index 1a5da01..697adfb 100644 --- a/requirements.txt +++ b/requirements.txt @@ -32,6 +32,8 @@ distro==1.9.0 # openai docstring-parser==0.18.0 # via anthropic +et-xmlfile==2.0.0 + # via openpyxl google-auth==2.56.3 # via google-genai google-genai==2.17.0 @@ -50,14 +52,20 @@ idna==3.18 # anyio # httpx # requests +jinja2==3.1.6 + # via visionaid jiter==0.16.0 # via # anthropic # openai lxml==6.1.1 # via visionaid +markupsafe==3.0.4 + # via jinja2 openai==2.53.0 # via visionaid +openpyxl==3.1.5 + # via visionaid pyasn1==0.6.4 # via pyasn1-modules pyasn1-modules==0.4.2 diff --git a/tests/evaluation_fixtures.py b/tests/evaluation_fixtures.py new file mode 100644 index 0000000..35875ce --- /dev/null +++ b/tests/evaluation_fixtures.py @@ -0,0 +1,70 @@ +"""Run CLI fixtures with approved mappings retained throughout execution and review.""" + +import json + +from vision_aid.evaluation import human_finding_mapping +from vision_aid.evaluation.cli import main +from vision_aid.evaluation.inputs.human_audit import ( + import_all_human_findings, + import_page_human_findings, +) + + +def _argument(arguments, name): + """Respect the CLI's last occurrence of a repeated option.""" + return next( + arguments[index + 1] + for index in reversed(range(len(arguments))) + if arguments[index] == name + ) + + +def run_with_mapping(workspace, arguments): + """Supply approved fixture decisions unless the caller selects a mapping.""" + if "--human-finding-mapping" in arguments: + return main(arguments) + audit_path = workspace / _argument(arguments, "--human-audit") + try: + audits = ( + import_all_human_findings(audit_path) + if arguments[0] == "run-all" + else { + _argument(arguments, "--source-url"): import_page_human_findings( + audit_path, + _argument(arguments, "--source-url"), + ), + } + ) + except (OSError, ValueError, KeyError): + return main(arguments) + checksum = next(iter(audits.values())).human_audit_sha256 + if (human_finding_mapping.HUMAN_FINDING_MAPPING_DIRECTORY / f"{checksum}.json").is_file(): + return main(arguments) + decisions = {} + for url, audit in audits.items(): + for finding in audit.human_findings: + decision = decisions.setdefault( + finding.human_finding_id, + { + "human_finding_id": finding.human_finding_id, + "classification": "llm_eligible", + "classification_reason": "Approved fixture decisions", + "programmatic_by_page": {}, + }, + ) + decision["programmatic_by_page"][url] = { + "expectations": [], + "reason": "Approved fixture decisions", + } + source = workspace / "fixture-human-finding-mapping.json" + source.write_text( + json.dumps( + { + "schema_version": 1, + "human_audit_sha256": checksum, + "approved_by": "Approved fixture decisions", + "decisions": list(decisions.values()), + } + ) + ) + return main([*arguments, "--human-finding-mapping", str(source)]) diff --git a/tests/test_audit_request.py b/tests/test_audit_request.py new file mode 100644 index 0000000..6bfe9d8 --- /dev/null +++ b/tests/test_audit_request.py @@ -0,0 +1,35 @@ +"""Provider failures retain the evidence consumed by evaluator execution.""" + +from types import SimpleNamespace + +from processing_scripts.llm_client.audit import AuditRequestClient, AuditRequestConfig + + +def test_failure_exposes_request_and_provider_information(): + """Failures retain the metadata needed for later classification.""" + class ProviderFailure(Exception): + status_code = 429 + code = "rate_limit_exceeded" + request_id = "req_failed_1" + + def fail(**kwargs): + """Raise a provider-shaped failure without a network request.""" + raise ProviderFailure("try again later") + + fake = SimpleNamespace(chat=SimpleNamespace(completions=SimpleNamespace(create=fail))) + result = AuditRequestClient("unused", provider_client=fake, request_config=AuditRequestConfig( + model="gpt-6-luna", reasoning_effort="medium", temperature=None, + )).call("audit") + assert not result["success"] + assert result["error"] == "try again later" + assert result["provider"] == {"name": "openai", "endpoint": "chat.completions"} + assert result["model"] == "gpt-6-luna" + assert result["usage"]["total_tokens"] == 0 and result["stop_reason"] is None + assert result["failure"] == { + "type": "ProviderFailure", "message": "try again later", + "code": "rate_limit_exceeded", "status_code": 429, "request_id": "req_failed_1", + } + assert result["request"] == { + "model": "gpt-6-luna", "reasoning_effort": "medium", "max_output_tokens": 8192, + "sampling": {"temperature": None, "omitted_parameters": ["temperature"]}, + } diff --git a/tests/test_evaluation.py b/tests/test_evaluation.py new file mode 100644 index 0000000..2d33d8a --- /dev/null +++ b/tests/test_evaluation.py @@ -0,0 +1,711 @@ +"""Run, review, and report Evaluation runs through the public CLI.""" + +import csv +import hashlib +import json +from contextlib import nullcontext +from decimal import Decimal +from pathlib import Path +from types import SimpleNamespace + +import openpyxl +import pytest +from bs4 import BeautifulSoup +from evaluation_fixtures import run_with_mapping + +from processing_scripts.llm.registry import PROMPT_REGISTRY +from vision_aid.evaluation.cli import main +from vision_aid.evaluation.execution import llm_audit + +MODELS = ( + "claude-haiku-5-5", "claude-opus-5-5", "claude-sonnet-5-5", + "gpt-6-luna", "gpt-6.1-sol", +) +CATEGORIES = [spec.name for spec in PROMPT_REGISTRY if not spec.is_summary] +REVIEWER = "Test reviewer" +# Synthetic markup that activates every non-summary prompt category. +HTML = ( + 'Home' + '

Home

' + "

News

Year
2026
" + '
' + '

Full name

' + '
Plan' + '
' + '
Team' + 'Donate' + 'SalesLogo' + '
' + "
Footer
" +) + + +@pytest.fixture +def workspace(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> Path: + """Build a Human audit and Benchmark snapshot with credentials available.""" + human_audit_excel = openpyxl.Workbook() + sheet = human_audit_excel.active + sheet.title = "Defect report" + sheet.append([ + "Sr. #", "element name", "Browser Combination", "Page name", "Issue Title", + "Steps to Reproduce", "actual result", "expected result", + "Recommendation for Fix", "WCAG Sc", "Type of Change", "comment", + ]) + sheet.append([1, "First link", "Chrome", "Home", "Unclear link", "Open page", + "Vague link\nfor everyone", "Descriptive", "Rename", "2.4.4", "Code", ""]) + for number, title, wcag in ( + (2, "Missing destination", "2.4.4"), (3, "Skipped heading", "1.3.1"), + (4, "Hidden menu", "2.1.1"), (5, "Unclear claim", "4.1.2"), + (6, "No captions", "1.2.2"), + ): + sheet.append([number, "Element", "Chrome", "Global", title, "", "", "", "", wcag]) + sheet.append([7, "Other", "Chrome", "Careers", "Outside scope"]) + human_audit_excel.save(tmp_path / "human-audit.xlsx") + (tmp_path / "home.html").write_text(HTML) + monkeypatch.chdir(tmp_path) + monkeypatch.setenv("ANTHROPIC_API_KEY", "synthetic") + return tmp_path + + +def run_cli(workspace: Path, *extra: str) -> int: + """Exercise the public CLI with approved fixture decisions.""" + return run_with_mapping(workspace, [ + "run", "--human-audit", str(workspace / "human-audit.xlsx"), + "--html", str(workspace / "home.html"), + "--source-url", "https://example.test/", "--max-cost-usd", "1", *extra, + ]) + + +def report(run: Path) -> str: + """Score a reviewed run and return readable HTML text.""" + assert main(["report", "--run-dir", str(run), "--reviewer", REVIEWER]) == 0 + return BeautifulSoup((run / "report.html").read_text(encoding="utf-8"), "html.parser").get_text(" ", strip=True) + + +def refused(run: Path, capsys: pytest.CaptureFixture[str], reviewer: str = REVIEWER) -> str: + """Assert reporting fails without writing a report, returning the error.""" + with pytest.raises(SystemExit) as error: + main(["report", "--run-dir", str(run), "--reviewer", reviewer]) + assert error.value.code == 2 + assert not (run / "report.html").exists() + return capsys.readouterr().err + + +def only_run(workspace: Path) -> Path: + """Find the single run created under the workspace.""" + (run,) = (workspace / ".model-evaluation" / "runs").iterdir() + return run + + +def load(run: Path, name: str): + """Read one saved JSON artifact.""" + return json.loads((run / name).read_text()) + + +def edit_review(run: Path, *decisions: dict[str, str]) -> None: + """Apply reviewer decisions to review.csv rows in Human audit order.""" + path = run / "review.csv" + with path.open(newline="") as stream: + rows = sorted(csv.DictReader(stream), key=lambda row: int(row["source_row"])) + for row, decision in zip(rows, decisions): + row.update(decision) + with path.open("w", newline="") as stream: + writer = csv.DictWriter(stream, fieldnames=rows[0]) + writer.writeheader() + writer.writerows(reversed(rows)) + + +def fake_client(monkeypatch: pytest.MonkeyPatch, *responses: dict) -> list[str]: + """Replace the request client, replaying responses and recording prompts.""" + prompts: list[str] = [] + default = {"success": True, "response": '[{"issues": ["Vague link"]}]', + "usage": {"input_tokens": 1, "output_tokens": 1}} + + def call(prompt: str) -> dict: + """Return the next scripted response.""" + prompts.append(prompt) + return responses[len(prompts) - 1] if len(prompts) <= len(responses) else default + + monkeypatch.setattr(llm_audit, "LLMRequestClient", + lambda **_: SimpleNamespace(call=call)) + return prompts + + +def recording_provider_usage( + monkeypatch, model, *usages, stream_failure_at=None, + text='{"issues": ["Synthetic failure"]}', +): + """Replay SDK usage payloads, then zero usage, through the public CLI.""" + requests = [] + + def create(**kwargs): + """Return a Chat Completion or streamed Message with supplied usage.""" + index = len(requests) + requests.append(kwargs) + usage = usages[index] if index < len(usages) else {} + if model.startswith("gpt-"): + return SimpleNamespace( + id="synthetic-price", model=model, + choices=[SimpleNamespace(message=SimpleNamespace(content=text), finish_reason="stop")], + usage=SimpleNamespace( + prompt_tokens=usage.get("prompt_tokens", 0), + completion_tokens=usage.get("completion_tokens", 0), + total_tokens=usage.get("prompt_tokens", 0) + usage.get("completion_tokens", 0), + prompt_tokens_details=usage.get("prompt_tokens_details"), + completion_tokens_details=usage.get("completion_tokens_details"), + ), + ) + + def events(): + """Expose reported usage before optional transport failure.""" + yield SimpleNamespace(type="message_start", message=SimpleNamespace( + id="synthetic-price", model=model, type="message", usage=SimpleNamespace( + input_tokens=usage.get("input_tokens", 0), output_tokens=0, + cache_read_input_tokens=usage.get("cache_read_input_tokens", 0), + cache_creation_input_tokens=usage.get("cache_creation_input_tokens", 0), + ))) + yield SimpleNamespace(type="content_block_start", content_block=SimpleNamespace( + type="text", text=text)) + yield SimpleNamespace(type="message_delta", delta=SimpleNamespace(stop_reason="end_turn"), + usage=SimpleNamespace(output_tokens=usage.get("output_tokens", 0))) + if index == stream_failure_at: + raise TimeoutError("synthetic stream failure") + yield SimpleNamespace(type="message_stop") + + return nullcontext(events()) + + if model.startswith("gpt-"): + monkeypatch.setenv("OPENAI_API_KEY", "synthetic-openai") + monkeypatch.setattr("anthropic.Anthropic", lambda **_: pytest.fail("wrong provider")) + monkeypatch.setattr("openai.OpenAI", lambda **_: SimpleNamespace( + chat=SimpleNamespace(completions=SimpleNamespace(create=create)))) + else: + monkeypatch.setattr("openai.OpenAI", lambda **_: pytest.fail("wrong provider")) + monkeypatch.setattr("anthropic.Anthropic", lambda **_: SimpleNamespace( + messages=SimpleNamespace(create=create))) + return requests + + +@pytest.mark.parametrize("model", [None, "gpt-6-luna"]) +def test_preview_preserves_evidence_without_provider_access(workspace, monkeypatch, capsys, model): + """Credentials alone never buy requests, and all homepage rows survive.""" + monkeypatch.delenv("ANTHROPIC_API_KEY", raising=False) + monkeypatch.delenv("OPENAI_API_KEY", raising=False) + monkeypatch.setattr("anthropic.Anthropic", lambda **_: pytest.fail("provider access")) + monkeypatch.setattr("openai.OpenAI", lambda **_: pytest.fail("provider access")) + assert run_cli(workspace, *(["--model", model] if model else [])) == 0 + run = only_run(workspace) + manifest = load(run, "run.json") + selected = model or MODELS[0] + config = manifest["configuration"] + assert config["model"] == selected + provider, endpoint = (("openai", "chat.completions") if selected.startswith("gpt-") + else ("anthropic", "messages")) + assert (config["provider"], config["endpoint"]) == (provider, endpoint) + assert config["temperature"] == "omitted" and config["summaries"] == "disabled" + assert config["max_output_tokens"] == 24192 + output = capsys.readouterr().out + assert f"provider: {provider}; endpoint: {endpoint}" in output + assert "Cost guardrail: $1" in output + assert config["reasoning_effort"] == "medium" + if selected.startswith("gpt-"): + assert "thinking" not in config + assert "thinking: none" in output + else: + assert config["thinking"]["type"] == "adaptive" + assert manifest["complete"] is False and manifest["incomplete_reason"] == "preview" + assert [prompt["name"] for prompt in manifest["prompts"]] == CATEGORIES + assert (run / "snapshot.html").read_text(encoding="utf-8", errors="replace") == HTML + audit = workspace / "human-audit.xlsx" + assert manifest["human_audit_sha256"] == hashlib.sha256(audit.read_bytes()).hexdigest() + assert manifest["snapshot_sha256"] == hashlib.sha256(HTML.encode()).hexdigest() + assert manifest["pricing_sha256"] == hashlib.sha256( + (run / "artifacts/pricing.json").read_bytes() + ).hexdigest() + raw = load(run, "artifacts/raw-programmatic-findings.json") + normalized = load(run, "artifacts/normalized-programmatic-findings.json") + assert not (run / "normalized-programmatic-findings.json").exists() + instructions = (run / "README.md").read_text(encoding="utf-8") + assert "review.csv" in instructions and "--live" in instructions + assert f"report --run-dir {run}" in instructions + assert raw and [item["raw_source"] for item in normalized] == raw + assert all(item["run_id"] == run.name for item in normalized) + with (run / "review.csv").open(newline="") as stream: + rows = list(csv.DictReader(stream)) + assert [row["source_row"] for row in rows] == ["2", "3", "4", "5", "6", "7"] + assert rows[0]["actual result"] == "Vague link\nfor everyone" + assert all(row["classification"] == "llm_eligible" for row in rows) + + +@pytest.mark.parametrize("model", [MODELS[0], "gpt-6-luna"]) +def test_live_run_without_approval_or_with_unsupported_model_buys_nothing( + workspace, monkeypatch, capsys, model, +): + """Declined approval prepares a preview; invalid models create no run.""" + monkeypatch.setenv("OPENAI_API_KEY", "synthetic-openai") + monkeypatch.setattr("openai.OpenAI", lambda **_: pytest.fail("paid call")) + monkeypatch.setattr("anthropic.Anthropic", lambda **_: pytest.fail("paid call")) + with pytest.raises(SystemExit) as error: + run_cli(workspace, "--model", "gpt-4o", "--live", "--approve-live") + assert error.value.code == 2 and "invalid choice" in capsys.readouterr().err + assert not (workspace / ".model-evaluation").exists() + monkeypatch.setattr("builtins.input", lambda _: "no") + for interactive in (False, True): + monkeypatch.setattr( + "sys.stdin", SimpleNamespace(isatty=lambda interactive=interactive: interactive), + ) + assert run_cli(workspace, "--model", model, "--live") == 0 + runs = list((workspace / ".model-evaluation" / "runs").iterdir()) + assert len(runs) == 2 + for run in runs: + assert load(run, "run.json")["incomplete_reason"] == "preview" + assert not (run / "artifacts" / "raw-llm-responses.json").exists() + + +@pytest.mark.parametrize("model,thinking,effort,cost", [ + (MODELS[0], {"type": "adaptive"}, "medium", "0.003915"), + (MODELS[1], {"type": "adaptive"}, "medium", "0.1548"), + (MODELS[2], {"type": "adaptive"}, "medium", "0.0783"), +]) +def test_presets_stream_every_category_and_report_saved_prices( + workspace, monkeypatch, model, thinking, effort, cost, +): + """Each preset's request settings, final text, model, and costs persist.""" + requests, options = [], [] + monkeypatch.delenv("OPENAI_API_KEY", raising=False) + monkeypatch.setattr("openai.OpenAI", lambda **_: pytest.fail("wrong provider")) + + def create(**kwargs): + """Stream thinking, then final text, with cache usage.""" + requests.append(kwargs) + stop = "max_tokens" if len(requests) == 1 else "end_turn" + text = "[" if len(requests) == 1 else '[{"issues": ["Synthetic failure"], "location_hint": "body"}]' + usage = SimpleNamespace(input_tokens=1000, output_tokens=1, + cache_read_input_tokens=500, cache_creation_input_tokens=100) + return nullcontext(iter([ + SimpleNamespace(type="message_start", message=SimpleNamespace( + id="synthetic", model=model, type="message", usage=usage)), + SimpleNamespace(type="content_block_start", + content_block=SimpleNamespace(type="thinking", thinking="SECRET")), + SimpleNamespace(type="content_block_delta", + delta=SimpleNamespace(type="thinking_delta", thinking="SECRET")), + SimpleNamespace(type="content_block_delta", + delta=SimpleNamespace(type="signature_delta", signature="SECRET")), + SimpleNamespace(type="content_block_start", + content_block=SimpleNamespace(type="redacted_thinking", data="SECRET")), + SimpleNamespace(type="content_block_start", content_block=SimpleNamespace( + type="text", text=text[:1])), + SimpleNamespace(type="content_block_delta", + delta=SimpleNamespace(type="text_delta", text=text[1:])), + SimpleNamespace(type="message_delta", delta=SimpleNamespace(stop_reason=stop), + usage=SimpleNamespace(output_tokens=200)), + SimpleNamespace(type="message_stop"), + ])) + + def client(**kwargs): + """Record client options and expose the fake Messages API.""" + options.append(kwargs) + return SimpleNamespace(messages=SimpleNamespace(create=create)) + + monkeypatch.setattr("anthropic.Anthropic", client) + assert run_cli(workspace, "--model", model, "--live", "--approve-live") == 0 + assert options == [{"api_key": "synthetic", "max_retries": 0}] + assert len(requests) == len(CATEGORIES) + for request in requests: + assert request["model"] == model and request["thinking"] == thinking + assert request["max_tokens"] == 24192 and request["stream"] is True + assert "temperature" not in request + assert request.get("output_config") == ({"effort": effort} if effort else None) + run = only_run(workspace) + manifest = load(run, "run.json") + assert manifest["complete"] is True and manifest["estimated_cost_usd"] == cost + assert manifest["configuration"]["reasoning_effort"] == effort + raw = load(run, "artifacts/raw-llm-responses.json") + assert all(item["result"]["provider"]["response_model"] == model for item in raw) + assert raw[0]["result"]["response"] == "[" + assert raw[0]["result"]["stop_reason"] == "max_tokens" + assert all(item["result"]["response"] == '[{"issues": ["Synthetic failure"], "location_hint": "body"}]' + for item in raw[1:]) + assert "SECRET" not in json.dumps(raw) + assert "page_title" in manifest["format_failures"] + findings = load(run, "normalized-llm-findings.json") + assert findings and all(finding["model"] == model for finding in findings) + assert all(finding["problem"] == "Synthetic failure" for finding in findings) + assert all(finding["finding_id"].startswith("llm-") for finding in findings) + text = report(run) + assert model in text and run.name in text + assert f"Estimated run cost: ${Decimal(cost):.6f} USD" in text + assert manifest["usage"] == { + "input_tokens": 18000, "output_tokens": 3600, + "cached_input_tokens": 9000, "cache_creation_input_tokens": 1800, + } + + +@pytest.mark.parametrize("model,cost", [ + ("gpt-6-luna", "0.0036"), + ("gpt-6.1-sol", "0.072"), +]) +def test_gpt_presets_use_openai_key_and_save_response_identity( + workspace, monkeypatch, model, cost, +): + """GPT requests use the selected preset and preserve provider evidence for review.""" + requests, options = [], [] + monkeypatch.delenv("ANTHROPIC_API_KEY") + monkeypatch.setenv("OPENAI_API_KEY", "synthetic-openai") + monkeypatch.setattr("anthropic.Anthropic", lambda **_: pytest.fail("wrong provider")) + + def create(**kwargs): + """Return final text and reported usage through Chat Completions.""" + requests.append(kwargs) + return SimpleNamespace( + id="synthetic-gpt", model=f"{model}-provider", service_tier="default", + system_fingerprint="synthetic-fingerprint", + choices=[SimpleNamespace(message=SimpleNamespace( + content='[{"issues": ["Synthetic failure"], "location_hint": "body"}]'), + finish_reason="stop")], + usage=SimpleNamespace(prompt_tokens=1000, completion_tokens=200, + total_tokens=1200, prompt_tokens_details=None, + completion_tokens_details=SimpleNamespace(reasoning_tokens=150)), + ) + + def client(**kwargs): + """Record the SDK credentials and expose the provider request boundary.""" + options.append(kwargs) + return SimpleNamespace(chat=SimpleNamespace(completions=SimpleNamespace(create=create))) + + monkeypatch.setattr("openai.OpenAI", client) + assert run_cli(workspace, "--model", model, "--live", "--approve-live") == 0 + assert options == [{"api_key": "synthetic-openai", "max_retries": 0}] + assert len(requests) == len(CATEGORIES) + for request in requests: + assert request["model"] == model + assert request["reasoning_effort"] == "medium" + assert request["max_completion_tokens"] == 24192 + assert set(request) == {"model", "messages", "reasoning_effort", "max_completion_tokens"} + run = only_run(workspace) + manifest = load(run, "run.json") + assert manifest["complete"] is True and manifest["estimated_cost_usd"] == cost + raw = load(run, "artifacts/raw-llm-responses.json") + for item in raw: + result = item["result"] + assert result["model"] == f"{model}-provider" + assert result["provider"]["response_id"] == "synthetic-gpt" + assert result["provider"]["response_model"] == f"{model}-provider" + assert result["usage"]["provider_usage"]["completion_tokens_details"] == {"reasoning_tokens": 150} + assert result["usage"]["output_tokens"] == 200 + assert result["request"]["reasoning_effort"] == "medium" + assert "thinking" not in result["request"] + findings = load(run, "normalized-llm-findings.json") + assert findings and all(item["model"] == model for item in findings) + text = report(run) + assert f"Estimated run cost: ${Decimal(cost):.6f} USD" in text + + +@pytest.mark.parametrize("model,cost", [("gpt-6-luna", "0.0002345"), ("gpt-6.1-sol", "0.00467")]) +def test_openai_prices_cached_subsets_and_reasoning_once(workspace, monkeypatch, model, cost): + """Cache reads/writes replace ordinary input; output already includes reasoning.""" + details = SimpleNamespace(cached_tokens=200, cache_write_tokens=100) + requests = recording_provider_usage(monkeypatch, model, { + "prompt_tokens": 1000, "completion_tokens": 300, "prompt_tokens_details": details, + "completion_tokens_details": SimpleNamespace(reasoning_tokens=250), + }) + assert run_cli(workspace, "--model", model, "--live", "--approve-live") == 0 + assert len(requests) == len(CATEGORIES) + run = only_run(workspace) + manifest = load(run, "run.json") + assert manifest["estimated_cost_usd"] == cost + # Aggregate reported usage and raw provider evidence retain their existing meaning. + assert manifest["usage"] == {"input_tokens": 1000, "cached_input_tokens": 200, + "cache_creation_input_tokens": 0, "output_tokens": 300} + (first, *rest) = load(run, "artifacts/raw-llm-responses.json") + assert first["result"]["usage"]["provider_usage"]["prompt_tokens_details"] == { + "cached_tokens": 200, "cache_write_tokens": 100, + } + assert first["result"]["usage"]["provider_usage"]["completion_tokens_details"] == { + "reasoning_tokens": 250, + } + assert first["billable_usage"] == {"input_tokens": 700, "cached_input_tokens": 200, + "cache_creation_input_tokens": 100, "output_tokens": 300} + assert first["estimated_cost_usd"] == cost + assert all(item["estimated_cost_usd"] == "0" for item in rest) + + +@pytest.mark.parametrize("model,threshold,costs,total", [ + ("gpt-6-luna", 272000, ["0.0271895", "0.0543767"], "0.0815662"), + ("gpt-6.1-sol", 272000, ["0.54377", "1.087494"], "1.631264"), + ("claude-haiku-5-5", 100000, ["0.0099895", "0.049948"], "0.0599375"), +]) +def test_each_request_uses_total_input_and_inclusive_price_threshold( + workspace, monkeypatch, model, threshold, costs, total, +): + """Equality gets the lower tier; one extra token prices the whole request higher.""" + if model.startswith("gpt-"): + usages = [{"prompt_tokens": size, "completion_tokens": 10, + "prompt_tokens_details": SimpleNamespace(cached_tokens=200, cache_write_tokens=100)} + for size in (threshold, threshold + 1)] + else: + usages = [{"input_tokens": size - 300, "output_tokens": 10, + "cache_read_input_tokens": 200, "cache_creation_input_tokens": 100} + for size in (threshold, threshold + 1)] + requests = recording_provider_usage(monkeypatch, model, *usages) + assert run_cli(workspace, "--model", model, "--max-cost-usd", "10", + "--live", "--approve-live") == 0 + assert len(requests) == len(CATEGORIES) + run = only_run(workspace) + assert load(run, "run.json")["estimated_cost_usd"] == total + raw = load(run, "artifacts/raw-llm-responses.json") + assert [item["estimated_cost_usd"] for item in raw[:2]] == costs + assert [item["billable_usage"]["input_tokens"] for item in raw[:2]] == [threshold - 300, threshold - 299] + assert all(item["estimated_cost_usd"] == "0" for item in raw[2:]) + + +@pytest.mark.parametrize("limited", [False, True]) +@pytest.mark.parametrize("model,short_size,long_size,costs,total,partial,guardrail", [ + ("gpt-6-luna", 136001, 272001, + ["0.0136051", "0.0136051", "0.0544077", "0.000015"], "0.0816329", "0.0816179", "0.03"), + ("gpt-6.1-sol", 136001, 272001, + ["0.272102", "0.272102", "1.088154", "0.0003"], "1.632658", "1.632358", "0.6"), + ("claude-haiku-5-5", 50001, 100001, + ["0.0050051", "0.0050051", "0.0500255", "0.000015"], "0.0600507", "0.0600357", "0.012"), +]) +def test_mixed_request_sizes_accumulate_cost_and_drive_guardrail( + workspace, monkeypatch, model, short_size, long_size, costs, total, partial, guardrail, limited, +): + """Combined short input never triggers long rates or stops execution early.""" + input_key, output_key = (("prompt_tokens", "completion_tokens") if model.startswith("gpt-") + else ("input_tokens", "output_tokens")) + usages = [{input_key: size, output_key: 10} for size in (short_size, short_size, long_size, 100)] + requests = recording_provider_usage(monkeypatch, model, *usages) + assert run_cli(workspace, "--model", model, "--max-cost-usd", guardrail if limited else "10", + "--live", "--approve-live") == (1 if limited else 0) + run = only_run(workspace) + manifest = load(run, "run.json") + assert manifest["estimated_cost_usd"] == (partial if limited else total) + assert len(requests) == (3 if limited else len(CATEGORIES)) + assert manifest["incomplete_reason"] == ("cost guardrail exhausted" if limited else None) + raw = load(run, "artifacts/raw-llm-responses.json") + assert [item["estimated_cost_usd"] for item in raw[:4]] == (costs[:3] if limited else costs) + assert manifest["usage"]["input_tokens"] == short_size * 2 + long_size + (0 if limited else 100) + assert manifest["usage"]["output_tokens"] == (30 if limited else 40) + assert load(run, "normalized-llm-findings.json") + + +def test_stream_failure_retains_observed_tiered_cost_and_partial_usage(workspace, monkeypatch, capsys): + """A failed stream still contributes observed input, caches, and output to cost.""" + model = "claude-haiku-5-5" + requests = recording_provider_usage(monkeypatch, model, + {"input_tokens": 1000, "output_tokens": 10}, + {"input_tokens": 99701, "cache_read_input_tokens": 200, + "cache_creation_input_tokens": 100, "output_tokens": 10}, + stream_failure_at=1, + ) + assert run_cli(workspace, "--model", model, "--live", "--approve-live") == 1 + assert len(requests) == 2 + run = only_run(workspace) + manifest = load(run, "run.json") + assert manifest["incomplete_reason"] == "synthetic stream failure" + assert manifest["estimated_cost_usd"] == "0.050053" + assert manifest["usage"] == {"input_tokens": 100701, "cached_input_tokens": 200, + "cache_creation_input_tokens": 100, "output_tokens": 20} + raw = load(run, "artifacts/raw-llm-responses.json") + assert [item["estimated_cost_usd"] for item in raw] == ["0.000105", "0.049948"] + assert raw[1]["result"]["response"] == '{"issues": ["Synthetic failure"]}' + assert raw[1]["result"]["failure"]["type"] == "TimeoutError" + assert raw[1]["result"]["stop_reason"] == "end_turn" + assert raw[1]["result"]["usage"]["reasoning_tokens"] is None + assert raw[1]["result"]["usage"]["provider_usage"] == { + "input_tokens": 99701, "cache_read_input_tokens": 200, + "cache_creation_input_tokens": 100, "output_tokens": 10, + } + assert len(load(run, "normalized-llm-findings.json")) == 1 + assert "Incomplete" in refused(run, capsys) + + +@pytest.mark.parametrize("failure,calls,cost,reason", [ + (True, 2, "0.02002", "synthetic failure"), + (False, 1, "0.01001", "cost guardrail exhausted"), +]) +def test_failure_or_exhaustion_stops_and_keeps_partial_evidence( + workspace, monkeypatch, capsys, failure, calls, cost, reason, +): + """Requests stop, billed usage and findings remain, and reporting refuses.""" + usage = {"input_tokens": 100000, "output_tokens": 20} + success = {"success": True, "response": '{"issues": ["Unclear title"]}', "usage": usage} + stopped = {"success": False, "response": None, "error": "synthetic failure", "usage": usage} + prompts = fake_client(monkeypatch, success, stopped if failure else success) + limit = "1" if failure else "0.01" + assert run_cli(workspace, "--max-cost-usd", limit, "--live", "--approve-live") == 1 + assert len(prompts) == calls + run = only_run(workspace) + manifest = load(run, "run.json") + assert manifest["complete"] is False and manifest["incomplete_reason"] == reason + assert manifest["usage"]["input_tokens"] == calls * 100000 + assert manifest["estimated_cost_usd"] == cost + assert len(load(run, "artifacts/raw-llm-responses.json")) == calls + assert len(load(run, "normalized-llm-findings.json")) == 1 + assert "Incomplete" in refused(run, capsys) + + +@pytest.mark.parametrize("name,field,message", [ + ("run.json", "run_id", "does not belong"), + ("normalized-llm-findings.json", "run_id", "do not belong"), + ("artifacts/normalized-programmatic-findings.json", "page_url", "do not belong"), + ("artifacts/human-findings.json", "human_audit_sha256", "identity differs"), + ("artifacts/pricing.json", None, "pricing.json"), +]) +def test_report_rejects_unrelated_or_missing_evidence( + workspace, monkeypatch, capsys, name, field, message, +): + """Reports cannot combine evidence from different or partial runs.""" + fake_client(monkeypatch) + assert run_cli(workspace, "--live", "--approve-live") == 0 + run = only_run(workspace) + if field is None: + (run / name).unlink() + else: + evidence = load(run, name) + (evidence[0] if isinstance(evidence, list) else evidence)[field] = "unrelated" + (run / name).write_text(json.dumps(evidence)) + assert message in refused(run, capsys) + + +def test_llm_findings_follow_the_tools_rules(workspace, monkeypatch): + """Findings come from the tool's per-check rules, before filtering or deduplication.""" + def ok(response: object) -> dict: + """Script one successful response as JSON text.""" + return {"success": True, "response": json.dumps(response), "usage": {}} + + fake_client( + monkeypatch, + ok({"is_descriptive": False, "issues": ["Title is generic"]}), + ok({"structure_clear": False, "issues": ["Skips h2", "Two h1s", "No outline"], + "vague_headings": ["News", "More"]}), + # Kept although the tool's false-positive filter would drop it. + ok([{"text": "Contact", "is_clear": False, "reason": "Ambiguous", + "location_hint": "Footer"}, {"text": "Home", "is_clear": True}]), + # The table prompt asks for one object, not a list. + ok({"is_data_table": True, "caption_clear": False, + "header_clarity_issues": ["Year is vague"], "issues": ["No caption"]}), + # A list check answered with one object is unreadable. + ok({"title": "Video", "is_descriptive": False}), + # Text where a list belongs is unreadable, not one finding per character. + ok({"structure_appropriate": False, "issues": "No main"}), + ok([]), + ok([{"field_id": None, "placeholder": "Email", "reason": "Placeholder only"}]), + ok([{"legend_is_meaningful": False, "issues": []}]), + ok([{"field_id": "n", "requirement_in_label": False, + "requirement_in_instructions": False, "issues": []}]), + ok([]), + ok([{"src": "team.png", "alt": "Team", "quality": "poor", + "issues": ["Too short", "Omits names"]}]), + ok([]), + ok([]), + ok([]), + ok([{"has_accessible_name": False, "issues": [], + "reason": "No 1.1.1 failure; presentational"}]), + ok([]), + ok([{"src": "v.mp4", "has_captions_track": False, "issues": []}, + {"src": "v.mp4", "has_captions_track": False, "issues": ["No captions track"]}]), + ) + assert run_cli(workspace, "--live", "--approve-live") == 0 + run = only_run(workspace) + manifest = load(run, "run.json") + assert manifest["complete"] is True + assert manifest["format_failures"] == ["iframe_titles", "landmark_structure"] + findings = load(run, "normalized-llm-findings.json") + assert [(item["prompt"], item["problem"]) for item in findings] == [ + ("page_title", "Title is generic"), + ("heading_structure", "Skips h2"), + ("heading_structure", "Two h1s"), + ("heading_structure", "No outline"), + ("heading_structure", 'Heading "News" is vague or unclear'), + ("heading_structure", 'Heading "More" is vague or unclear'), + ("link_clarity", "Ambiguous"), + ("table_semantics", "No caption; Year is vague"), + ("placeholder_as_label", "Placeholder only"), + ("informative_alt_quality", "Too short; Omits names"), + ("media_captions", "No captions track"), + ] + assert len({item["finding_id"] for item in findings}) == len(findings) + screen_only = [item["prompt"] for item in findings if item["screen_only"]] + assert screen_only == ["table_semantics", "placeholder_as_label", "media_captions"] + link = findings[6] + assert (link["element"], link["location"]) == (' "Contact"', "Footer") + assert link["raw_source"]["text"] == "Contact" and link["wcag_evidence"] == ["2.4.4"] + assert findings[7]["raw_source"]["issues"] == ["No caption"] + vague = findings[4]["raw_source"] + assert vague["issue"] == "News" and vague["response"]["vague_headings"] == ["News", "More"] + media = findings[-1] + assert media["raw_source"]["issues"] == ["No captions track"] + edit_review( + run, *[{}] * 5, + {"llm_finding_id": media["finding_id"], + "match_reason": "No captions on the video"}, + ) + text = report(run) + assert ("LLM (media captions check; shown on screen only, not in the tool's " + "CSV report): No captions track") in text + + +def test_single_page_report_is_standalone_html(workspace, monkeypatch): + """A completed review produces offline HTML for the evaluated page and all Human findings.""" + fake_client(monkeypatch) + assert run_cli(workspace, "--source-url", "https://example.test/contact/", + "--live", "--approve-live") == 0 + run = only_run(workspace) + legacy = run / "report.md" + legacy.write_text("Superseded report") + assert main(["report", "--run-dir", str(run), "--reviewer", REVIEWER]) == 0 + document = BeautifulSoup((run / "report.html").read_text(), "html.parser") + assert "https://example.test/contact/" in document.header.get_text() + assert not legacy.exists() + assert document.style and not document.select("script[src], link[rel=stylesheet], img[src]") + table = document.find("table", attrs={"id": "findings"}) + assert [row.th.get_text(strip=True) for row in table.tbody.find_all("tr", recursive=False)] == [ + "Unclear link", "Missing destination", "Skipped heading", "Hidden menu", "Unclear claim", "No captions", + ] + assert "Vague link\nfor everyone" in document.get_text() + + +@pytest.mark.parametrize("match_count", [1, 2]) +def test_report_embeds_canonical_and_complete_matching_evidence(workspace, monkeypatch, match_count): + """Single and joint matches retain originals, provenance and escaped evidence.""" + fake_client(monkeypatch) + assert run_cli(workspace, "--live", "--approve-live") == 0 + run = only_run(workspace) + findings = load(run, "normalized-llm-findings.json") + hostile = '' + findings[0].update(problem=hostile, element="", location="Footer", checklist="Synthetic checklist 42") + (run / "normalized-llm-findings.json").write_text(json.dumps(findings)) + edit_review(run, { + "llm_finding_id": ";".join(item["finding_id"] for item in findings[:match_count]), + "match_reason": "Together they fully cover the link", "review_notes": hostile, + }, *[{}] * 5) + monkeypatch.setattr(llm_audit, "LLMRequestClient", lambda **_: pytest.fail("report made a paid request")) + assert main(["report", "--run-dir", str(run), "--reviewer", hostile]) == 0 + document = BeautifulSoup((run / "report.html").read_text(), "html.parser") + evidence = document.select_one("#findings tbody tr details") + text = evidence.get_text(" ", strip=True) + assert "Vague link\nfor everyone" in text and "Unclear link" in text + assert hostile in text and not document.select("header script, #findings script, img, [onerror]") + assert "Synthetic checklist 42" in text + assert all(item["finding_id"] in text for item in findings[:match_count]) + assert "Together they fully cover the link" in text and "Footer" in text + assert "" in text and findings[0]["page_url"] in text + matches = evidence.select('.matched') + assert len(matches) == match_count + + +def test_report_requires_complete_rendering_data(workspace, monkeypatch, capsys): + """Absent required fields cannot publish HTML or discard a legacy report.""" + fake_client(monkeypatch) + assert run_cli(workspace, "--live", "--approve-live") == 0 + run = only_run(workspace) + manifest = load(run, "run.json") + del manifest["configuration"]["model"] + (run / "run.json").write_text(json.dumps(manifest)) + legacy = run / "report.md" + legacy.write_text("Previous report") + assert "Cannot render complete report" in refused(run, capsys) + assert legacy.read_text() == "Previous report" + assert not list(run.glob(".report.html.*.tmp")) diff --git a/tests/test_evaluation_capture.py b/tests/test_evaluation_capture.py new file mode 100644 index 0000000..8db0e1a --- /dev/null +++ b/tests/test_evaluation_capture.py @@ -0,0 +1,129 @@ +"""Capture Benchmark snapshots through the public evaluator CLI.""" + +import hashlib +import json +from pathlib import Path + +import openpyxl +import pytest +import requests + +from vision_aid.evaluation.cli import main + + +def audit(path: Path, *urls: str | None) -> Path: + """Create a synthetic URL list without private Human audit evidence.""" + human_audit_excel = openpyxl.Workbook() + sheet = human_audit_excel.active + sheet.title = "URL list" + sheet.append(["#", "Web page name", "URL", "Status"]) + for number, url in enumerate(urls, start=1): + sheet.append([number, "Repeated name", url, ""]) + human_audit_excel.save(path) + return path + + +def capture(human_audit: Path, output: Path) -> int: + """Invoke capture through the public CLI with the supplied audit and output.""" + return main(["capture", "--human-audit", str(human_audit), "--output-dir", str(output)]) + + +def test_capture_saves_only_listed_urls_in_order_and_preserves_redirect_identity( + tmp_path, monkeypatch, capsys, +): + """A key-free capture saves separate HTML and a reusable relative mapping.""" + source = audit(tmp_path / "audit.xlsx", "https://example.test/old/", None, + "https://example.test/contact/") + output = tmp_path / "snapshots" + downloads = [] + + def get(url, **options): + """Record downloads and return separate HTML with a redirected URL.""" + downloads.append((url, options)) + response = requests.Response() + response.status_code = 200 + response.url = "https://example.test/redirected/" + response._content = (b"First page" if len(downloads) == 1 + else b"Second page") + return response + + monkeypatch.setattr(requests, "get", get) + monkeypatch.delenv("ANTHROPIC_API_KEY", raising=False) + monkeypatch.delenv("OPENAI_API_KEY", raising=False) + monkeypatch.setattr("anthropic.Anthropic", lambda **_: pytest.fail("provider access")) + monkeypatch.setattr("openai.OpenAI", lambda **_: pytest.fail("provider access")) + assert capture(source, output) == 0 + mapping = json.loads((output / "snapshots.json").read_text()) + assert mapping == { + "human_audit_filename": "audit.xlsx", + "human_audit_sha256": hashlib.sha256(source.read_bytes()).hexdigest(), + "snapshots": [ + {"source_url": "https://example.test/old/", "html": "page-001.html"}, + {"source_url": "https://example.test/contact/", "html": "page-002.html"}, + ], + } + assert (output / "page-001.html").read_text() == "First page" + assert (output / "page-002.html").read_text() == "Second page" + assert [url for url, _ in downloads] == [ + "https://example.test/old/", "https://example.test/contact/", + ] + assert all(options.get("allow_redirects", True) for _, options in downloads) + assert str(output / "snapshots.json") in capsys.readouterr().out + + +@pytest.mark.parametrize("status", [404, 503]) +def test_capture_stops_after_http_error_and_keeps_saved_evidence( + tmp_path, monkeypatch, capsys, status, +): + """Failed download leaves earlier HTML but no complete snapshot mapping.""" + source = audit(tmp_path / "audit.xlsx", "https://example.test/first/", + "https://example.test/failing/", "https://example.test/later/") + output = tmp_path / "snapshots" + downloads = [] + + def get(url, **options): + """Return saved HTML first, then the requested HTTP error.""" + downloads.append(url) + response = requests.Response() + response.status_code = 200 if len(downloads) == 1 else status + response.url = url + response._content = b"Saved first page" + return response + + monkeypatch.setattr(requests, "get", get) + with pytest.raises(SystemExit) as error: + capture(source, output) + assert error.value.code == 2 + assert downloads == ["https://example.test/first/", "https://example.test/failing/"] + assert {path.name for path in output.iterdir()} == {"page-001.html"} + assert (output / "page-001.html").read_text() == "Saved first page" + assert str(status) in capsys.readouterr().err + + +def test_capture_requires_a_new_directory_before_downloading(tmp_path, monkeypatch, capsys): + """Saved Benchmark snapshots cannot be replaced by another capture.""" + source = audit(tmp_path / "audit.xlsx", "https://example.test/") + output = tmp_path / "snapshots" + output.mkdir() + (output / "saved.html").write_text("Previously captured") + monkeypatch.setattr(requests, "get", lambda *_, **__: pytest.fail("download")) + with pytest.raises(SystemExit) as error: + capture(source, output) + assert error.value.code == 2 + assert (output / "saved.html").read_text() == "Previously captured" + assert "exists" in capsys.readouterr().err.lower() + + +@pytest.mark.parametrize("urls", [(), (None, " ")]) +def test_capture_rejects_an_empty_url_list_before_downloading( + tmp_path, monkeypatch, capsys, urls, +): + """An audit without targets cannot publish a complete snapshot set.""" + source = audit(tmp_path / "audit.xlsx", *urls) + output = tmp_path / "snapshots" + monkeypatch.setattr(requests, "get", lambda *_, **__: pytest.fail("download")) + with pytest.raises(SystemExit) as error: + capture(source, output) + assert error.value.code == 2 + assert not output.exists() + assert "no target URLs" in capsys.readouterr().err diff --git a/tests/test_evaluation_human_finding_mapping.py b/tests/test_evaluation_human_finding_mapping.py new file mode 100644 index 0000000..940ebec --- /dev/null +++ b/tests/test_evaluation_human_finding_mapping.py @@ -0,0 +1,369 @@ +"""Exercise approved Human audit decisions through the evaluator CLI.""" + +import csv +import json + +import pytest +from test_evaluation import ( + edit_review, + fake_client, + load, + only_run, + refused, + report, + run_cli, +) +from test_evaluation import workspace as workspace # Expose shared pytest fixture. + +URL = "https://example.test/" + + +@pytest.fixture +def approved(workspace): + """Author independent page-level expectations from a synthetic Human audit.""" + (workspace / "home.html").write_text( + '

Home

Click here' + ) + assert run_cli(workspace) == 0 + seed = only_run(workspace) + human = load(seed, "artifacts/human-findings.json") + classifications = [ + "programmatic", "llm_eligible", "llm_eligible", + "unavailable_evidence", "ambiguous", "programmatic", + ] + expectations = [ + [ + {"rule": "PAGE_TITLE_001", "evidence": { + "raw_source": {"location": {"tag": "head"}}}}, + {"rule": "LANG_001", "evidence": { + "problem": " element missing lang attribute."}}, + ], + [{"rule": "LAND_001", "evidence": { + "problem": "Page does not contain a
landmark."}}], + [], [], [], + [{"rule": "HEAD_003", "evidence": { + "problem": "The page does not contain an

element."}}], + ] + mapping = { + "schema_version": 1, + "human_audit_sha256": human["human_audit_sha256"], + "approved_by": "Independent synthetic audit review", + "decisions": [ + { + "human_finding_id": finding["human_finding_id"], + "classification": classification, + "classification_reason": f"Approved responsibility for row {index + 2}", + "programmatic_by_page": {URL: { + "expectations": matches, + "reason": "Complete page defect" if matches else "No existing rule catches this defect", + }}, + } + for index, (finding, classification, matches) in enumerate(zip( + human["human_findings"], classifications, expectations, strict=True + )) + ], + } + path = workspace / "human-finding-mapping.json" + path.write_text(json.dumps(mapping)) + return path, seed + + +def review_rows(run): + """Read the operator-facing review rows in original audit order.""" + with (run / "review.csv").open(newline="") as stream: + return sorted(csv.DictReader(stream), key=lambda row: int(row["source_row"])) + + +def human_finding_mapping_run(workspace, path, *extra): + """Prepare a mapping-backed run through the normal command.""" + root = workspace / ".model-evaluation/runs" + before = set(root.iterdir()) + assert run_cli(workspace, "--human-finding-mapping", str(path), *extra) == 0 + (run,) = set(root.iterdir()) - before + return run + + +def test_preparation_populates_approved_decisions_and_preserves_human_evidence( + workspace, approved, monkeypatch, +): + """Preview resolves joint and cross-catches and keeps source evidence intact.""" + path, seed = approved + monkeypatch.setattr("anthropic.Anthropic", lambda **_: pytest.fail("paid request")) + run = human_finding_mapping_run(workspace, path) + original = review_rows(seed) + rows = review_rows(run) + assert [row["classification"] for row in rows] == [ + "programmatic", "llm_eligible", "llm_eligible", + "unavailable_evidence", "ambiguous", "programmatic", + ] + findings = { + item["finding_id"]: item + for item in load(run, "artifacts/normalized-programmatic-findings.json") + } + title_id, lang_id = rows[0]["programmatic_finding_id"].split(";") + assert findings[title_id]["problem_family"] == "PAGE_TITLE_001" + assert findings[title_id]["raw_source"]["location"]["tag"] == "head" + assert findings[lang_id]["problem_family"] == "LANG_001" + assert rows[1]["programmatic_finding_id"] + assert all(not row["programmatic_finding_id"] for row in rows[2:5]) + assert rows[0]["programmatic_match_reason"] == "Complete page defect" + for previous, current in zip(original, rows, strict=True): + for field in ("human_finding_id", "source_sheet", "source_row", "page_scope", + "source_element", "actual result", "Issue Title"): + assert current[field] == previous[field] + assert load(run, "artifacts/human-findings.json") == load(seed, "artifacts/human-findings.json") + assert load(run, "artifacts/human-finding-mapping.json") == json.loads(path.read_text()) + + +def test_models_share_fixed_decisions_but_keep_independent_llm_review( + workspace, approved, monkeypatch, +): + """Each model resolves its own IDs; only independent LLM decisions affect its score.""" + path, _ = approved + fake_client(monkeypatch, + {"success": True, "response": '{"issues": ["Missing h1"]}', "usage": {}}, + {"success": True, "response": '[{"text": "Click here", "is_clear": false, "issues": ["Vague link"]}]', "usage": {}}, + ) + first = human_finding_mapping_run(workspace, path, "--live", "--approve-live") + fake_client(monkeypatch) + second = human_finding_mapping_run(workspace, path, "--model", "claude-sonnet-5-5", "--live", "--approve-live") + llm = load(first, "normalized-llm-findings.json") + assert len(llm) == 2 + edit_review(first, + {"llm_finding_id": llm[0]["finding_id"], "match_reason": "Same defect despite remediation wording"}, + {}, + {"llm_finding_id": llm[1]["finding_id"], "match_reason": "Full semantic match", "review_notes": "Different example remains valid"}, + ) + fixed = ["classification", "classification_reason", "programmatic_match_reason"] + first_rows, second_rows = review_rows(first), review_rows(second) + by_run = [ + {item["finding_id"]: item["problem_family"] for item in load(run, "artifacts/normalized-programmatic-findings.json")} + for run in (first, second) + ] + for a, b in zip(first_rows, second_rows, strict=True): + assert [a[key] for key in fixed] == [b[key] for key in fixed] + assert len(a["programmatic_finding_id"].split(";")) == len(b["programmatic_finding_id"].split(";")) + assert {by_run[0][identity] for identity in a["programmatic_finding_id"].split(";") if identity} == { + by_run[1][identity] for identity in b["programmatic_finding_id"].split(";") if identity + } + if a["programmatic_finding_id"]: + assert a["programmatic_finding_id"] != b["programmatic_finding_id"] + # Later deliberate shared edits must not change either saved interpretation. + mapping = json.loads(path.read_text()) + mapping["decisions"][0]["classification_reason"] = "A later manual update" + path.write_text(json.dumps(mapping)) + first_report, second_report = report(first), report(second) + assert "Independent synthetic audit review" in first_report + assert "Complete page defect" in first_report + assert "Same defect despite remediation wording" in first_report + assert "Different example remains valid" in first_report + assert "Approved responsibility for row 2" in first_report + assert "A later manual update" not in first_report + assert "LLM: 1 caught / 2 eligible" in first_report and "LLM: 0 caught / 2 eligible" in second_report + assert "Overall: 4 caught / 6 eligible" in first_report and "Overall: 3 caught / 6 eligible" in second_report + assert "Programmatic: 2 caught / 2 eligible" in first_report and "Programmatic: 2 caught / 2 eligible" in second_report + + +@pytest.mark.parametrize("column,value", [ + ("classification", "llm_eligible"), + ("classification_reason", "A run-specific reinterpretation"), + ("programmatic_finding_id", ""), + ("programmatic_match_reason", "A different justification"), + ("actual result", "Changed original Human evidence"), +]) +def test_reporting_rejects_edited_approved_decisions_and_removes_stale_report( + workspace, approved, monkeypatch, capsys, column, value, +): + """Editable review cells cannot rewrite fixed decisions or original evidence.""" + path, _ = approved + fake_client(monkeypatch) + run = human_finding_mapping_run(workspace, path, "--live", "--approve-live") + report(run) + edit_review(run, {column: value}) + message = refused(run, capsys) + assert "Human finding" in message + assert "changed" in message + + +@pytest.mark.parametrize("change,error", [ + ("audit", "Human audit identity differs"), + ("decision", "missing approved decision"), + ("reason", "needs approved classification_reason"), + ("page", "missing explicit Programmatic page decision"), + ("missing", "resolved to 0 findings"), + ("reuse", "finding ID reuse"), + ("programmatic_absence", "requires expected coverage"), + ("not_testable", "not testable"), +]) +def test_preparation_rejects_invalid_or_unresolved_approved_expectations( + workspace, approved, capsys, change, error, +): + """Missing decisions and evidence fail with useful finding/page context.""" + path, _ = approved + mapping = json.loads(path.read_text()) + if change == "audit": + mapping["human_audit_sha256"] = "unrelated-audit" + elif change == "decision": + mapping["decisions"].pop(0) + elif change == "reason": + mapping["decisions"][0]["classification_reason"] = "" + elif change == "page": + mapping["decisions"][0]["programmatic_by_page"]["https://elsewhere.test/"] = mapping["decisions"][0]["programmatic_by_page"].pop(URL) + elif change == "missing": + mapping["decisions"][0]["programmatic_by_page"][URL]["expectations"][0]["evidence"] = {"element": "unreported-element"} + elif change == "reuse": + mapping["decisions"][1]["programmatic_by_page"][URL]["expectations"] = mapping["decisions"][0]["programmatic_by_page"][URL]["expectations"] + elif change == "programmatic_absence": + mapping["decisions"][0]["programmatic_by_page"][URL]["expectations"] = [] + elif change == "not_testable": + mapping["decisions"][0]["classification"] = "unavailable_evidence" + path.write_text(json.dumps(mapping)) + with pytest.raises(SystemExit) as caught: + run_cli(workspace, "--human-finding-mapping", str(path)) + assert caught.value.code == 2 + assert error in capsys.readouterr().err + + +@pytest.mark.parametrize("change,count", [("missing", 0), ("duplicate", 2)]) +def test_reporting_requires_unique_actual_checker_evidence( + workspace, approved, monkeypatch, capsys, change, count, +): + """A saved expected catch cannot manufacture credit after evidence is lost.""" + path, _ = approved + fake_client(monkeypatch) + run = human_finding_mapping_run(workspace, path, "--live", "--approve-live") + findings = load(run, "artifacts/normalized-programmatic-findings.json") + if change == "missing": + findings = [item for item in findings if item["problem_family"] != "PAGE_TITLE_001"] + else: + duplicate = next(item.copy() for item in findings if item["problem_family"] == "PAGE_TITLE_001") + duplicate["finding_id"] = "programmatic-second-valid-id" + findings.append(duplicate) + (run / "artifacts/normalized-programmatic-findings.json").write_text(json.dumps(findings)) + message = refused(run, capsys) + assert "PAGE_TITLE_001" in message + assert "Human finding human-" in message + assert f"resolved to {count} findings" in message + + +def test_reporting_rejects_llm_credit_for_not_testable_findings( + workspace, approved, monkeypatch, capsys, +): + """LLM edits retain existing not-testable restrictions even with approved decisions.""" + path, _ = approved + fake_client(monkeypatch, + {"success": True, "response": '{"issues": ["A concrete defect"]}', "usage": {}}, + ) + run = human_finding_mapping_run(workspace, path, "--live", "--approve-live") + identity = load(run, "normalized-llm-findings.json")[0]["finding_id"] + edit_review(run, {}, {}, {}, {"llm_finding_id": identity, "match_reason": "Claimed catch"}) + assert "not testable" in refused(run, capsys) + + +def test_reporting_rejects_modified_saved_approved_source( + workspace, approved, monkeypatch, capsys, +): + """Historical interpretation is pinned to the copy saved with the run.""" + path, _ = approved + fake_client(monkeypatch) + run = human_finding_mapping_run(workspace, path, "--live", "--approve-live") + copied = run / "artifacts/human-finding-mapping.json" + mapping = json.loads(copied.read_text()) + mapping["decisions"][0]["classification_reason"] = "Rewritten history" + copied.write_text(json.dumps(mapping)) + assert "checksum changed" in refused(run, capsys) + + +@pytest.mark.parametrize("retain_local_source", [False, True]) +def test_bundled_human_finding_mapping_is_automatic_and_cannot_be_removed_to_unlock_reporting( + workspace, approved, monkeypatch, capsys, retain_local_source, +): + """Known audits automatically use approved decisions and require saved approval.""" + path, _ = approved + mapping = json.loads(path.read_text()) + bundled = workspace / "bundled-mappings" + bundled.mkdir() + (bundled / f"{mapping['human_audit_sha256']}.json").write_text(path.read_text()) + monkeypatch.setattr("vision_aid.evaluation.human_finding_mapping.HUMAN_FINDING_MAPPING_DIRECTORY", bundled) + fake_client(monkeypatch) + before = set((workspace / ".model-evaluation/runs").iterdir()) + assert run_cli(workspace, "--live", "--approve-live") == 0 + (run,) = set((workspace / ".model-evaluation/runs").iterdir()) - before + assert review_rows(run)[0]["classification"] == "programmatic" + report(run) + (run / "artifacts/human-finding-mapping.json").unlink() + manifest = load(run, "run.json") + manifest.pop("human_finding_mapping_sha256") + manifest.pop("human_finding_mapping_approved_by") + if not retain_local_source: + (bundled / f"{mapping['human_audit_sha256']}.json").unlink() + (run / "run.json").write_text(json.dumps(manifest)) + assert "prepare a new run with --human-finding-mapping PATH" in refused(run, capsys) + + +@pytest.mark.parametrize("change,error", [ + ("unknown", "Unknown LLM finding ID"), + ("source", "Unknown LLM finding ID"), + ("reuse", "Conflicting LLM finding ID"), +]) +def test_human_finding_mapping_does_not_relax_llm_identity_or_reuse_validation( + workspace, approved, monkeypatch, capsys, change, error, +): + """Editable LLM choices must still belong to this run and one Human finding.""" + path, _ = approved + fake_client(monkeypatch, + {"success": True, "response": '{"issues": ["A concrete defect"]}', "usage": {}}, + ) + run = human_finding_mapping_run(workspace, path, "--live", "--approve-live") + llm = load(run, "normalized-llm-findings.json")[0]["finding_id"] + if change == "unknown": + identity = "llm-another-run" + elif change == "source": + identity = review_rows(run)[0]["programmatic_finding_id"].split(";")[0] + else: + identity = llm + edit_review(run, + {"llm_finding_id": identity, "match_reason": "Claimed semantic match"}, + {"llm_finding_id": llm, "match_reason": "Reused match"} if change == "reuse" else {}, + ) + assert error in refused(run, capsys) + + +def test_single_page_uses_listed_page_local_findings_and_globals( + workspace, capsys, +): + """A non-home URL receives its own Human findings and approved page expectations.""" + import openpyxl + + careers = "https://example.test/careers/" + human_audit_excel = openpyxl.load_workbook(workspace / "human-audit.xlsx") + sheet = human_audit_excel["Defect report"] + sheet.cell(2, 6, f"Open {URL}") + sheet.cell(8, 6, f"Open {careers}") + urls = human_audit_excel.create_sheet("URL list") + urls.append(["URL"]) + urls.append([URL]) + urls.append([careers]) + human_audit_excel.save(workspace / "human-audit.xlsx") + human_audit_excel.close() + assert run_cli(workspace, "--source-url", careers) == 0 + seed = only_run(workspace) + human = load(seed, "artifacts/human-findings.json") + assert [finding["source_row"] for finding in human["human_findings"]] == [3, 4, 5, 6, 7, 8] + path = workspace / "careers-human-finding-mapping.json" + path.write_text(json.dumps({ + "schema_version": 1, "human_audit_sha256": human["human_audit_sha256"], + "approved_by": "Non-home reviewer", "decisions": [ + { + "human_finding_id": item["human_finding_id"], + "classification": "llm_eligible", "classification_reason": "Model judgment required", + "programmatic_by_page": {careers: {"expectations": [], "reason": "No complete checker catch"}}, + } for item in human["human_findings"] + ], + })) + run = human_finding_mapping_run(workspace, path, "--source-url", careers) + assert [row["source_row"] for row in review_rows(run)] == ["3", "4", "5", "6", "7", "8"] + assert all(row["classification"] == "llm_eligible" for row in review_rows(run)) + with pytest.raises(SystemExit): + run_cli(workspace, "--source-url", "https://example.test/unlisted/") + assert "not listed in the Human audit" in capsys.readouterr().err diff --git a/tests/test_full_execution.py b/tests/test_full_execution.py new file mode 100644 index 0000000..8c6e39c --- /dev/null +++ b/tests/test_full_execution.py @@ -0,0 +1,254 @@ +"""Execute Full evaluations through the CLI with controlled provider replies.""" + +import json + +import pytest +from test_evaluation import recording_provider_usage +from test_full_preview import URLS, full_workspace, only_full, preview + +# Re-export the shared fixture for pytest discovery. +__all__ = ["full_workspace"] + + +def children(run, manifest): + """Load each page's child directory and manifest in evaluation order.""" + return [(run / page["run_dir"], json.loads((run / page["run_dir"] / "run.json").read_text())) + for page in manifest["pages"]] + + +def test_declined_full_approval_makes_no_requests_after_showing_all_pages( + full_workspace, monkeypatch, capsys, +): + """One refusal covers the whole prepared scope, even with credentials present.""" + monkeypatch.setenv("ANTHROPIC_API_KEY", "synthetic") + monkeypatch.setattr("sys.stdin.isatty", lambda: True) + approvals = [] + + def decline(prompt): + """Verify the prepared scope is displayed before declining approval.""" + approvals.append(prompt) + run, manifest = only_full(full_workspace) + assert all((child / "review.csv").exists() for child, _ in children(run, manifest)) + displayed = capsys.readouterr().out + assert all(url in displayed for url in URLS) + assert "Pricing:" in displayed and "Cost guardrail: $2" in displayed + return "no" + + monkeypatch.setattr("builtins.input", decline) + assert preview(full_workspace, "--live") == 0 + assert len(approvals) == 1 + _, manifest = only_full(full_workspace) + assert not manifest["complete"] and manifest["estimated_cost_usd"] == "0" + assert "declined" in capsys.readouterr().out + + +@pytest.mark.parametrize("model", ["claude-haiku-5-5", "gpt-6-luna"]) +def test_one_approval_executes_ordered_pages_with_one_fixed_preset( + full_workspace, monkeypatch, capsys, model, +): + """Each child's requests use the approved model and evidence belongs to its page.""" + monkeypatch.setenv("ANTHROPIC_API_KEY", "synthetic") + requests = recording_provider_usage(monkeypatch, model) + monkeypatch.setattr("sys.stdin.isatty", lambda: True) + approvals = [] + + def approve(prompt): + """Approve once after every page is prepared and before requests begin.""" + assert not requests + approvals.append(prompt) + run, manifest = only_full(full_workspace) + assert all((child / "snapshot.html").exists() for child, _ in children(run, manifest)) + return "yes" + + monkeypatch.setattr("builtins.input", approve) + assert preview(full_workspace, "--live", "--model", model) == 0 + run, manifest = only_full(full_workspace) + saved = children(run, manifest) + assert not (run / "report.html").exists() + assert len(approvals) == 1 and manifest["complete"] + assert manifest["incomplete_reason"] is None + assert len(requests) == sum(len(child["prompts"]) for _, child in saved) > 0 + assert all(request["model"] == model for request in requests) + identities = set() + for child, child_manifest in saved: + assert child_manifest["complete"] + assert child_manifest["configuration"] == manifest["configuration"] + assert not (child / "report.md").exists() + assert not (child / "report.html").exists() + assert (child / "artifacts/raw-llm-responses.json").exists() + for finding in json.loads((child / "normalized-llm-findings.json").read_text()): + assert finding["page_url"] == child_manifest["source_url"] + assert finding["run_id"] == child.name + assert finding["finding_id"] not in identities + identities.add(finding["finding_id"]) + assert "Full evaluation complete" in capsys.readouterr().out + + +def test_allowance_exhausted_between_pages_keeps_later_pages_unexecuted( + full_workspace, monkeypatch, +): + """Finishing one page consumes the single allowance rather than resetting it.""" + requests = recording_provider_usage(monkeypatch, "gpt-6-luna", + {"completion_tokens": 1_000_000}, + {"completion_tokens": 1_000_000}) + assert preview(full_workspace, "--model", "gpt-6-luna", "--max-cost-usd", "1", + "--live", "--approve-live") == 1 + run, manifest = only_full(full_workspace) + assert len(requests) == 2 + assert not manifest["complete"] and "exhausted" in manifest["incomplete_reason"] + assert manifest["estimated_cost_usd"] == "1" + assert manifest["usage"]["output_tokens"] == 2_000_000 + assert [page["status"] for page in manifest["pages"]] == [ + "complete", "not_executed", "not_executed", + ] + assert [child["complete"] for _, child in children(run, manifest)] == [True, False, False] + assert not (run / manifest["pages"][1]["run_dir"] / "normalized-llm-findings.json").exists() + + +def test_partial_page_consumes_only_remaining_allowance(full_workspace, monkeypatch): + """A second page cannot repeat the first page's full spending allowance.""" + requests = recording_provider_usage(monkeypatch, "gpt-6-luna", + *[{"completion_tokens": 500_000}] * 6) + assert preview(full_workspace, "--model", "gpt-6-luna", "--max-cost-usd", "0.75", + "--live", "--approve-live") == 1 + run, manifest = only_full(full_workspace) + saved = children(run, manifest) + assert len(requests) == 3 + assert manifest["estimated_cost_usd"] == "0.75" + assert manifest["usage"]["output_tokens"] == 1_500_000 + assert [page["status"] for page in manifest["pages"]] == [ + "complete", "incomplete", "not_executed", + ] + assert saved[1][1]["max_cost_usd"] == "0.25" + assert saved[1][1]["estimated_cost_usd"] == "0.25" + assert saved[1][1]["incomplete_reason"] == "cost guardrail exhausted" + assert len(json.loads((saved[1][0] / "artifacts/raw-llm-responses.json").read_text())) == 1 + + +def test_failed_stream_usage_is_retained_and_later_pages_stop(full_workspace, monkeypatch): + """A request failure retains reported usage and partial evidence alongside prior pages.""" + monkeypatch.setenv("ANTHROPIC_API_KEY", "synthetic") + requests = recording_provider_usage( + monkeypatch, "claude-haiku-5-5", + {"output_tokens": 1_000}, {"output_tokens": 1_000}, + {"input_tokens": 100, "output_tokens": 2_000}, stream_failure_at=2, + ) + assert preview(full_workspace, "--live", "--approve-live") == 1 + run, manifest = only_full(full_workspace) + saved = children(run, manifest) + assert len(requests) == 3 + assert manifest["estimated_cost_usd"] == "0.00201" + assert manifest["usage"]["input_tokens"] == 100 + assert manifest["usage"]["output_tokens"] == 4_000 + assert "synthetic stream failure" in manifest["incomplete_reason"] + assert [page["status"] for page in manifest["pages"]] == [ + "complete", "incomplete", "not_executed", + ] + assert saved[0][1]["complete"] + assert (saved[0][0] / "normalized-llm-findings.json").exists() + assert (saved[1][0] / "normalized-llm-findings.json").exists() + raw = json.loads((saved[1][0] / "artifacts/raw-llm-responses.json").read_text()) + assert not raw[0]["result"]["success"] + assert raw[0]["result"]["usage"]["output_tokens"] == 2_000 + + +@pytest.mark.parametrize("last_request", [1, 5]) +def test_overshoot_completes_only_when_no_more_work_remains( + full_workspace, monkeypatch, last_request, +): + """The underway request may overshoot; completion depends on remaining work.""" + usages = [{"completion_tokens": 0}] * last_request + [{"completion_tokens": 1_000_000}] + requests = recording_provider_usage(monkeypatch, "gpt-6-luna", *usages) + expected = last_request == 5 + assert preview(full_workspace, "--model", "gpt-6-luna", "--max-cost-usd", "0.1", + "--live", "--approve-live") == (0 if expected else 1) + run, manifest = only_full(full_workspace) + assert len(requests) == last_request + 1 + assert manifest["complete"] is expected + assert manifest["estimated_cost_usd"] == "0.5" + assert children(run, manifest)[0][1]["complete"] + if expected: + assert all(page["complete"] for page in manifest["pages"]) + else: + assert "exhausted" in manifest["incomplete_reason"] + + +def test_fresh_full_attempt_repeats_requests_with_new_child_identities( + full_workspace, monkeypatch, +): + """Saved HTML is reused, while every new execution starts with fresh requests.""" + requests = recording_provider_usage(monkeypatch, "gpt-6-luna") + assert preview(full_workspace, "--model", "gpt-6-luna", "--live", "--approve-live") == 0 + first, first_manifest = only_full(full_workspace) + first_count = len(requests) + assert preview(full_workspace, "--model", "gpt-6-luna", "--live", "--approve-live") == 0 + runs = list((full_workspace / ".model-evaluation/full-runs").iterdir()) + assert len(runs) == 2 and len(requests) == first_count * 2 + second = next(run for run in runs if run != first) + second_manifest = json.loads((second / "full.json").read_text()) + assert {page["run_id"] for page in first_manifest["pages"]}.isdisjoint( + page["run_id"] for page in second_manifest["pages"] + ) + + +def test_provider_initialization_failure_records_stopping_reason(full_workspace, monkeypatch): + """Failed provider setup retains prepared evidence and identifies unfinished pages.""" + monkeypatch.setenv("ANTHROPIC_API_KEY", "synthetic") + + def fail(**options): + """Simulate failure while initializing the selected provider.""" + raise ValueError("synthetic provider initialization failure") + + monkeypatch.setattr("anthropic.Anthropic", fail) + assert preview(full_workspace, "--live", "--approve-live") == 1 + run, manifest = only_full(full_workspace) + assert manifest["estimated_cost_usd"] == "0" + assert "synthetic provider initialization failure" in manifest["incomplete_reason"] + assert [page["status"] for page in manifest["pages"]] == [ + "incomplete", "not_executed", "not_executed", + ] + assert "synthetic provider initialization failure" in children(run, manifest)[0][1]["incomplete_reason"] + + +def test_noninteractive_approval_requires_live_before_preparing(full_workspace, capsys): + """An approval flag alone cannot create or authorize a Full evaluation.""" + with pytest.raises(SystemExit) as error: + preview(full_workspace, "--approve-live") + assert error.value.code == 2 + assert "--approve-live requires --live" in capsys.readouterr().err + assert not (full_workspace / ".model-evaluation").exists() + + +@pytest.mark.parametrize("model,key", [ + ("gpt-6-luna", "OPENAI_API_KEY"), ("claude-haiku-5-5", "ANTHROPIC_API_KEY"), +]) +def test_live_requires_the_selected_providers_credentials( + full_workspace, monkeypatch, capsys, model, key, +): + """The other provider's credentials cannot authorize this fixed preset.""" + monkeypatch.setenv(key, "") + monkeypatch.setenv("ANTHROPIC_API_KEY" if key == "OPENAI_API_KEY" else "OPENAI_API_KEY", + "synthetic-other-provider") + with pytest.raises(SystemExit) as error: + preview(full_workspace, "--model", model, "--live", "--approve-live") + assert error.value.code == 2 + assert f"requires {key}" in capsys.readouterr().err + run, manifest = only_full(full_workspace) + assert not manifest["complete"] + assert all((child / "review.csv").exists() for child, _ in children(run, manifest)) + + +def test_parent_sums_all_reported_usage_categories(full_workspace, monkeypatch): + """Cache usage remains a separate provider-reported category in parent totals.""" + monkeypatch.setenv("ANTHROPIC_API_KEY", "synthetic") + recording_provider_usage(monkeypatch, "claude-haiku-5-5", { + "input_tokens": 100, "output_tokens": 1_000, + "cache_read_input_tokens": 1_000, "cache_creation_input_tokens": 1_000, + }) + assert preview(full_workspace, "--live", "--approve-live") == 0 + _, manifest = only_full(full_workspace) + assert manifest["usage"] == { + "input_tokens": 100, "output_tokens": 1_000, + "cached_input_tokens": 1_000, "cache_creation_input_tokens": 1_000, + } + assert manifest["estimated_cost_usd"] == "0.000645" diff --git a/tests/test_full_human_finding_mapping.py b/tests/test_full_human_finding_mapping.py new file mode 100644 index 0000000..a05c829 --- /dev/null +++ b/tests/test_full_human_finding_mapping.py @@ -0,0 +1,293 @@ +"""Enforce shared approved decisions through the Full evaluation CLI.""" + +import hashlib +import json + +import openpyxl +import pytest +from bs4 import BeautifulSoup +from test_evaluation import recording_provider_usage +from test_full_execution import children +from test_full_preview import URLS, full_workspace, only_full, preview +from test_full_report import ( + refused_report, + report_all, + review_rows, + save_json, + save_review, +) + +__all__ = ["full_workspace"] +HUMAN_FINDING_MAPPING_ARTIFACT = "artifacts/human-finding-mapping.json" + + +def load(path): + """Read saved operator-visible evidence.""" + return json.loads(path.read_text()) + + +@pytest.fixture +def full_approved(full_workspace): + """Eight originals: two Programmatic, four LLM, and two not testable.""" + path = full_workspace / "human-audit.xlsx" + human_audit_excel = openpyxl.load_workbook(path) + sheet = human_audit_excel["Defect report"] + for number, title in enumerate(("Shared model defect", "Unclear defect", "Partial defect"), start=6): + sheet.append([number, "Element", "Chrome", "Global", title, f"Open {URLS[0]}", + "Original\nevidence", "Expected", "", "1.1.1"]) + human_audit_excel.save(path) + human_audit_excel.close() + mapping_path = full_workspace / "snapshots/snapshots.json" + mapping = load(mapping_path) + mapping["human_audit_sha256"] = hashlib.sha256(path.read_bytes()).hexdigest() + save_json(mapping_path, mapping) + for number in range(1, 4): + (full_workspace / f"snapshots/page-{number:03d}.html").write_text( + '' + ) + assert preview(full_workspace) == 0 + seed, _ = only_full(full_workspace) + original = load(seed / "human-findings.json") + decisions = [] + for finding in original["human_findings"]: + title = finding["problem"] + classification = { + "First local": "programmatic", "Shared defect": "programmatic", + "Shared second defect": "unavailable_evidence", "Unclear defect": "ambiguous", + }.get(title, "llm_eligible") + pages = URLS if finding["page_scope"] == "global" else [ + original["page_assignments"][finding["human_finding_id"]] + ] + expectations = { + "First local": [ + {"rule": "PAGE_TITLE_001", "evidence": {"raw_source": {"location": {"tag": "head"}}}}, + {"rule": "LANG_001", "evidence": {"problem": " element missing lang attribute."}}, + ], + "Shared defect": [{"rule": "HEAD_003", "evidence": { + "problem": "The page does not contain an

element."}}], + "Contact local": [{"rule": "LAND_001", "evidence": { + "problem": "Page does not contain a
landmark."}}], + }.get(title, []) + decisions.append({ + "human_finding_id": finding["human_finding_id"], "classification": classification, + "classification_reason": f"Approved responsibility: {title}", + "programmatic_by_page": { + url: {"expectations": expectations, "reason": f"Approved complete evidence: {title}"} + for url in pages + }, + }) + mapping = {"schema_version": 1, "human_audit_sha256": original["human_audit_sha256"], + "approved_by": "Independent Full audit reviewer", "decisions": decisions} + human_finding_mapping_path = full_workspace / "approved-human-finding-mapping.json" + save_json(human_finding_mapping_path, mapping) + return human_finding_mapping_path, seed + + +def approved_run(workspace, mapping, *extra): + """Create a separate Full evaluation with the selected approved decisions.""" + root = workspace / ".model-evaluation/full-runs" + before = set(root.iterdir()) + assert preview(workspace, "--human-finding-mapping", str(mapping), *extra) == 0 + (run,) = set(root.iterdir()) - before + return run, load(run / "full.json") + + +def test_full_preparation_saves_one_human_finding_mapping_and_resolves_each_pages_own_ids( + full_workspace, full_approved, +): + """Every child, including a Globals-only page, uses the parent's frozen source.""" + path, seed = full_approved + run, manifest = approved_run(full_workspace, path) + mapping = load(path) + assert load(run / HUMAN_FINDING_MAPPING_ARTIFACT) == mapping + assert manifest["human_finding_mapping_sha256"] == hashlib.sha256((run / HUMAN_FINDING_MAPPING_ARTIFACT).read_bytes()).hexdigest() + assert load(run / "human-findings.json") == load(seed / "human-findings.json") + previous = load(seed / "full.json") + ids = set() + for (child, saved), old_page in zip(children(run, manifest), previous["pages"], strict=True): + assert load(child / HUMAN_FINDING_MAPPING_ARTIFACT) == mapping + assert saved["human_finding_mapping_sha256"] == manifest["human_finding_mapping_sha256"] + assert load(child / "artifacts/human-findings.json") == load(seed / old_page["run_dir"] / "artifacts/human-findings.json") + local_ids = {item["finding_id"] for item in load(child / "artifacts/normalized-programmatic-findings.json")} + rows = review_rows(child) + assert all(row["classification"] and row["classification_reason"] for row in rows) + selected = {identity for row in rows for identity in row["programmatic_finding_id"].split(";") if identity} + assert selected <= local_ids + assert not ids.intersection(selected) + ids.update(selected) + joint = next((row for row in rows if row["Issue Title"] == "First local"), None) + if joint: + assert len(joint["programmatic_finding_id"].split(";")) == 2 + globals_only = review_rows(run / manifest["pages"][2]["run_dir"]) + assert len(globals_only) == 5 and all(row["page_scope"] == "global" for row in globals_only) + + +@pytest.fixture +def completed_approved(full_workspace, full_approved, monkeypatch): + """Review one complete Global LLM catch while keeping partial evidence uncredited.""" + path, _ = full_approved + recording_provider_usage(monkeypatch, "gpt-6-luna") + run, manifest = approved_run(full_workspace, path, "--model", "gpt-6-luna", "--live", "--approve-live") + for number, (child, _) in enumerate(children(run, manifest)): + rows = review_rows(child) + llm = load(child / "normalized-llm-findings.json") + for row in rows: + if row["Issue Title"] == "Shared model defect" and number == 1: + row.update(llm_finding_id=llm[0]["finding_id"], match_reason="Complete semantic Global match") + elif row["Issue Title"] == "Partial defect": + row["review_notes"] = f"Partial evidence only on page {number + 1}" + save_review(child, rows) + return run, manifest + + +def test_combined_reporting_rejects_a_child_with_different_saved_approved_decisions( + completed_approved, capsys, +): + """Even a locally valid child cannot substitute another audit-wide decision source.""" + run, manifest = completed_approved + assert report_all(run) == 0 + child = run / manifest["pages"][2]["run_dir"] + path = child / HUMAN_FINDING_MAPPING_ARTIFACT + mapping = load(path) + # This local finding does not appear on child three, so its review still validates. + mapping["decisions"][0]["classification_reason"] = "Different approved responsibility" + path.write_text(json.dumps(mapping, ensure_ascii=False, indent=2) + "\n") + saved = load(child / "run.json") + saved["human_finding_mapping_sha256"] = hashlib.sha256(path.read_bytes()).hexdigest() + save_json(child / "run.json", saved) + assert "human finding mapping" in refused_report(run, capsys).lower() + + +def rates(document): + """Read the report's public source counts rather than reconstructing scoring.""" + overview = document.find("figure", attrs={"aria-labelledby": "detection-chart-title"}) + return [node.get_text(" ", strip=True) for node in overview.select(".chart-value")] + + +def test_combined_and_child_reports_keep_complete_matches_and_source_denominators(completed_approved): + """Two P catches, a P cross-catch and one complete LLM Global catch cover four originals.""" + run, manifest = completed_approved + assert report_all(run) == 0 + document = BeautifulSoup((run / "report.html").read_text(), "html.parser") + assert rates(document) == [ + "LLM: 1 caught / 4 eligible; 25.0%", + "Programmatic: 2 caught / 2 eligible; 100.0%", + "Overall: 4 caught / 8 eligible; 50.0%", + ] + child_counts = [ + ["LLM: 0 caught / 3 eligible; 0.0%", "Programmatic: 2 caught / 2 eligible; 100.0%", "Overall: 2 caught / 7 eligible; 28.6%"], + ["LLM: 1 caught / 3 eligible; 33.3%", "Programmatic: 1 caught / 1 eligible; 100.0%", "Overall: 3 caught / 6 eligible; 50.0%"], + ["LLM: 0 caught / 2 eligible; 0.0%", "Programmatic: 1 caught / 1 eligible; 100.0%", "Overall: 1 caught / 5 eligible; 20.0%"], + ] + text = document.get_text(" ", strip=True) + assert "Independent Full audit reviewer" in text and "Combined reviewer" in text + assert "Complete semantic Global match" in text + assert "Approved complete evidence: First local" in text + assert len(document.select("#findings > tbody > tr")) == 8 + assert not document.select("script[src], link[rel=stylesheet], iframe") + for (child, _), expected in zip(children(run, manifest), child_counts, strict=True): + page = BeautifulSoup((child / "report.html").read_text(), "html.parser") + assert rates(page) == expected + assert "Independent Full audit reviewer" in page.get_text() + assert not page.select("script[src], link[rel=stylesheet], iframe") + programmatic = {item["finding_id"]: item for item in load(child / "artifacts/normalized-programmatic-findings.json")} + for row in review_rows(child): + for identity in row["programmatic_finding_id"].split(";"): + if identity: + assert identity in text + assert programmatic[identity]["problem"] in text + assert programmatic[identity]["run_id"] in text + + +@pytest.mark.parametrize("change,error", [ + ("classification", "decision changed: classification"), + ("classification_reason", "decision changed: classification_reason"), + ("programmatic_finding_id", "decision changed: programmatic_finding_id"), + ("missing_evidence", "resolved to 0 findings"), + ("foreign_id", "decision changed: programmatic_finding_id"), + ("missing_child_mapping", "Human finding mapping"), + ("missing_parent_mapping", "Human finding mapping"), + ("parent_source", "checksum changed"), +]) +def test_combined_report_rejects_changed_approved_decisions_or_missing_expected_evidence( + completed_approved, capsys, change, error, +): + """Every failure removes previously successful combined rates before returning an error.""" + run, manifest = completed_approved + assert report_all(run) == 0 + (run / "report.md").write_text("superseded report") + child = run / manifest["pages"][0]["run_dir"] + rows = review_rows(child) + if change == "missing_evidence": + path = child / "artifacts/normalized-programmatic-findings.json" + save_json(path, [item for item in load(path) if item["problem_family"] != "PAGE_TITLE_001"]) + elif change in {"missing_child_mapping", "missing_parent_mapping"}: + target = child if change == "missing_child_mapping" else run + (target / HUMAN_FINDING_MAPPING_ARTIFACT).unlink() + elif change == "parent_source": + path = run / HUMAN_FINDING_MAPPING_ARTIFACT + path.write_text(path.read_text() + " ") + else: + if change == "foreign_id": + other = run / manifest["pages"][1]["run_dir"] + rows[0]["programmatic_finding_id"] = load(other / "artifacts/normalized-programmatic-findings.json")[0]["finding_id"] + else: + rows[0][change] = "llm_eligible" if change == "classification" else "" + save_review(child, rows) + assert error.lower() in refused_report(run, capsys).lower() + + +def test_different_models_keep_equal_combined_programmatic_counts_with_independent_llm_findings( + full_workspace, full_approved, completed_approved, monkeypatch, +): + """Run-local IDs and a genuine model miss change LLM rates while checker rates agree.""" + first, first_manifest = completed_approved + path, _ = full_approved + + recording_provider_usage(monkeypatch, "gpt-6.1-sol", text='{"issues": []}') + second, second_manifest = approved_run(full_workspace, path, "--model", "gpt-6.1-sol", "--live", "--approve-live") + # Changing the original selected file does not rewrite either saved interpretation. + source = load(path) + source["decisions"][0]["classification_reason"] = "Future manual update" + save_json(path, source) + for (a, _), (b, _) in zip(children(first, first_manifest), children(second, second_manifest), strict=True): + a_findings = load(a / "artifacts/normalized-programmatic-findings.json") + b_findings = load(b / "artifacts/normalized-programmatic-findings.json") + assert {item["finding_id"] for item in a_findings}.isdisjoint(item["finding_id"] for item in b_findings) + assert load(a / "normalized-llm-findings.json") + assert load(b / "normalized-llm-findings.json") == [] + for left, right in zip(review_rows(a), review_rows(b), strict=True): + for field in ("classification", "classification_reason", "programmatic_match_reason"): + assert left[field] == right[field] + lookup_a = {item["finding_id"]: item["problem_family"] for item in a_findings} + lookup_b = {item["finding_id"]: item["problem_family"] for item in b_findings} + assert {lookup_a[identity] for identity in left["programmatic_finding_id"].split(";") if identity} == { + lookup_b[identity] for identity in right["programmatic_finding_id"].split(";") if identity + } + assert report_all(first) == report_all(second) == 0 + first_report = BeautifulSoup((first / "report.html").read_text(), "html.parser") + second_report = BeautifulSoup((second / "report.html").read_text(), "html.parser") + assert rates(first_report)[1] == rates(second_report)[1] == "Programmatic: 2 caught / 2 eligible; 100.0%" + assert rates(first_report)[0] == "LLM: 1 caught / 4 eligible; 25.0%" + assert rates(second_report)[0] == "LLM: 0 caught / 4 eligible; 0.0%" + assert rates(second_report)[2] == "Overall: 3 caught / 8 eligible; 37.5%" + assert "Future manual update" not in first_report.get_text() + second_report.get_text() + + +def test_full_preparation_automatically_selects_the_audits_bundled_human_finding_mapping( + full_workspace, full_approved, monkeypatch, +): + """The audit checksum selects shared decisions once for all pages without a flag.""" + path, _ = full_approved + mapping = load(path) + directory = full_workspace / "bundled-mappings" + directory.mkdir() + (directory / f"{mapping['human_audit_sha256']}.json").write_bytes(path.read_bytes()) + monkeypatch.setattr("vision_aid.evaluation.human_finding_mapping.HUMAN_FINDING_MAPPING_DIRECTORY", directory) + before = set((full_workspace / ".model-evaluation/full-runs").iterdir()) + assert preview(full_workspace) == 0 + (run,) = set((full_workspace / ".model-evaluation/full-runs").iterdir()) - before + manifest = load(run / "full.json") + assert load(run / HUMAN_FINDING_MAPPING_ARTIFACT) == mapping + assert all(saved["human_finding_mapping_sha256"] == manifest["human_finding_mapping_sha256"] + for _, saved in children(run, manifest)) diff --git a/tests/test_full_preview.py b/tests/test_full_preview.py new file mode 100644 index 0000000..3d1e3a1 --- /dev/null +++ b/tests/test_full_preview.py @@ -0,0 +1,244 @@ +"""Preview Full evaluations through the public evaluator CLI.""" + +import csv +import hashlib +import json + +import openpyxl +import pytest +from evaluation_fixtures import run_with_mapping + +from vision_aid.evaluation.cli import main +from vision_aid.evaluation.inputs.human_audit import HEADERS + +URLS = ["https://example.test/", "https://example.test/contact/", + "https://example.test/empty/"] +HTML = 'Test' + + +@pytest.fixture +def full_workspace(tmp_path, monkeypatch): + """Save a synthetic Human audit and reusable captured snapshot mapping.""" + human_audit_excel = openpyxl.Workbook() + sheet = human_audit_excel.active + sheet.title = "Defect report" + sheet.append(HEADERS) + for number, page_name, title, steps in ( + (1, "Repeated name", "First local", f"Open {URLS[0]}"), + (2, "Repeated name", "Second local", f"Open {URLS[0]}"), + (3, "Home", "Contact local", "Open https://example.test/Contact/."), + (4, "Global", "Shared defect", f"Open {URLS[0]}"), + (5, "Global", "Shared second defect", f"Open {URLS[0]}"), + ): + sheet.append([number, "Element", "Chrome", page_name, title, steps, + "Original\nevidence", "Expected", f"Read {URLS[2]}", "1.1.1"]) + urls = human_audit_excel.create_sheet("URL list") + urls.append(["Page name", "URL"]) + for url in URLS: + urls.append(["Repeated name", url]) + human_audit_excel.save(tmp_path / "human-audit.xlsx") + saved = tmp_path / "snapshots" + saved.mkdir() + snapshots = [] + for number, url in enumerate(URLS, start=1): + filename = f"page-{number:03d}.html" + (saved / filename).write_text(HTML) + snapshots.append({"source_url": url, "html": filename}) + mapping = { + "human_audit_filename": "human-audit.xlsx", + "human_audit_sha256": hashlib.sha256((tmp_path / "human-audit.xlsx").read_bytes()).hexdigest(), + "snapshots": snapshots, + } + (saved / "snapshots.json").write_text(json.dumps(mapping)) + monkeypatch.chdir(tmp_path) + monkeypatch.delenv("ANTHROPIC_API_KEY", raising=False) + monkeypatch.delenv("OPENAI_API_KEY", raising=False) + monkeypatch.setattr("anthropic.Anthropic", lambda **_: pytest.fail("model request")) + monkeypatch.setattr("openai.OpenAI", lambda **_: pytest.fail("model request")) + monkeypatch.setattr("requests.get", lambda *_, **__: pytest.fail("HTTP download")) + return tmp_path + + +def preview(workspace, *extra): + """Preview with approved fixture decisions retained in every saved run.""" + return run_with_mapping(workspace, [ + "run-all", "--human-audit", str(workspace / "human-audit.xlsx"), + "--snapshots", str(workspace / "snapshots/snapshots.json"), + "--max-cost-usd", "2", *extra, + ]) + + +def only_full(workspace): + """Read the only saved Full evaluation manifest.""" + (run,) = (workspace / ".model-evaluation/full-runs").iterdir() + return run, json.loads((run / "full.json").read_text()) + + +@pytest.mark.parametrize("model", ["claude-haiku-5-5", "gpt-6-luna"]) +def test_preview_maps_original_findings_and_keeps_page_evidence(full_workspace, capsys, model): + """URL assignment, Globals and separate saved evidence survive key-free preview.""" + assert preview(full_workspace, "--model", model) == 0 + run, manifest = only_full(full_workspace) + instructions = (run / "README.md").read_text(encoding="utf-8") + assert "full.json" in instructions and "--live" in instructions + assert f"report-all --run-dir {run}" in instructions + assert manifest["configuration"]["model"] == model + assert [page["source_url"] for page in manifest["pages"]] == URLS + assert len({page["run_id"] for page in manifest["pages"]}) == 3 + assert manifest["complete"] is False and manifest["incomplete_reason"] == "preview" + assert manifest["max_cost_usd"] == "2" and manifest["estimated_cost_usd"] == "0" + assert all(value == 0 for value in manifest["usage"].values()) + memberships = [] + globals_by_page = [] + programmatic_ids = set() + prompt_count = 0 + for page in manifest["pages"]: + child = run / page["run_dir"] + child_manifest = json.loads((child / "run.json").read_text()) + assert child_manifest["run_id"] == child.name == page["run_id"] + assert child_manifest["source_url"] == page["source_url"] + assert child_manifest["configuration"] == manifest["configuration"] + prices = json.loads((child / "artifacts/pricing.json").read_text()) + assert model in prices["models"] + assert (child / "snapshot.html").read_text() == HTML + assert child_manifest["snapshot_sha256"] == hashlib.sha256(HTML.encode()).hexdigest() + prompt_count += len(child_manifest["prompts"]) + with (child / "review.csv").open(newline="") as stream: + rows = list(csv.DictReader(stream)) + memberships.append([row["Issue Title"] for row in rows]) + globals_by_page.append({row["human_finding_id"] for row in rows if row["page_scope"] == "global"}) + assert all(row["actual result"] == "Original\nevidence" for row in rows) + assert all(row["classification"] == "llm_eligible" for row in rows) + findings = json.loads((child / "artifacts/normalized-programmatic-findings.json").read_text()) + assert not (child / "normalized-programmatic-findings.json").exists() + instructions = (child / "README.md").read_text(encoding="utf-8") + assert "review.csv" in instructions + assert f"report --run-dir {child}" in instructions + assert findings and all(item["page_url"] == page["source_url"] for item in findings) + assert not programmatic_ids.intersection(item["finding_id"] for item in findings) + programmatic_ids.update(item["finding_id"] for item in findings) + assert memberships == [ + ["First local", "Second local", "Shared defect", "Shared second defect"], + ["Contact local", "Shared defect", "Shared second defect"], + ["Shared defect", "Shared second defect"], + ] + assert globals_by_page[0] == globals_by_page[1] == globals_by_page[2] + originals = json.loads((run / "human-findings.json").read_text()) + assert len(originals["human_findings"]) == 5 + assert len(originals["page_assignments"]) == 3 + output = capsys.readouterr().out + assert all(url in output for url in URLS) + assert f"prompts: {prompt_count}" in output + assert "Cost guardrail: $2" in output and "Pricing:" in output + assert "Preview complete; no paid requests made." in output + + +def change_audit(workspace, steps): + """Replace a reproduction URL and refresh its captured identity.""" + path = workspace / "human-audit.xlsx" + human_audit_excel = openpyxl.load_workbook(path) + human_audit_excel["Defect report"].cell(2, 6, steps) + human_audit_excel.save(path) + mapping_path = workspace / "snapshots/snapshots.json" + mapping = json.loads(mapping_path.read_text()) + mapping["human_audit_sha256"] = hashlib.sha256(path.read_bytes()).hexdigest() + mapping_path.write_text(json.dumps(mapping)) + + +@pytest.mark.parametrize("steps", [ + "No reproduction URL", "Open https://example.test/EMPTY/", + f"Open {URLS[0]} and {URLS[1]}", +]) +def test_invalid_page_assignments_stop_before_preparation(full_workspace, steps, capsys): + """Missing, arbitrary case aliases and ambiguous targets cannot start a run.""" + change_audit(full_workspace, steps) + with pytest.raises(SystemExit) as error: + preview(full_workspace) + assert error.value.code == 2 + assert "exactly one listed target URL" in capsys.readouterr().err + assert not (full_workspace / ".model-evaluation").exists() + + +@pytest.mark.parametrize("problem", [ + "checksum", "missing", "duplicate", "reordered", "outside", "missing_html", +]) +def test_snapshot_mapping_must_match_the_human_audit(full_workspace, problem, capsys): + """Mismatched capture identities and page sets never prepare an evaluation.""" + path = full_workspace / "snapshots/snapshots.json" + mapping = json.loads(path.read_text()) + if problem == "checksum": + mapping["human_audit_sha256"] = "other audit" + elif problem == "missing": + mapping["snapshots"].pop() + elif problem == "duplicate": + mapping["snapshots"].append(mapping["snapshots"][0]) + elif problem == "reordered": + mapping["snapshots"].reverse() + elif problem == "outside": + mapping["snapshots"][0]["html"] = "../human-audit.xlsx" + else: + (full_workspace / "snapshots/page-001.html").unlink() + path.write_text(json.dumps(mapping)) + with pytest.raises(SystemExit) as error: + preview(full_workspace) + assert error.value.code == 2 + assert not (full_workspace / ".model-evaluation").exists() + assert "Snapshot mapping" in capsys.readouterr().err + + +def test_page_preparation_failure_retains_parent_and_earlier_evidence(full_workspace, capsys): + """A failed HTML read stops preparation and records every unfinished member.""" + (full_workspace / "snapshots/page-002.html").write_bytes(b"\xff") + with pytest.raises(SystemExit) as error: + preview(full_workspace) + assert error.value.code == 2 + run, manifest = only_full(full_workspace) + assert manifest["complete"] is False + assert "preparation failed" in manifest["incomplete_reason"] + assert [page["status"] for page in manifest["pages"]] == [ + "prepared", "preparation_failed", "unprepared", + ] + first, second, third = [run / page["run_dir"] for page in manifest["pages"]] + assert (first / "review.csv").is_file() + assert (first / "artifacts/raw-programmatic-findings.json").is_file() + assert (second / "snapshot.html").read_bytes() == b"\xff" + assert not third.exists() + assert str(run) in capsys.readouterr().out + + +def test_audit_without_human_findings_cannot_start_evaluation(full_workspace, capsys): + """Listed pages alone provide no Human findings to score.""" + path = full_workspace / "human-audit.xlsx" + human_audit_excel = openpyxl.load_workbook(path) + human_audit_excel["Defect report"].delete_rows(2, 5) + human_audit_excel.save(path) + mapping_path = full_workspace / "snapshots/snapshots.json" + mapping = json.loads(mapping_path.read_text()) + mapping["human_audit_sha256"] = hashlib.sha256(path.read_bytes()).hexdigest() + mapping_path.write_text(json.dumps(mapping)) + with pytest.raises(SystemExit) as error: + preview(full_workspace) + assert error.value.code == 2 + assert "no Human findings" in capsys.readouterr().err + assert not (full_workspace / ".model-evaluation").exists() + + +def test_one_page_selection_and_human_ids_remain_compatible(full_workspace): + """The one-page command selects the URL's local findings and stable Globals.""" + assert preview(full_workspace) == 0 + full_run, _ = only_full(full_workspace) + originals = json.loads((full_run / "human-findings.json").read_text()) + assert main([ + "run", "--human-audit", str(full_workspace / "human-audit.xlsx"), + "--html", str(full_workspace / "snapshots/page-001.html"), + "--source-url", URLS[0], "--max-cost-usd", "2", + "--human-finding-mapping", str(full_workspace / "fixture-human-finding-mapping.json"), + ]) == 0 + (page_run,) = (full_workspace / ".model-evaluation/runs").iterdir() + evidence = json.loads((page_run / "artifacts/human-findings.json").read_text()) + assert [finding["problem"] for finding in evidence["human_findings"]] == [ + "First local", "Second local", "Shared defect", "Shared second defect", + ] + assert evidence["human_findings"] == ( + originals["human_findings"][:2] + originals["human_findings"][3:] + ) diff --git a/tests/test_full_report.py b/tests/test_full_report.py new file mode 100644 index 0000000..b99afd4 --- /dev/null +++ b/tests/test_full_report.py @@ -0,0 +1,245 @@ +"""Report pooled Full evaluation results through the public CLI.""" + +import csv +import hashlib +import json + +import openpyxl +import pytest +from bs4 import BeautifulSoup +from test_evaluation import recording_provider_usage +from test_full_execution import children +from test_full_preview import URLS, full_workspace, only_full, preview + +from vision_aid.evaluation.cli import main + +# Re-export the shared fixture for pytest discovery. +__all__ = ["full_workspace"] + + +def save_json(path, value): + """Save a deliberate evidence mutation or synthetic input.""" + path.write_text(json.dumps(value), encoding="utf-8") + + +def review_rows(child): + """Read the operator-editable decisions for one page.""" + with (child / "review.csv").open(newline="") as stream: + return list(csv.DictReader(stream)) + + +def save_review(child, rows): + """Save reviewer decisions while keeping all original columns.""" + path = child / "review.csv" + with path.open(newline="") as stream: + fields = csv.DictReader(stream).fieldnames + with path.open("w", newline="") as stream: + writer = csv.DictWriter(stream, fieldnames=fields) + writer.writeheader() + writer.writerows(rows) + + +def report_all(run, reviewer="Combined reviewer"): + """Invoke deterministic combined reporting with reviewer attribution.""" + return main(["report-all", "--run-dir", str(run), "--reviewer", reviewer]) + + +def refused_report(run, capsys, reviewer="Combined reviewer"): + """Assert invalid evidence cannot publish a final combined report.""" + with pytest.raises(SystemExit) as error: + report_all(run, reviewer) + assert error.value.code == 2 + assert not (run / "report.md").exists() + assert not (run / "report.html").exists() + return capsys.readouterr().err + + +@pytest.fixture +def completed_review(full_workspace, monkeypatch): + """Complete an approved run and record only its LLM match decisions.""" + recording_provider_usage(monkeypatch, "gpt-6-luna", { + "prompt_tokens": 1000, "completion_tokens": 100, + }) + assert preview(full_workspace, "--model", "gpt-6-luna", "--live", "--approve-live") == 0 + run, manifest = only_full(full_workspace) + for child, _ in children(run, manifest): + findings = json.loads((child / "normalized-llm-findings.json").read_text()) + rows = review_rows(child) + for row, finding in zip(rows, findings): + row.update(llm_finding_id=finding["finding_id"], match_reason="Complete fixture match") + save_review(child, rows) + return run, manifest + + +@pytest.mark.parametrize("problem", [ + "parent_incomplete", "parent_id", "missing_page", "duplicate_page", "reordered_pages", + "child_parent_id", "expected_ids", "child_configuration", "human_evidence", + "missing_snapshot", "snapshot_bytes", "pricing_bytes", "missing_raw_llm", + "parent_cost", "parent_usage", +]) +def test_report_requires_complete_membership_and_owned_evidence(completed_review, problem, capsys): + """Completed flags cannot hide omitted pages or altered child evidence.""" + run, manifest = completed_review + first = run / manifest["pages"][0]["run_dir"] + if problem == "parent_incomplete": + manifest["complete"] = False + elif problem == "parent_id": + manifest["run_id"] = "foreign-full-evaluation" + elif problem == "missing_page": + manifest["pages"].pop() + elif problem == "duplicate_page": + manifest["pages"].append(manifest["pages"][0]) + elif problem == "reordered_pages": + manifest["pages"].reverse() + elif problem == "child_parent_id": + manifest["pages"][0]["run_id"] = "foreign-child" + elif problem == "expected_ids": + manifest["pages"][0]["human_finding_ids"].pop() + elif problem == "child_configuration": + saved = json.loads((first / "run.json").read_text()) + saved["configuration"]["model"] = "claude-haiku-5-5" + save_json(first / "run.json", saved) + elif problem == "human_evidence": + path = first / "artifacts/human-findings.json" + saved = json.loads(path.read_text()) + saved["human_findings"][0]["problem"] = "Replaced Human finding evidence" + save_json(path, saved) + elif problem == "missing_snapshot": + (first / "snapshot.html").unlink() + elif problem == "snapshot_bytes": + (first / "snapshot.html").write_text("another page") + elif problem == "pricing_bytes": + path = first / "artifacts/pricing.json" + path.write_text(path.read_text() + " ") + elif problem == "missing_raw_llm": + (first / "artifacts/raw-llm-responses.json").unlink() + elif problem == "parent_cost": + manifest["estimated_cost_usd"] = "7" + elif problem == "parent_usage": + manifest["usage"]["output_tokens"] = 99999 + save_json(run / "full.json", manifest) + refused_report(run, capsys) + + +def test_empty_page_without_globals_still_has_a_report(full_workspace, monkeypatch): + """A listed page with no Human findings can complete its empty review.""" + path = full_workspace / "human-audit.xlsx" + human_audit_excel = openpyxl.load_workbook(path) + human_audit_excel["Defect report"].delete_rows(5, 2) + human_audit_excel.save(path) + mapping_path = full_workspace / "snapshots/snapshots.json" + mapping = json.loads(mapping_path.read_text()) + mapping["human_audit_sha256"] = hashlib.sha256(path.read_bytes()).hexdigest() + save_json(mapping_path, mapping) + recording_provider_usage(monkeypatch, "gpt-6-luna") + assert preview(full_workspace, "--model", "gpt-6-luna", "--live", "--approve-live") == 0 + run, manifest = only_full(full_workspace) + assert report_all(run) == 0 + empty = run / manifest["pages"][2]["run_dir"] + text = BeautifulSoup((empty / "report.html").read_text(), "html.parser").get_text(" ", strip=True) + assert "Overall: 0 caught / 0 eligible; Not defined: no imported Human findings" in text + parent = BeautifulSoup((run / "report.html").read_text(), "html.parser") + assert "Overall: 0 caught / 3 eligible; 0.0%" in parent.get_text(" ", strip=True) + comparison = parent.find("figure", attrs={"aria-labelledby": "page-chart-title"}) + assert [item.get_text(" ", strip=True) for item in comparison.select(".chart-value")] == [ + f"{url}: 0 caught / {eligible} eligible; " + ("0.0%" if eligible else "Not defined: no eligible Human findings") + for url, eligible in zip(URLS, [2, 1, 0], strict=True) + ] + + +def test_invalid_revised_review_withholds_stale_combined_rates(completed_review, capsys): + """A report regenerated after an invalid edit must not leave earlier rates.""" + run, manifest = completed_review + assert report_all(run) == 0 + (run / "report.md").write_text("superseded Markdown") + first = run / manifest["pages"][0]["run_dir"] + rows = review_rows(first) + rows[0]["classification"] = "" + save_review(first, rows) + assert "decision changed: classification" in refused_report(run, capsys) + assert (first / "report.html").exists() + + +@pytest.mark.parametrize("problem", [ + "unclassified", "source_identity", "wrong_source_id", "foreign_id", + "missing_reason", "reused_id", "missing_child", "missing_review", +]) +def test_every_page_review_uses_the_existing_validation(completed_review, capsys, problem): + """Unfinished or invalid page decisions cannot receive pooled credit.""" + run, manifest = completed_review + first = run / manifest["pages"][0]["run_dir"] + rows = review_rows(first) + if problem == "unclassified": + rows[0]["classification"] = "" + elif problem == "source_identity": + rows[0]["source_element"] = "Changed source element" + elif problem == "wrong_source_id": + rows[0]["llm_finding_id"] = json.loads( + (first / "artifacts/normalized-programmatic-findings.json").read_text())[0]["finding_id"] + elif problem == "foreign_id": + other = run / manifest["pages"][1]["run_dir"] + rows[0]["llm_finding_id"] = json.loads( + (other / "normalized-llm-findings.json").read_text())[0]["finding_id"] + elif problem == "missing_reason": + rows[0]["match_reason"] = "" + elif problem == "reused_id": + rows[1]["llm_finding_id"] = rows[0]["llm_finding_id"] + elif problem == "missing_child": + (first / "run.json").unlink() + elif problem == "missing_review": + (first / "review.csv").unlink() + if problem not in {"missing_child", "missing_review"}: + save_review(first, rows) + refused_report(run, capsys) + + +def test_completed_child_report_survives_later_parent_failure(full_workspace, monkeypatch, capsys): + """An incomplete parent withholds combined rates while its finished page reports.""" + monkeypatch.setenv("ANTHROPIC_API_KEY", "synthetic") + recording_provider_usage(monkeypatch, "claude-haiku-5-5", stream_failure_at=2) + assert preview(full_workspace, "--live", "--approve-live") == 1 + run, manifest = only_full(full_workspace) + first = run / manifest["pages"][0]["run_dir"] + assert main(["report", "--run-dir", str(first), "--reviewer", "Page reviewer"]) == 0 + saved = (first / "report.html").read_bytes() + assert "Incomplete Full evaluations" in refused_report(run, capsys) + assert (first / "report.html").read_bytes() == saved + + +def test_reviewer_description_is_required(completed_review, capsys): + """Scoring cannot discard the identity of the reviewer making the decisions.""" + run, _ = completed_review + assert "reviewer description" in refused_report(run, capsys, reviewer=" ") + + +@pytest.mark.parametrize("response", ["", "not JSON"]) +def test_combined_report_keeps_unreadable_response_evidence(full_workspace, monkeypatch, response): + """Successful Unreadable responses remain complete and are counted across pages.""" + recording_provider_usage(monkeypatch, "gpt-6-luna", text=response) + assert preview(full_workspace, "--model", "gpt-6-luna", "--live", "--approve-live") == 0 + run, manifest = only_full(full_workspace) + assert manifest["complete"] + for child, saved in children(run, manifest): + assert saved["complete"] and len(saved["format_failures"]) == len(saved["prompts"]) + assert json.loads((child / "normalized-llm-findings.json").read_text()) == [] + assert all(item["result"]["response"] == response for item in json.loads( + (child / "artifacts/raw-llm-responses.json").read_text())) + assert report_all(run) == 0 + assert sum(len(json.loads((child / "run.json").read_text())["format_failures"]) + for child, _ in children(run, manifest)) == 6 + + +def test_successful_full_regeneration_replaces_only_its_generated_markdown(completed_review, capsys): + """HTML replaces regenerated Markdown without sweeping historical outputs.""" + run, manifest = completed_review + (run / "report.md").write_text("old combined result") + historical = run.parent / "historical-run" + historical.mkdir() + (historical / "report.md").write_text("historical result") + for page in manifest["pages"]: + (run / page["run_dir"] / "report.md").write_text("old page result") + assert report_all(run) == 0 + assert f"Combined HTML report: {run / 'report.html'}" in capsys.readouterr().out + assert not (run / "report.md").exists() + assert all(not (run / page["run_dir"] / "report.md").exists() for page in manifest["pages"]) + assert (historical / "report.md").read_text() == "historical result" diff --git a/tests/test_human_finding_mapping_authoring.py b/tests/test_human_finding_mapping_authoring.py new file mode 100644 index 0000000..6100519 --- /dev/null +++ b/tests/test_human_finding_mapping_authoring.py @@ -0,0 +1,341 @@ +"""Create approved audit decisions through human-edited CSV files and the CLI.""" + +import csv +import json + +import pytest +from bs4 import BeautifulSoup +from test_evaluation import recording_provider_usage +from test_full_preview import URLS, full_workspace, only_full, preview + +from vision_aid.evaluation.cli import main + +__all__ = ["full_workspace"] + + +def rows(path): + """Read a spreadsheet-friendly CSV produced for the operator.""" + with path.open(encoding="utf-8-sig", newline="") as stream: + return list(csv.DictReader(stream)) + + +def prepare(workspace): + """Prepare a Human finding mapping review independently of model selection and execution.""" + directory = workspace / "human-finding-mapping-review" + assert main([ + "human-finding-mapping-prepare", + "--human-audit", str(workspace / "human-audit.xlsx"), + "--snapshots", str(workspace / "snapshots/snapshots.json"), + "--output-dir", str(directory), + ]) == 0 + return directory + + +def save_rows(path, entered): + """Save reviewer edits as an ordinary spreadsheet CSV.""" + with path.open("w", encoding="utf-8-sig", newline="") as stream: + writer = csv.DictWriter(stream, fieldnames=entered[0]) + writer.writeheader() + writer.writerows(entered) + + +def fill_review(directory): + """Record one joint catch, a cross-catch and explicit absences for five originals.""" + classifications = rows(directory / "classifications.csv") + for row, classification in zip( + classifications, + [ + "programmatic", + "llm_eligible", + "unavailable_evidence", + "llm_eligible", + "ambiguous", + ], + strict=True, + ): + row.update( + classification=classification, + classification_reason="Reviewed original defect against saved evidence", + ) + save_rows(directory / "classifications.csv", list(reversed(classifications))) + matches = rows(directory / "programmatic-review.csv") + catalog = rows(directory / "programmatic-findings.csv") + for row in matches: + rules = { + "First local": ["LANG_001", "HEAD_003"], + "Second local": ["LAND_001"], + }.get(row["Issue Title"], []) + row["programmatic_finding_refs"] = ";".join( + next( + f["finding_ref"] + for f in catalog + if f["source_url"] == row["source_url"] and f["rule"] == rule + ) + for rule in rules + ) + row["programmatic_reason"] = ( + "Jointly covers the complete defect" + if rules + else "No existing rule fully covers this defect" + ) + save_rows(directory / "programmatic-review.csv", list(reversed(matches))) + + +def approve(directory): + """Approve reviewed CSV choices using only the supported CLI.""" + return main([ + "human-finding-mapping-approve", + "--human-audit", str(directory.parent / "human-audit.xlsx"), + "--snapshots", str(directory.parent / "snapshots/snapshots.json"), + "--review-dir", str(directory), "--reviewer", "Human CSV reviewer", + ]) + + +def test_human_finding_mapping_preparation_lists_originals_once_and_readable_page_evidence( + full_workspace, +): + """Reviewers classify five originals once and choose matches for nine page rows.""" + directory = prepare(full_workspace) + classifications = rows(directory / "classifications.csv") + matches = rows(directory / "programmatic-review.csv") + findings = rows(directory / "programmatic-findings.csv") + assert len(classifications) == 5 + assert len({row["human_finding_id"] for row in classifications}) == 5 + assert all(row["actual result"] == "Original\nevidence" for row in classifications) + assert all( + row["classification"] == row["classification_reason"] == "" + for row in classifications + ) + assert len(matches) == 9 + assert {row["source_url"] for row in matches} == set(URLS) + assert all( + row["programmatic_finding_refs"] == row["programmatic_reason"] == "" + for row in matches + ) + assert findings and all( + row["finding_ref"] and row["rule"] and row["problem"] for row in findings + ) + assert any(row["rule"] == "LAND_001" for row in findings) + assert any(row["element"] == "img" and row["target"] == "x" for row in findings) + assert len({row["finding_ref"] for row in findings}) == len(findings) + assert (directory / "README.md").is_file() + assert {path.name for path in directory.iterdir()} == { + "classifications.csv", "programmatic-review.csv", "programmatic-findings.csv", "README.md", + } + + +def test_csv_approval_builds_reusable_expectations_consumed_by_new_full_run( + full_workspace, monkeypatch +): + """A reviewer never authors JSON or selectors; the approved result drives run scoring.""" + directory = prepare(full_workspace) + fill_review(directory) + assert approve(directory) == 0 + mapping = directory / "human-finding-mapping.json" + approved = json.loads(mapping.read_text()) + assert approved["approved_by"] == "Human CSV reviewer" + assert len(approved["decisions"]) == 5 + assert sum(len(d["programmatic_by_page"]) for d in approved["decisions"]) == 9 + first = next( + d for d in approved["decisions"] if d["classification"] == "programmatic" + ) + assert { + e["rule"] for e in first["programmatic_by_page"][URLS[0]]["expectations"] + } == {"LANG_001", "HEAD_003"} + assert '"run_id"' not in mapping.read_text() + assert "human-finding-mapping-review" not in mapping.read_text() + monkeypatch.setenv("OPENAI_API_KEY", "synthetic") + recording_provider_usage(monkeypatch, "gpt-6-luna") + assert ( + preview( + full_workspace, + "--human-finding-mapping", + str(mapping), + "--model", + "gpt-6-luna", + "--live", + "--approve-live", + ) + == 0 + ) + run, _ = only_full(full_workspace) + assert ( + main(["report-all", "--run-dir", str(run), "--reviewer", "LLM reviewer"]) == 0 + ) + text = BeautifulSoup((run / "report.html").read_text(), "html.parser").get_text( + " ", strip=True + ) + assert "Programmatic: 1 caught / 1 eligible" in text + assert "LLM: 0 caught / 2 eligible" in text + assert "Overall: 2 caught / 5 eligible" in text + assert "Human finding mapping approved by" in text + + +@pytest.mark.parametrize("command", ["run", "run-all"]) +@pytest.mark.parametrize("live", [False, True]) +def test_new_evaluations_require_approved_decisions_before_creating_a_run( + full_workspace, capsys, command, live +): + """New audits must complete Human finding mapping review before either evaluation entry point.""" + arguments = [ + command, + "--human-audit", + str(full_workspace / "human-audit.xlsx"), + "--max-cost-usd", + "1", + ] + if command == "run-all": + arguments += ["--snapshots", str(full_workspace / "snapshots/snapshots.json")] + else: + arguments += [ + "--html", + str(full_workspace / "snapshots/page-001.html"), + "--source-url", + URLS[0], + ] + if live: + arguments += ["--live", "--approve-live"] + with pytest.raises(SystemExit) as error: + main(arguments) + assert error.value.code == 2 + message = capsys.readouterr().err + assert ( + "approved Human finding mapping" in message + and "human-finding-mapping-prepare" in message + and "--human-finding-mapping" in message + ) + assert not (full_workspace / ".model-evaluation").exists() + + +@pytest.mark.parametrize( + "change,error", + [ + ("classification", "valid classification"), + ("classification_reason", "classification_reason"), + ("missing_original", "every original review row"), + ("duplicate_original", "unknown or repeated"), + ("original_evidence", "original Human finding evidence changed"), + ("missing_page", "every original review row"), + ("duplicate_page", "unknown or repeated"), + ("reason", "programmatic_reason"), + ("unknown_ref", "unknown or wrong-page"), + ("wrong_page", "unknown or wrong-page"), + ("reuse", "finding ID reuse"), + ("duplicate_ref", "finding ID reuse"), + ("programmatic_absence", "requires expected coverage"), + ("not_testable_catch", "not testable"), + ("snapshot", "programmatic-findings.csv"), + ], +) +def test_csv_approval_rejects_incomplete_or_conflicting_reviews( + full_workspace, capsys, change, error +): + """Errors identify review cells or evidence; incomplete choices never become approved JSON.""" + directory = prepare(full_workspace) + fill_review(directory) + classifications = rows(directory / "classifications.csv") + matches = rows(directory / "programmatic-review.csv") + first = next(row for row in matches if row["Issue Title"] == "First local") + second = next(row for row in matches if row["Issue Title"] == "Second local") + if change == "classification": + classifications[0]["classification"] = "guess" + elif change == "classification_reason": + classifications[0]["classification_reason"] = "" + elif change == "missing_original": + classifications.pop() + elif change == "duplicate_original": + classifications[0] = classifications[1].copy() + elif change == "original_evidence": + classifications[0]["actual result"] = "Rewritten original" + elif change == "missing_page": + matches.pop() + elif change == "duplicate_page": + matches[0] = matches[1].copy() + elif change == "reason": + first["programmatic_reason"] = "" + elif change == "unknown_ref": + first["programmatic_finding_refs"] = "P999-F999" + elif change == "wrong_page": + first["programmatic_finding_refs"] = next( + row["finding_ref"] + for row in rows(directory / "programmatic-findings.csv") + if row["source_url"] == URLS[1] + ) + elif change == "reuse": + second["programmatic_finding_refs"] = first["programmatic_finding_refs"] + elif change == "duplicate_ref": + first["programmatic_finding_refs"] += ( + ";" + first["programmatic_finding_refs"].split(";")[0] + ) + elif change == "programmatic_absence": + first["programmatic_finding_refs"] = "" + elif change == "not_testable_catch": + next(row for row in classifications if row["Issue Title"] == "First local")[ + "classification" + ] = "unavailable_evidence" + elif change == "snapshot": + (full_workspace / "snapshots/page-001.html").write_text("changed snapshot") + save_rows(directory / "classifications.csv", classifications) + save_rows(directory / "programmatic-review.csv", matches) + with pytest.raises(SystemExit) as failure: + approve(directory) + assert failure.value.code == 2 + assert error in capsys.readouterr().err + assert not (directory / "human-finding-mapping.json").exists() + + +def test_catalog_edits_are_rejected_and_approval_never_overwrites( + full_workspace, capsys +): + """The catalog must match rerun checkers, and approval cannot replace an existing mapping.""" + directory = prepare(full_workspace) + fill_review(directory) + catalog = rows(directory / "programmatic-findings.csv") + edited = [row | {"rule": "INVENTED_RULE", "problem": "Invented checker evidence"} for row in catalog] + save_rows(directory / "programmatic-findings.csv", edited) + output = full_workspace / "approved/human-finding-mapping.json" + arguments = [ + "human-finding-mapping-approve", + "--human-audit", str(full_workspace / "human-audit.xlsx"), + "--snapshots", str(full_workspace / "snapshots/snapshots.json"), + "--review-dir", + str(directory), + "--reviewer", + "Reviewer", + "--output", + str(output), + ] + with pytest.raises(SystemExit): + main(arguments) + assert "original Programmatic finding evidence changed" in capsys.readouterr().err + assert not output.exists() + save_rows(directory / "programmatic-findings.csv", catalog) + assert main(arguments) == 0 + original = output.read_bytes() + assert b"INVENTED_RULE" not in original + with pytest.raises(SystemExit): + main(arguments) + assert "File exists" in capsys.readouterr().err + assert output.read_bytes() == original + + +def test_csv_review_requires_reviewer_and_keeps_previous_work_on_repeated_preparation( + full_workspace, capsys, +): + """An unfinished or already-open review cannot be approved or silently replaced.""" + directory = prepare(full_workspace) + fill_review(directory) + original = (directory / "classifications.csv").read_bytes() + with pytest.raises(SystemExit): + prepare(full_workspace) + assert (directory / "classifications.csv").read_bytes() == original + with pytest.raises(SystemExit) as failure: + main([ + "human-finding-mapping-approve", + "--human-audit", str(full_workspace / "human-audit.xlsx"), + "--snapshots", str(full_workspace / "snapshots/snapshots.json"), + "--review-dir", str(directory), "--reviewer", " ", + ]) + assert failure.value.code == 2 + assert "reviewer description is required" in capsys.readouterr().err + assert not (directory / "human-finding-mapping.json").exists() diff --git a/uv.lock b/uv.lock index 4693dfe..21a4146 100644 --- a/uv.lock +++ b/uv.lock @@ -324,6 +324,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/a7/5f/ed01f9a3cdffbd5a008556fc7b2a08ddb1cc6ace7effa7340604b1d16699/docstring_parser-0.18.0-py3-none-any.whl", hash = "sha256:b3fcbed555c47d8479be0796ef7e19c2670d428d72e96da63f3a40122860374b", size = 22484, upload-time = "2026-04-14T04:09:18.638Z" }, ] +[[package]] +name = "et-xmlfile" +version = "2.0.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/d3/38/af70d7ab1ae9d4da450eeec1fa3918940a5fafb9055e934af8d6eb0c2313/et_xmlfile-2.0.0.tar.gz", hash = "sha256:dab3f4764309081ce75662649be815c4c9081e88f0837825f90fd28317d4da54", size = 17234, upload-time = "2024-10-25T17:25:40.039Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/c1/8b/5fe2cc11fee489817272089c4203e679c63b570a5aaeb18d852ae3cbba6a/et_xmlfile-2.0.0-py3-none-any.whl", hash = "sha256:7a91720bc756843502c3b7504c77b8fe44217c85c537d85037f0f536151b2caa", size = 18059, upload-time = "2024-10-25T17:25:39.051Z" }, +] + [[package]] name = "google-auth" version = "2.56.3" @@ -409,6 +418,27 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/1e/5e/d4e9f1a599fb8e573b7b87160658329fbf28d19eac2718f51fc3def3aa5a/idna-3.18-py3-none-any.whl", hash = "sha256:7f952cbe720b688055e3f87de14f5c3e5fdaa8bc3928985c4077ca689de849a2", size = 65455, upload-time = "2026-06-02T14:34:06.319Z" }, ] +[[package]] +name = "iniconfig" +version = "2.3.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/72/34/14ca021ce8e5dfedc35312d08ba8bf51fdd999c576889fc2c24cb97f4f10/iniconfig-2.3.0.tar.gz", hash = "sha256:c76315c77db068650d49c5b56314774a7804df16fee4402c1f19d6d15d8c4730", size = 20503, upload-time = "2025-10-18T21:55:43.219Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/cb/b1/3846dd7f199d53cb17f49cba7e651e9ce294d8497c8c150530ed11865bb8/iniconfig-2.3.0-py3-none-any.whl", hash = "sha256:f631c04d2c48c52b84d0d0549c99ff3859c98df65b3101406327ecc7d53fbf12", size = 7484, upload-time = "2025-10-18T21:55:41.639Z" }, +] + +[[package]] +name = "jinja2" +version = "3.1.6" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "markupsafe" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/df/bf/f7da0350254c0ed7c72f3e33cef02e048281fec7ecec5f032d4aac52226b/jinja2-3.1.6.tar.gz", hash = "sha256:0137fb05990d35f1275a587e9aee6d56da821fc83491a0fb838183be43f66d6d", size = 245115, upload-time = "2025-03-05T20:05:02.478Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/62/a1/3d680cbfd5f4b8f15abc1d571870c5fc3e594bb582bc3b64ea099db13e56/jinja2-3.1.6-py3-none-any.whl", hash = "sha256:85ece4451f492d0c13c5dd7c13a64681a86afae63a5f347908daf103ce6d2f67", size = 134899, upload-time = "2025-03-05T20:05:00.369Z" }, +] + [[package]] name = "jiter" version = "0.16.0" @@ -597,6 +627,134 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/db/a4/441aee36c6f6b249823d20fd91f9be9ab89d7c5a8ae542a4a4ca6d342d56/lxml-6.1.1-pp311-pypy311_pp73-win_amd64.whl", hash = "sha256:ed21202aec73cda4d55d1ce57b389aadb90ffb044e6cd1080b8347efe1b1ec84", size = 3508989, upload-time = "2026-05-18T19:18:38.158Z" }, ] +[[package]] +name = "markupsafe" +version = "3.0.4" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/38/9b/e422a865e1d5d57d0e509b4e0bf1c1a70a7f6382c29a5aa428df994c8bc8/markupsafe-3.0.4.tar.gz", hash = "sha256:2e9ad7dd851bf45fab9f75cbff4cb493fee9979e8d8c7c9c3ee119022518edd6", size = 153777, upload-time = "2026-10-02T23:07:22.29Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/32/55/18dbb4778b30ada5ce071608503cc3edc9e14e13d868c17a6d178fc30f7a/markupsafe-3.0.4-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:9e25feb9e330b63edb0278a0acdf85e50d0cb0fbf49c3084abbe4e24ae195346", size = 11520, upload-time = "2026-10-02T23:04:36.299Z" }, + { url = "https://files.pythonhosted.org/packages/6c/14/0b05f79b4733e264a18d08fe08fa1df7347630ff32a6cb82180d9dccec55/markupsafe-3.0.4-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:7d3391b2188d18737cb2fa147028b1096236eaa7e156446c650a489fa2cadc91", size = 12063, upload-time = "2026-10-02T23:04:37.291Z" }, + { url = "https://files.pythonhosted.org/packages/ca/3a/63ba10b6c1463216b3e4df669a9f0e5a3b0c3071557d2e8229e3968c79fb/markupsafe-3.0.4-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:849dd2bb0e5e4ab2b71c7191726a4a8d5aa8a610daa584728cbee0b710ddc4ef", size = 24321, upload-time = "2026-10-02T23:04:38.262Z" }, + { url = "https://files.pythonhosted.org/packages/1a/2e/5f015261b76ad633d187ef6f388b413aedd64a8773c4df59e530a0be5525/markupsafe-3.0.4-cp311-cp311-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:befb4158af32106b9a93db8d6d1d1cbbd418c0d5aca0cabb7b1780abf0c89169", size = 28845, upload-time = "2026-10-02T23:04:39.295Z" }, + { url = "https://files.pythonhosted.org/packages/33/cf/26e594b26be40c2f1fec63ccf8a8b99d0335a5b2cbe84835c7d82a994375/markupsafe-3.0.4-cp311-cp311-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:71f88e749ea29f67f21f3b36433c1dc54c7729ed2a6d9e2da2e0d9e0d7b224eb", size = 24701, upload-time = "2026-10-02T23:04:40.348Z" }, + { url = "https://files.pythonhosted.org/packages/81/a5/a513b76c139a3915b43404324e55c0b7979ae4f0d39eb6f075b0282e90a8/markupsafe-3.0.4-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:6da83a088f8ef93b2d483a8232a4dbf4d69d3d8496b568a03c56becac43e1808", size = 22961, upload-time = "2026-10-02T23:04:41.372Z" }, + { url = "https://files.pythonhosted.org/packages/46/cf/4c66192c100b4542bcbe392ae06696b670f66927be3ac38a213234778ff9/markupsafe-3.0.4-cp311-cp311-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:8f0fac8b13d14bb06c68195f849371924ae53dd7b1c00fed24650f704383b692", size = 21920, upload-time = "2026-10-02T23:04:42.429Z" }, + { url = "https://files.pythonhosted.org/packages/cb/17/ac3662678bfbad649893117ada2ba44dc30bf56884e84f13154a792b10f1/markupsafe-3.0.4-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:4a7cdc2a420ca01058182da4253329764d4bfa055564d1eced90e6ba1e8b1d3d", size = 23726, upload-time = "2026-10-02T23:04:43.531Z" }, + { url = "https://files.pythonhosted.org/packages/f7/af/fe47cee339180a69ebca3c57fb3483d0f5cbd1e8337d1871fb1d9c1aebee/markupsafe-3.0.4-cp311-cp311-musllinux_1_2_armv7l.whl", hash = "sha256:83b3944fea42a8400edf92fd1770fb8d0d4f7de651353bd2d8525a92dba69a21", size = 22539, upload-time = "2026-10-02T23:04:44.618Z" }, + { url = "https://files.pythonhosted.org/packages/12/32/d55440ba140442800e02d799c9cb5ab597bf6ebdb1177b5ea39a11f797bd/markupsafe-3.0.4-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:8138eb83940ec7299024d92d4dee45f601b9e6c5ffde9d25f4e35e326203c707", size = 23963, upload-time = "2026-10-02T23:04:45.656Z" }, + { url = "https://files.pythonhosted.org/packages/50/9d/9c86042cb364c2ad4c971e6d1247929effd25f714cd7ee11b05e6316445b/markupsafe-3.0.4-cp311-cp311-musllinux_1_2_riscv64.whl", hash = "sha256:811d02d5122171c1941357efd8f9bf4ffe907b7f0a1a4e729a880e4be3f46e3e", size = 21506, upload-time = "2026-10-02T23:04:46.671Z" }, + { url = "https://files.pythonhosted.org/packages/75/ef/5b824f03ba40c3b3652b6272d083d2fc4fcdd440de519a3a39ba2c3e7262/markupsafe-3.0.4-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:50b5bedc9ed8a94fc8857a42ef4f84a81ea88f8d4f05dc8705fb23ee6d8dcca7", size = 22955, upload-time = "2026-10-02T23:04:47.698Z" }, + { url = "https://files.pythonhosted.org/packages/1e/e8/44cfcb5ea40e5e43cec7793ef90704ed0c475280839076c4345757eb8e59/markupsafe-3.0.4-cp311-cp311-win32.whl", hash = "sha256:2e5a7cd7fdd14fcb1ae5d7d8bf23d24fbd1daefd1fbca2580132e1ea75f098b5", size = 14094, upload-time = "2026-10-02T23:04:48.788Z" }, + { url = "https://files.pythonhosted.org/packages/91/89/f2b509f7bf79352e40117824c1070dbeafd4df67031d3fa98165a3134228/markupsafe-3.0.4-cp311-cp311-win_amd64.whl", hash = "sha256:fdb4ca07ab75ffadab4a8b135ad59cdbb3156b99310f3d565370da74a15d6bd3", size = 14328, upload-time = "2026-10-02T23:04:49.801Z" }, + { url = "https://files.pythonhosted.org/packages/2b/5a/ccf22672a0f64dc682306e288f0dabcb06c7a201f3bb6e6cbf86d9e8ad03/markupsafe-3.0.4-cp311-cp311-win_arm64.whl", hash = "sha256:569d65055d367e3dcdf30c3f41119467b73d9ee9faf332bdf40402644f5ac08e", size = 14188, upload-time = "2026-10-02T23:04:50.847Z" }, + { url = "https://files.pythonhosted.org/packages/81/09/4c59d56b8461ae8eb0d8ba34bb25b7e618547044679d58a82ef9b2479fc1/markupsafe-3.0.4-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:61631e08084be9e21a8967ec3139c7616ed7c5e9368e05c86d1b39562c8a57b6", size = 11658, upload-time = "2026-10-02T23:04:51.876Z" }, + { url = "https://files.pythonhosted.org/packages/a2/f0/d6613774d86fbf6d145751d43c59875e47a6f9f17daee0aef173bd36d90e/markupsafe-3.0.4-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:0930db9bdc62d22944e10b066448bb65dc9abe9112880c7cab8da54db4284d5f", size = 12050, upload-time = "2026-10-02T23:04:52.931Z" }, + { url = "https://files.pythonhosted.org/packages/0d/f2/8f18e0b806eb13c1f8d07d917a720831ead54253a6dec011fbc78098a6f8/markupsafe-3.0.4-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:6a45c3d514f2436064db00d7fc8778d888f0236ebfed649b53d13a59e69ad51b", size = 24363, upload-time = "2026-10-02T23:04:53.895Z" }, + { url = "https://files.pythonhosted.org/packages/60/ce/fa07dbe8a5675558fa36dea033e19995bc783de2dec5f540ccb9030b06aa/markupsafe-3.0.4-cp312-cp312-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:1e1451fab512d1bcc3dc26988ec1edb0b82c2db909132872cd9356070a6b63df", size = 28525, upload-time = "2026-10-02T23:04:54.905Z" }, + { url = "https://files.pythonhosted.org/packages/85/40/be87c01f3868ec217f8a2015089d71c22c8c5a75324822e5ed1cdd87210d/markupsafe-3.0.4-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:bd3ce56ae2cbae3ba82b683bc425cd7e48d2ed8b10f3e818186b6f5646d9271c", size = 24733, upload-time = "2026-10-02T23:04:56.229Z" }, + { url = "https://files.pythonhosted.org/packages/4f/a7/aeedb5140afa41fc74c225e9184ab96723a6e873b6ee1c9fede7283456d8/markupsafe-3.0.4-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:8e124f974786f831d6043728e38296969d3579db8896fe004682f5758e613581", size = 22985, upload-time = "2026-10-02T23:04:57.521Z" }, + { url = "https://files.pythonhosted.org/packages/c3/fc/e91352bb08c6a59da3ef0909d457bf95a5f5908fbf151b30a06d9dbcfbb4/markupsafe-3.0.4-cp312-cp312-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:c02e8f18bdedba082cef725942ac823b9b60656db07f7e265cb31618dfd00d77", size = 22001, upload-time = "2026-10-02T23:04:58.597Z" }, + { url = "https://files.pythonhosted.org/packages/5d/f8/bffee5e7d2a3deb59748a797650a48af7e672025cf641a79344a771ad106/markupsafe-3.0.4-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:9f098115c247e11d138ab83a28fa0323c77015007ea2df73ba5fd714dfefd67c", size = 23793, upload-time = "2026-10-02T23:04:59.686Z" }, + { url = "https://files.pythonhosted.org/packages/ed/59/b853d6628ecb4d658e1d637224846d5e9bb4adf4f8df97f3be9f29dce2ec/markupsafe-3.0.4-cp312-cp312-musllinux_1_2_armv7l.whl", hash = "sha256:d5f93ebbeb8032d47e349328ec8662d973d9b05a70b3c35df1f91fe419b84749", size = 22640, upload-time = "2026-10-02T23:05:00.768Z" }, + { url = "https://files.pythonhosted.org/packages/09/b2/1506df394f0f075797c418d0301498f49e43be194e3ffcb49e6fe6ccf022/markupsafe-3.0.4-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:64511c54db4e4987aef4c41923235927428729e8174c5dba488429be70a998ed", size = 24074, upload-time = "2026-10-02T23:05:01.813Z" }, + { url = "https://files.pythonhosted.org/packages/c7/81/5ed69cda630ac69ef60d06c09ba5a7f84ff66a2e28cf986fd5614ab3c6e6/markupsafe-3.0.4-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:e1a622f13970d81f95d0c72f9dc090dce9085fccfa4c9f2174377ee32bd15786", size = 21563, upload-time = "2026-10-02T23:05:03.239Z" }, + { url = "https://files.pythonhosted.org/packages/0c/fe/fb1e79be0fea60aa32602ebefc9c35a82bb42b4df157285ab7dfec12341a/markupsafe-3.0.4-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:c9a7f43c0b202b334cc9184af09bb8f21d3a209e038efaf106936fb69e6b026e", size = 23048, upload-time = "2026-10-02T23:05:04.479Z" }, + { url = "https://files.pythonhosted.org/packages/c8/52/7632a53360671a9b750cdbabaf9cdd89f18b42248b8e4cb42c0b0296e459/markupsafe-3.0.4-cp312-cp312-win32.whl", hash = "sha256:f0ec3b750b59375eab5b0fb2b9254810c00a3375be6d789899f1055a1d556237", size = 14108, upload-time = "2026-10-02T23:05:05.513Z" }, + { url = "https://files.pythonhosted.org/packages/3f/bf/62495e180b7000aaf30000fff849e933f74264638057176cf46852500adc/markupsafe-3.0.4-cp312-cp312-win_amd64.whl", hash = "sha256:11935df9bf455ed0c04eb87bcd720f02b1fe5e02128a9430f23aed6f93336fc7", size = 14303, upload-time = "2026-10-02T23:05:06.538Z" }, + { url = "https://files.pythonhosted.org/packages/c5/8e/4c24208776a65878d656996945aacfbfe010d3720d1a98fc0eb8491fc03b/markupsafe-3.0.4-cp312-cp312-win_arm64.whl", hash = "sha256:a4bbd2d87dd233b9fc5812160c3d0ffbe42edc22a26ce0469f58479ede633fe9", size = 14174, upload-time = "2026-10-02T23:05:07.617Z" }, + { url = "https://files.pythonhosted.org/packages/6d/18/4bc5ba32499e87bb2b0ef5b3a9bb9c00a131fa961ddf0be548cb550f548b/markupsafe-3.0.4-cp313-cp313-android_24_arm64_v8a.whl", hash = "sha256:de8b364c423ef0a4bad9069657d617f9a5d2b2062457a89b1fa16ee199c399c1", size = 12786, upload-time = "2026-10-02T23:05:08.709Z" }, + { url = "https://files.pythonhosted.org/packages/4e/6f/17f0c099bf25f3e31e63cc19244d9f6af861a9a4ab778c203997903cfdd0/markupsafe-3.0.4-cp313-cp313-android_24_x86_64.whl", hash = "sha256:34bdde374c5932765d7dc685c4a1d191a3207852d67e8e0a9eb6ea85156181f1", size = 12510, upload-time = "2026-10-02T23:05:09.93Z" }, + { url = "https://files.pythonhosted.org/packages/11/af/1a141081b905036ee904ec4bd945e1f70b4e1b32d33c4e59e8cf1d58b247/markupsafe-3.0.4-cp313-cp313-ios_13_0_arm64_iphoneos.whl", hash = "sha256:6bd9e1788e15bfcf6a9082de42e30387e7b85d211ab21e57a939bb8cfaaf8d96", size = 11583, upload-time = "2026-10-02T23:05:10.884Z" }, + { url = "https://files.pythonhosted.org/packages/e7/0a/a89385ae590232622a03e091805cff12f24fabe6c11e0e8bae096cece81c/markupsafe-3.0.4-cp313-cp313-ios_13_0_arm64_iphonesimulator.whl", hash = "sha256:5066b244f576f91afc8ee3ba029a89f99d39c79b1853fe9d39bea9f0afbec148", size = 11801, upload-time = "2026-10-02T23:05:11.913Z" }, + { url = "https://files.pythonhosted.org/packages/ed/85/ea548dc013962eb73653124bc595635fbf9e0fa41d1f181a967ccb784dfb/markupsafe-3.0.4-cp313-cp313-ios_13_0_x86_64_iphonesimulator.whl", hash = "sha256:7a83aa6e4805df46fed18e989d3d16f86ef60cb50bbc8d9ce3a6be89165fbf6e", size = 11443, upload-time = "2026-10-02T23:05:12.887Z" }, + { url = "https://files.pythonhosted.org/packages/cc/72/15f2e5ec9cf2eb00d5cdfe968d94e4156a7bd7303832c3f3b2c403a36839/markupsafe-3.0.4-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:2d1b7d9308288661f56672b1b157d75fc536714d3638487bbea17b6318a78248", size = 11665, upload-time = "2026-10-02T23:05:13.829Z" }, + { url = "https://files.pythonhosted.org/packages/ca/e0/4030bea613677e333c8a2c901fd405055f657f9d06acba5b7357984b6ef7/markupsafe-3.0.4-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:73e77980c7207854f00fc4e71fb1626868d5740ab4012623d55c7a99ad122a72", size = 12061, upload-time = "2026-10-02T23:05:14.807Z" }, + { url = "https://files.pythonhosted.org/packages/f3/a5/28b76a7449eb702966b88bef599e2360b411fbb3afeee8fe560939be06ec/markupsafe-3.0.4-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:7018d4af1cd272e847aa5917983ab5e83e4f6579f9dbfecd4a79c0ca80b144c2", size = 24395, upload-time = "2026-10-02T23:05:15.909Z" }, + { url = "https://files.pythonhosted.org/packages/07/6c/21232811afc3a063b5e934b1ae2efda52f46154ec382f585149c020e61fe/markupsafe-3.0.4-cp313-cp313-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:c90d5b3d4e944e065a301d741b3c1d784f6bd1f503aa68b4967e32b2ba313d85", size = 27183, upload-time = "2026-10-02T23:05:16.976Z" }, + { url = "https://files.pythonhosted.org/packages/14/38/6ccdfa5b59049cb36fb80cbc80aee9cf1fc9bb77d1335ad435f2070b08cf/markupsafe-3.0.4-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:18a801868a884f216e784d7d14db2a4077143ce7610440aee2ce8f734e7cfcde", size = 24772, upload-time = "2026-10-02T23:05:18.209Z" }, + { url = "https://files.pythonhosted.org/packages/63/e0/cec6865dfe88cb48fedd4b20aed6af5158e41092adcbf3e028bcc6ec2108/markupsafe-3.0.4-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:434139499bb20b502ed3baa1f169e618f924a97e7a777fea1a49446d80106cf6", size = 23017, upload-time = "2026-10-02T23:05:19.286Z" }, + { url = "https://files.pythonhosted.org/packages/ee/76/6ed4940bb7648a9aac457c14f870cfdd5105f139a0fb1f29cd61fafa47d1/markupsafe-3.0.4-cp313-cp313-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:9e227f3dbe6bde7491cf0a9965d00b88c6b1a4a95d11480ddf88bb96d397c19f", size = 22036, upload-time = "2026-10-02T23:05:20.352Z" }, + { url = "https://files.pythonhosted.org/packages/a1/4f/ed476226d4fe46a09090a36025bf319296810028df55eb12f1253b540f3a/markupsafe-3.0.4-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:b8cd1f918b26fd7b1832ece557cc18f2d8747309ff8b3f0ef9d4250c5ad67a39", size = 23786, upload-time = "2026-10-02T23:05:21.576Z" }, + { url = "https://files.pythonhosted.org/packages/9a/35/66ff30450e35ef5fba9ebc930c9411747e537fd9447b65e44f5007e2b84d/markupsafe-3.0.4-cp313-cp313-musllinux_1_2_armv7l.whl", hash = "sha256:a5fcffb37e602b0b3c1638a97746b9b96125caa9bcf6fa41d337a9261de231ee", size = 22672, upload-time = "2026-10-02T23:05:22.922Z" }, + { url = "https://files.pythonhosted.org/packages/32/0b/72f45ce4b4efcbca4b80cf1b06703eff0be8d37e82abb78f66c85a7ead1e/markupsafe-3.0.4-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:5989cb26b2e1efc6a42216a9f6b5ee495ce5ace2e5b352a9af489976b32d1ee2", size = 24107, upload-time = "2026-10-02T23:05:24.175Z" }, + { url = "https://files.pythonhosted.org/packages/d2/03/71776e5fdcba04614b384cc102e8a4198208579d896fd1394cb7cb9aa900/markupsafe-3.0.4-cp313-cp313-musllinux_1_2_riscv64.whl", hash = "sha256:add96447a86d205ab616665d53b2950ee81083757f56e6ea833c8b2917646b46", size = 21615, upload-time = "2026-10-02T23:05:25.215Z" }, + { url = "https://files.pythonhosted.org/packages/ab/5f/801ce02a02e7aee0f784b1ec7843026178f6adeb9c93ac67eb1992a9a84d/markupsafe-3.0.4-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:2628d3a8cb648ecebb3c5d6b0a1052d400e4d8b7ac0fb786be8d285b50040d17", size = 23076, upload-time = "2026-10-02T23:05:26.423Z" }, + { url = "https://files.pythonhosted.org/packages/4a/85/c43776625428f3bb4a61e8633940400e3efe6409e3c6f5bff26de5e45618/markupsafe-3.0.4-cp313-cp313-win32.whl", hash = "sha256:672d207103e6b16ca098611b0f9efad6bc00afd47c03d6ef62186495ca677dc0", size = 14118, upload-time = "2026-10-02T23:05:27.716Z" }, + { url = "https://files.pythonhosted.org/packages/6f/36/163da64de88a13db79214ef75fa041be7fa13bdb42261cf5b7484de14bfb/markupsafe-3.0.4-cp313-cp313-win_amd64.whl", hash = "sha256:1f1f9477e174582b0a1b583d60b66e1f2cf5d3fe12cee985e4aedf44766600e5", size = 14313, upload-time = "2026-10-02T23:05:28.749Z" }, + { url = "https://files.pythonhosted.org/packages/9f/a8/9b662783ffaa1149221432a923cee562f78b9cbbb8baa3df9b3753e63e1e/markupsafe-3.0.4-cp313-cp313-win_arm64.whl", hash = "sha256:06de8ef6331f6e822c28d577dc8bf43fe398800477c49498f38fc38b67ff33fc", size = 14189, upload-time = "2026-10-02T23:05:29.917Z" }, + { url = "https://files.pythonhosted.org/packages/5c/c3/a944f3b0df22bd129e96915b9f4e98d2eeca6516687d7618304a966c3c74/markupsafe-3.0.4-cp314-cp314-android_24_arm64_v8a.whl", hash = "sha256:4ed644d75aa94a2baf7ec3a96eaa160ea58c742eb9d27c6506053c5c40fc84ed", size = 12787, upload-time = "2026-10-02T23:05:30.971Z" }, + { url = "https://files.pythonhosted.org/packages/d4/d6/a44863f69d88b6c7e27889108f70d47aed259edf89d5df3c5fca1eac87d6/markupsafe-3.0.4-cp314-cp314-android_24_x86_64.whl", hash = "sha256:6d2a9efe686f9de00d0d1ea32a4a5a86d558a2277501bd78d964214eab625e59", size = 12525, upload-time = "2026-10-02T23:05:32.263Z" }, + { url = "https://files.pythonhosted.org/packages/17/8f/168ba80e532dd6a93f96f8f706f1ad41d7990b6e1aeedc1cc0d211a33497/markupsafe-3.0.4-cp314-cp314-ios_13_0_arm64_iphoneos.whl", hash = "sha256:8781a792a070cf2bd1b86d3aa943894115faaba6e88122a7bf32d62072742453", size = 11589, upload-time = "2026-10-02T23:05:33.251Z" }, + { url = "https://files.pythonhosted.org/packages/32/b3/aa2c95a574d3af39403a469b295886eb9b6d448da568cbebb5a2cbfdc2e5/markupsafe-3.0.4-cp314-cp314-ios_13_0_arm64_iphonesimulator.whl", hash = "sha256:971a3bbb75d97ae4e2e8f7d4834236f86f85f0c85e04ab2e191db1123b04f80b", size = 11801, upload-time = "2026-10-02T23:05:34.315Z" }, + { url = "https://files.pythonhosted.org/packages/60/d0/34b810107d83840e768bf485de795893ebbae35b26ab061b487adfa0a692/markupsafe-3.0.4-cp314-cp314-ios_13_0_x86_64_iphonesimulator.whl", hash = "sha256:8909c2f1c6dd65e054ac4b573a91c8384d1492281e55d82d159d653f7a13adf6", size = 11447, upload-time = "2026-10-02T23:05:35.302Z" }, + { url = "https://files.pythonhosted.org/packages/6c/ab/2f8488f0f817a39fca068d2b17daf446bf5cdb3eae28c3720af534d873b4/markupsafe-3.0.4-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:4cf3468d5ec187ffffcaca8e61929a37448f215dafc1386a12c750a72fe53634", size = 11722, upload-time = "2026-10-02T23:05:36.363Z" }, + { url = "https://files.pythonhosted.org/packages/ad/40/e2d117b048d47282ade906fbfd92814cbee5647afc13fda88a3406039372/markupsafe-3.0.4-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:52704c5d36eb6dda8866493decd61111fff86244c9b1ad225ca01b9e91e5970f", size = 12061, upload-time = "2026-10-02T23:05:37.397Z" }, + { url = "https://files.pythonhosted.org/packages/9a/a8/73a81135e85ba66217f5af7facb03bbb386807e1a729ab64532e4c802652/markupsafe-3.0.4-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:1caa2fa5a6184fb233153b35f654e6687bd555476f6170f29d8ee9be1a8b0af9", size = 24434, upload-time = "2026-10-02T23:05:38.407Z" }, + { url = "https://files.pythonhosted.org/packages/ac/ca/fa9216dd01efee2dfdacafe7df32b4d0170fbac694b0c258a193d6e53999/markupsafe-3.0.4-cp314-cp314-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:387d8cd30e69b3f0a72877b9ae717033396404e19095b17fe89753a981fda44f", size = 27183, upload-time = "2026-10-02T23:05:39.581Z" }, + { url = "https://files.pythonhosted.org/packages/fa/4e/a469509e538d37af51103b17b073126973f2b1cbf197ff32c7ddf025cfe5/markupsafe-3.0.4-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:051417f74bcaaefa316276e0ff723f541616ca51043d070da00249d9bddd3e3c", size = 24813, upload-time = "2026-10-02T23:05:40.671Z" }, + { url = "https://files.pythonhosted.org/packages/8f/db/d7282caf7ab03af44d5d6fdbaa019b35c7d7f1c90588b839c07cba640d6a/markupsafe-3.0.4-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:a8e9f292fcda89b324f2f5c91d13f1424a153e40fc2756f38ee23b15835ff300", size = 23049, upload-time = "2026-10-02T23:05:41.864Z" }, + { url = "https://files.pythonhosted.org/packages/30/f3/b6a425206e6964efda6acee544d0eb01d1501784d0b8e2dcc74986f33b17/markupsafe-3.0.4-cp314-cp314-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:df1ae86ff54725a01fa1a0510b914ca53a161b7050be74f6204e24aded5971d0", size = 22088, upload-time = "2026-10-02T23:05:43.014Z" }, + { url = "https://files.pythonhosted.org/packages/ea/8a/84d3582fc1f0d5bd466cdf2eebf175e172158a6e70701aacec1de1b35430/markupsafe-3.0.4-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:8965520ac587c94a4ac48b729be3d8b8de00af39699b17585dfb599babe77977", size = 23846, upload-time = "2026-10-02T23:05:44.098Z" }, + { url = "https://files.pythonhosted.org/packages/1c/65/db101cce51b7ba4864ac491a9859d297dd1adf0e55b103fee9db9c47c527/markupsafe-3.0.4-cp314-cp314-musllinux_1_2_armv7l.whl", hash = "sha256:340cbb1957ba99929cbf19a75626d36ba1ae21d1730b287d1cf7f824a20c4fc7", size = 22674, upload-time = "2026-10-02T23:05:45.23Z" }, + { url = "https://files.pythonhosted.org/packages/e0/49/ddee9813d71db0c7a5c9d97c832125e6758a0c844777f1cf076569bb0e22/markupsafe-3.0.4-cp314-cp314-musllinux_1_2_ppc64le.whl", hash = "sha256:3a93d9616ddecfb393727a0041a562cf0b15a244e20f2bd25efc7949be4c4f17", size = 24104, upload-time = "2026-10-02T23:05:46.398Z" }, + { url = "https://files.pythonhosted.org/packages/aa/0e/7d8518d726726870a2399d69fd30d0fa36c5e57a2132c336b58d7c491073/markupsafe-3.0.4-cp314-cp314-musllinux_1_2_riscv64.whl", hash = "sha256:d2e56fd3b00222722abfb3f5f0759ddbae4b90811b5ad4343c64030ad1bde70c", size = 21637, upload-time = "2026-10-02T23:05:47.48Z" }, + { url = "https://files.pythonhosted.org/packages/b4/b0/b505e8a361ba557dbf3b3aa7331ea39b00d2022a26e925ff8463b9714bb3/markupsafe-3.0.4-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:0d9c47709875fdb321452056622e930c52afbc07a7d780762fbb8b4d91ce6fa4", size = 23081, upload-time = "2026-10-02T23:05:48.611Z" }, + { url = "https://files.pythonhosted.org/packages/1c/ea/9cc3cea873f980c75cbdb6f4277ce30ee955de38be0b3d02f14c108e0698/markupsafe-3.0.4-cp314-cp314-win32.whl", hash = "sha256:38fc55594dab834470b6733dead2ee9e3f657fb0608c769dcafa0ba5ab52f45c", size = 14272, upload-time = "2026-10-02T23:05:49.707Z" }, + { url = "https://files.pythonhosted.org/packages/80/f0/5792ff768a410f93ee3f84fc19345295ffc352d2c936b424cb37e514714c/markupsafe-3.0.4-cp314-cp314-win_amd64.whl", hash = "sha256:c1bc67752d5f21013cfe430df4062441714eab79f65a6a05e01505957e9c35fe", size = 14501, upload-time = "2026-10-02T23:05:50.788Z" }, + { url = "https://files.pythonhosted.org/packages/5f/cf/3d074a8edffcc6899355232ff2543ae8d929733239596423b7db79698bc9/markupsafe-3.0.4-cp314-cp314-win_arm64.whl", hash = "sha256:7e1636da3d8dfc220b6dd10264db5f2b165e4888c4518594898fbe381049af8a", size = 14364, upload-time = "2026-10-02T23:05:51.857Z" }, + { url = "https://files.pythonhosted.org/packages/d9/31/87ce42159aae2163cf3bbbd0c44bc87780510eecab1ea3859099aed95dcb/markupsafe-3.0.4-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:805c8b84534fa10891890f0e4be39f3a99e94615d93e8836bf9fa1fdca2feeb2", size = 11759, upload-time = "2026-10-02T23:05:52.951Z" }, + { url = "https://files.pythonhosted.org/packages/5f/53/b047207eeb7752e960aca3eb1df5fb7eefa7dd4c62ac49bb156456c8a702/markupsafe-3.0.4-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:fa95848c929b6a75f6848d3c9793e59db365ee436776e57db835cdbfa79ba977", size = 12096, upload-time = "2026-10-02T23:05:54.066Z" }, + { url = "https://files.pythonhosted.org/packages/ee/51/4326c88a13c7b755657d44b4bb986f8c3d9843ecba7e22d98661d87f9a57/markupsafe-3.0.4-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:e916035e3e9930cbdfdd10abf48861340221857f45509565898e012263f7b289", size = 25699, upload-time = "2026-10-02T23:05:55.15Z" }, + { url = "https://files.pythonhosted.org/packages/f2/bb/990581b7474bfcf2cf34bed6ba5ea23bd87adb9d671213d68e88620e7a6b/markupsafe-3.0.4-cp314-cp314t-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:b4d12837e0203bbace818ff4a7461afdcd78bcd782351cea148139180d7bcffe", size = 27804, upload-time = "2026-10-02T23:05:56.29Z" }, + { url = "https://files.pythonhosted.org/packages/6b/89/89491878c28e8291f5aa2fffe2c2d57230d10ae366d55dd810b840513d78/markupsafe-3.0.4-cp314-cp314t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:5086f9975abb1ab531ee6afca1761e4b59a19b446f3f6522ed776963228cfe5a", size = 25912, upload-time = "2026-10-02T23:05:57.416Z" }, + { url = "https://files.pythonhosted.org/packages/30/77/680998b54efdea06fc114565cd739b6d059f826a0279219b218dfa750d29/markupsafe-3.0.4-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:b4a635a0487774f841cb1fb62e907e7195cc95bc761e053184b8acc3ceb20733", size = 23623, upload-time = "2026-10-02T23:05:58.557Z" }, + { url = "https://files.pythonhosted.org/packages/ae/75/2709f5ac5de9467b40b10e2bb8f89cc63dfb74582e09aa734b1124a217de/markupsafe-3.0.4-cp314-cp314t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:cb96e6e088d6cf71c1ea977510948320234824cf226e32f6f6e044f7a9c82b34", size = 23301, upload-time = "2026-10-02T23:05:59.94Z" }, + { url = "https://files.pythonhosted.org/packages/a0/c8/39eadc6c5b14c9c7679bfb98f4d4c6a97863b5beb91839aca4d2d6e16e55/markupsafe-3.0.4-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:8b5d563170ff8ba3181caa967c99a3c804d1dedb702c7cb93a6a7c32247da978", size = 24950, upload-time = "2026-10-02T23:06:01.289Z" }, + { url = "https://files.pythonhosted.org/packages/1a/5e/01037f8a43e8ccb0bffb4fbdc5212db05bf080fdd7286cd392332d58128a/markupsafe-3.0.4-cp314-cp314t-musllinux_1_2_armv7l.whl", hash = "sha256:396ec4e65cc889f69786b3b89478b471cee5a3bcf468b9d9bb03e1a30fb291fc", size = 23330, upload-time = "2026-10-02T23:06:02.441Z" }, + { url = "https://files.pythonhosted.org/packages/d4/f4/23e83ce0596bb0cbe670502d31df8f757bbd01a392aa486fa3b40d1ed399/markupsafe-3.0.4-cp314-cp314t-musllinux_1_2_ppc64le.whl", hash = "sha256:15ba9e28640feef770374b116a6f019c21f52404aeabe516aa7f800587b98cfc", size = 25239, upload-time = "2026-10-02T23:06:03.579Z" }, + { url = "https://files.pythonhosted.org/packages/88/5b/3708897368073cc683d524750474f41a77d2986152c380dcc55b20fdf340/markupsafe-3.0.4-cp314-cp314t-musllinux_1_2_riscv64.whl", hash = "sha256:d920abdfa61279ba1a2ef9484aab07bf03331f8c08a10120fa332353d06e6932", size = 22767, upload-time = "2026-10-02T23:06:04.699Z" }, + { url = "https://files.pythonhosted.org/packages/c6/61/ebda1307864b409e6b3115757a3d4a09cca46cfb6cc65191b5de226b424b/markupsafe-3.0.4-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:a9f54054101545a9a9cccefddf54316aa6e4491611fcbef9e91b3b6bebec04f6", size = 23668, upload-time = "2026-10-02T23:06:05.9Z" }, + { url = "https://files.pythonhosted.org/packages/09/15/98075cceac3b5ba0dbb8e4762a847be967d2befc349a2cf2d0ac77f62c9d/markupsafe-3.0.4-cp314-cp314t-win32.whl", hash = "sha256:12a606a492de952afcb43b59a14aaaaad120e708d3663dd0fdf2d738d427a691", size = 14301, upload-time = "2026-10-02T23:06:07.109Z" }, + { url = "https://files.pythonhosted.org/packages/0b/a3/768b560fcc4156685cb563d922b217810cfa7bc135773367f62f1f9d2078/markupsafe-3.0.4-cp314-cp314t-win_amd64.whl", hash = "sha256:a18f38cafc329bac5e3c2b96c765b4c96d3d103421ed22ab7988c1e3fce27464", size = 14526, upload-time = "2026-10-02T23:06:08.276Z" }, + { url = "https://files.pythonhosted.org/packages/93/63/da554b4c97a6b0ea3229ca7fe8cbfb620be81613d517f482e85958550537/markupsafe-3.0.4-cp314-cp314t-win_arm64.whl", hash = "sha256:eba154571c16e032112afac0dc2dfe9e63c2ceb7aedd07bb7eecf2ce26d4dd4c", size = 14421, upload-time = "2026-10-02T23:06:09.402Z" }, + { url = "https://files.pythonhosted.org/packages/a9/30/54d11c8ca027114898cab97421fb39e4ffd9ddf47cdbc44df2ec76722da9/markupsafe-3.0.4-cp315-cp315-android_24_arm64_v8a.whl", hash = "sha256:737c9c3981998eba27f11786f84fddcbabc74068b72a4a1f454ea02094b57b65", size = 12791, upload-time = "2026-10-02T23:06:10.485Z" }, + { url = "https://files.pythonhosted.org/packages/10/6d/97c913e253a14bd3cd0e15a5c56d13203b823fa7ee32498342896a072dc4/markupsafe-3.0.4-cp315-cp315-android_24_x86_64.whl", hash = "sha256:489505b03f692c3f376394e49194fa7a7f9e8558d6e293a7056a0032b0c38163", size = 12527, upload-time = "2026-10-02T23:06:11.834Z" }, + { url = "https://files.pythonhosted.org/packages/26/f9/b86d032042a4d597d9e1997f0e5f63a3eedaf11258e0a05760b0a0a826ea/markupsafe-3.0.4-cp315-cp315-ios_13_0_arm64_iphoneos.whl", hash = "sha256:077293e425f28ec737dbcad442a71752e28f8ae27cde3d68acd1fb212091cd92", size = 11602, upload-time = "2026-10-02T23:06:13.122Z" }, + { url = "https://files.pythonhosted.org/packages/f2/dc/73c14c1eedf0ac5fa3292ba43435e6c49d2c2050f33cebde541f8f4807f1/markupsafe-3.0.4-cp315-cp315-ios_13_0_arm64_iphonesimulator.whl", hash = "sha256:9348cbb300d224fe3b89793262cb093504d4ae927004468463f745188a193e4a", size = 11817, upload-time = "2026-10-02T23:06:14.227Z" }, + { url = "https://files.pythonhosted.org/packages/8f/69/2c2fcaa5fcee22d72c7819c0d536fd181c74a688e6143845419579cd2863/markupsafe-3.0.4-cp315-cp315-ios_13_0_x86_64_iphonesimulator.whl", hash = "sha256:b807e598953730f82e4eae3bd30f6a122cf6b31c398c6b504c0e04c13c170429", size = 11446, upload-time = "2026-10-02T23:06:15.574Z" }, + { url = "https://files.pythonhosted.org/packages/88/54/9e5ec76c62e6e2834d5a93623018c943e8b3bb41d663e3fd4c03303b9b85/markupsafe-3.0.4-cp315-cp315-macosx_10_15_x86_64.whl", hash = "sha256:799c39bdf5e2f1292fedd3009f7b3c9e760f10b2420cb9638d56920840ff6db8", size = 11719, upload-time = "2026-10-02T23:06:16.701Z" }, + { url = "https://files.pythonhosted.org/packages/96/24/3ec292b44064c16229e064d770b2625bd8ea941aa61f44905a9fa44942c0/markupsafe-3.0.4-cp315-cp315-macosx_11_0_arm64.whl", hash = "sha256:ae9dcb8fbe244cb82f8a6458b455b927a03685e383d9bacf1ea5ce180b96dc97", size = 12075, upload-time = "2026-10-02T23:06:17.855Z" }, + { url = "https://files.pythonhosted.org/packages/aa/85/b64fdb1f304848518742136983c24e96d967bfb59a0ea160e92736901ab0/markupsafe-3.0.4-cp315-cp315-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:4bced6e2a6dba6a28f7dd3c6ce14df1b2dd495923f16ea484cad03decd463b2b", size = 24656, upload-time = "2026-10-02T23:06:18.963Z" }, + { url = "https://files.pythonhosted.org/packages/9c/18/23997d4c65b355da6390d61cd56e0ab3befd6ba8dda25cb40c602bd0fa6b/markupsafe-3.0.4-cp315-cp315-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:3882fb412298575bae3b9c46868251f15cc69307359f87bb1b382e53d6e5a2c9", size = 27737, upload-time = "2026-10-02T23:06:20.117Z" }, + { url = "https://files.pythonhosted.org/packages/d4/36/35998dead3c6af88c38265a56e58100211f036234ab88eb2283fd4cbce44/markupsafe-3.0.4-cp315-cp315-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:04e7902ba80ee4bac1d50a549606527a1dcf0476cd81403db41099d3b60ec653", size = 25003, upload-time = "2026-10-02T23:06:21.284Z" }, + { url = "https://files.pythonhosted.org/packages/82/96/ef49135ce260db4ca4a12b119ed468449cd248db6b1468e2112b546d7a2e/markupsafe-3.0.4-cp315-cp315-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:925f929d6b59a8b3f8b8c6ac363cd0af7eecc81efb3071770b3c6717c450a369", size = 23111, upload-time = "2026-10-02T23:06:22.524Z" }, + { url = "https://files.pythonhosted.org/packages/50/7d/83126e338bd88c17a220668235368ad719fd4638e426739858cbb8508f77/markupsafe-3.0.4-cp315-cp315-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:f68edfc67aabac33708941f26f22a7b8e9f81429bc0cf249fcf7d66b23af8d19", size = 22403, upload-time = "2026-10-02T23:06:23.785Z" }, + { url = "https://files.pythonhosted.org/packages/83/dd/daf7e420de23c8206c365204e7b85e1251d8e19d34196a56336f316e5ed2/markupsafe-3.0.4-cp315-cp315-musllinux_1_2_aarch64.whl", hash = "sha256:e5c802729725bd07e2bc3ab7b76dc7e0bbfc53129d8f1eb1c002c24cf774717e", size = 24083, upload-time = "2026-10-02T23:06:25.037Z" }, + { url = "https://files.pythonhosted.org/packages/19/3c/11eecdc06bc44ad5570350085b572ebf049e8f9a38d1ece6d76640b739cd/markupsafe-3.0.4-cp315-cp315-musllinux_1_2_armv7l.whl", hash = "sha256:55ffd6ce583d97dc71dc92e930324c8c0d25aea7e3ade6ae54ef77cedb096811", size = 22931, upload-time = "2026-10-02T23:06:26.328Z" }, + { url = "https://files.pythonhosted.org/packages/0d/9e/ac0fd77f2a726e56ecc3ca0235d095feace1358d1b822406c2a2ef26a4dc/markupsafe-3.0.4-cp315-cp315-musllinux_1_2_ppc64le.whl", hash = "sha256:2cb3dd71fc6be918ad4264346a8ed69485f9b7ed7bf35495d8e22807cd6b8bea", size = 24299, upload-time = "2026-10-02T23:06:27.742Z" }, + { url = "https://files.pythonhosted.org/packages/d7/09/c6bd842ad58ff5b3bc76eeed7e9a42a6f11adc5d090ec697b72c9672731e/markupsafe-3.0.4-cp315-cp315-musllinux_1_2_riscv64.whl", hash = "sha256:94f5407f7bc64fa6463906b896f9904beeeb7dd8dc116ee8e9056c8714ff9916", size = 21948, upload-time = "2026-10-02T23:06:29.274Z" }, + { url = "https://files.pythonhosted.org/packages/a3/46/82f586711fed61e86faa1ee1bc317d68cd45a10c8bdbe3f7d1fdf9026ad8/markupsafe-3.0.4-cp315-cp315-musllinux_1_2_x86_64.whl", hash = "sha256:2dad610540cb2e6272855c178f08ae9a1c7ac258a7fb71660553a5f104b42741", size = 23150, upload-time = "2026-10-02T23:06:30.583Z" }, + { url = "https://files.pythonhosted.org/packages/19/2d/2dfdce99318abbfa26925195fbc17db188c46a1ec6457be121b6f9cfeb42/markupsafe-3.0.4-cp315-cp315-win32.whl", hash = "sha256:03470d1a8268e692ecf79ecd565593e59d44219377a7ead61f1f1b94c1f7ff6b", size = 14269, upload-time = "2026-10-02T23:06:31.949Z" }, + { url = "https://files.pythonhosted.org/packages/5b/ec/6000fd82e8791e58fcd0456ec20f098957e2b03d5ed02eb73241a577c0ba/markupsafe-3.0.4-cp315-cp315-win_amd64.whl", hash = "sha256:d882a373d8093c2941e01291b7ced96e9cbe4781da9a7751ca7e6c70385e5214", size = 14496, upload-time = "2026-10-02T23:06:33.258Z" }, + { url = "https://files.pythonhosted.org/packages/bc/66/e73bd5016421d5d6e2fb6de7dd609f9de020942ac8c626526bd8c6eeaf82/markupsafe-3.0.4-cp315-cp315-win_arm64.whl", hash = "sha256:353bd63081912ab8cfa6a0c7d185934cdf8426f04c618bba6bc4b394f2069b67", size = 14361, upload-time = "2026-10-02T23:06:34.539Z" }, + { url = "https://files.pythonhosted.org/packages/90/df/cb8c3dc98d313a951df2f8968f44e4cb5643df6d3cab749a530ce2f7d972/markupsafe-3.0.4-cp315-cp315t-macosx_10_15_x86_64.whl", hash = "sha256:c61750fadcd119d0825bcb7d7d675dd264dcc89cc05292aab5be68ebdbb374ad", size = 11760, upload-time = "2026-10-02T23:06:35.807Z" }, + { url = "https://files.pythonhosted.org/packages/d6/bb/4af9b3ca0753d654ac75f9531d5bd741bb77ca6e696f36807c475ffc099a/markupsafe-3.0.4-cp315-cp315t-macosx_11_0_arm64.whl", hash = "sha256:1c0df495a977d10460a94941799c72d5b5ab03d3858d949b55b5a66c8f371c99", size = 12108, upload-time = "2026-10-02T23:06:37.089Z" }, + { url = "https://files.pythonhosted.org/packages/3f/d4/b56429313aee5fd59b079c3df5615299959e25e7113eb6d8caadbdd7d38a/markupsafe-3.0.4-cp315-cp315t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:02fa4acbc6a3fc5c693c34d4dd8c1130b7fe99cc915181b0ddd6f72aeb296002", size = 25762, upload-time = "2026-10-02T23:06:38.419Z" }, + { url = "https://files.pythonhosted.org/packages/65/f5/34c181e891aa4f7d59c918584672e0c5eb7fffe76c1387d1246008bf4081/markupsafe-3.0.4-cp315-cp315t-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:05295589e619b9bed252a86b532b8e27350abc372d18ba89b59375325e91ec1e", size = 28400, upload-time = "2026-10-02T23:06:39.819Z" }, + { url = "https://files.pythonhosted.org/packages/ce/b5/ad14694fd0ac9a5ce30bc6498f2999378f418583dd1679cca5a1b512957e/markupsafe-3.0.4-cp315-cp315t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:be6cb0c799abb0e2ba3e618e6d28ddddf7e485f6c2ce938dfa237daf3905072c", size = 25946, upload-time = "2026-10-02T23:06:41.381Z" }, + { url = "https://files.pythonhosted.org/packages/d6/a8/26b606445387d0ceb1eb1f21840094b84e4e3c3c3983d80d10b89823b490/markupsafe-3.0.4-cp315-cp315t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:26e9867520db70d37f7fb421a7f0d8adb40171011fb84ce869afa1a83370dfa8", size = 23634, upload-time = "2026-10-02T23:06:42.748Z" }, + { url = "https://files.pythonhosted.org/packages/39/a2/b8814de672f1f0094d498bf646f2fec9d6356b503d28ef500b71c5095377/markupsafe-3.0.4-cp315-cp315t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:f03460ff076f70ab595bb45a0205ccea1971443575b6920c52e755dec2b3fbfe", size = 23488, upload-time = "2026-10-02T23:06:44.176Z" }, + { url = "https://files.pythonhosted.org/packages/db/c7/287223376fb73335a3cc5d6eb22c6ab01358cf33945a9c39c06b9dac3f4b/markupsafe-3.0.4-cp315-cp315t-musllinux_1_2_aarch64.whl", hash = "sha256:436e3ffc6310d3c41878c601db29098102fe5d8a467c49da4a4125254e0980f2", size = 25037, upload-time = "2026-10-02T23:06:45.646Z" }, + { url = "https://files.pythonhosted.org/packages/f9/29/4df8355e313426d19e62ba33e0253c009ca12a0894ee77d67fa67255361c/markupsafe-3.0.4-cp315-cp315t-musllinux_1_2_armv7l.whl", hash = "sha256:4e2c4809c14559aa7ef426f27fb35afbb38104c349a903bf8f3600456764bb38", size = 23543, upload-time = "2026-10-02T23:06:47.264Z" }, + { url = "https://files.pythonhosted.org/packages/71/e5/8377731e8495668dcc768f645e717df18318c841edaf023a99395f6da9b4/markupsafe-3.0.4-cp315-cp315t-musllinux_1_2_ppc64le.whl", hash = "sha256:da2af0d7aebfc2074080d72efa6ab8317c62481ef1f896f65d9999c1c01f4494", size = 25282, upload-time = "2026-10-02T23:06:48.795Z" }, + { url = "https://files.pythonhosted.org/packages/ed/5f/373456e37ceb1478d657d6fe769cbe0a39f0a8dfc1548eeb19c471eefdd9/markupsafe-3.0.4-cp315-cp315t-musllinux_1_2_riscv64.whl", hash = "sha256:aa2c838cc024642cc04c6854232f32b43e5e22833dd11119c1766c7873b8370d", size = 22993, upload-time = "2026-10-02T23:06:50.31Z" }, + { url = "https://files.pythonhosted.org/packages/d7/93/2cbd5628435afb6f541bbaced4bce0c2edac4b09a142e6e928b8b0da9858/markupsafe-3.0.4-cp315-cp315t-musllinux_1_2_x86_64.whl", hash = "sha256:b91cc9d336957239ff200f30097e6fea2dc6d6fb3c81e853eaa09eac904fd894", size = 23712, upload-time = "2026-10-02T23:06:51.759Z" }, + { url = "https://files.pythonhosted.org/packages/81/99/157e10966b033b363aeda5263e82596ee232a0b1d082fdbf90aa417ff083/markupsafe-3.0.4-cp315-cp315t-win32.whl", hash = "sha256:e49fb0d1ce92cfa0cb198cc5b1b11cdf9d0638658e2a2db2687e39db7c87fc78", size = 14300, upload-time = "2026-10-02T23:06:53.241Z" }, + { url = "https://files.pythonhosted.org/packages/33/05/55884815414c9706a23deca150b72c25a62109e65b0b6ce232077802c719/markupsafe-3.0.4-cp315-cp315t-win_amd64.whl", hash = "sha256:4f6e0852a0283b1b1fd776eeb7b766a5f440b3e2bd31ab51af3b400585f3965c", size = 14522, upload-time = "2026-10-02T23:06:54.729Z" }, + { url = "https://files.pythonhosted.org/packages/92/f9/ecbde7149e95b8a0f18e16d5d747f7dc06049d5da2e4f77f6f5e4a1f46a8/markupsafe-3.0.4-cp315-cp315t-win_arm64.whl", hash = "sha256:39dbacefc411633db5b4378b066a9aca70a3d7e2922c9e578d825f844026eeba", size = 14417, upload-time = "2026-10-02T23:06:56.246Z" }, +] + [[package]] name = "openai" version = "2.53.0" @@ -616,6 +774,36 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/78/0f/cc6afea3542a5142c5d8fc8211c5e059a8375105d004a41dfa2c7948dbb0/openai-2.53.0-py3-none-any.whl", hash = "sha256:c694ffc747a3c4d1663ef2b07b811315a476164ee5efa3a993967349ebca7618", size = 1659829, upload-time = "2026-08-03T21:41:59.581Z" }, ] +[[package]] +name = "openpyxl" +version = "3.1.5" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "et-xmlfile" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/3d/f9/88d94a75de065ea32619465d2f77b29a0469500e99012523b91cc4141cd1/openpyxl-3.1.5.tar.gz", hash = "sha256:cf0e3cf56142039133628b5acffe8ef0c12bc902d2aadd3e0fe5878dc08d1050", size = 186464, upload-time = "2024-06-28T14:03:44.161Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/c0/da/977ded879c29cbd04de313843e76868e6e13408a94ed6b987245dc7c8506/openpyxl-3.1.5-py2.py3-none-any.whl", hash = "sha256:5282c12b107bffeef825f4617dc029afaf41d0ea60823bbb665ef3079dc79de2", size = 250910, upload-time = "2024-06-28T14:03:41.161Z" }, +] + +[[package]] +name = "packaging" +version = "26.3" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/7d/fa/3944b40b07da9ce895c0e6303a5ab7d53da063554f534556b134a54d6093/packaging-26.3.tar.gz", hash = "sha256:94edc256424af38762eb31306eed28beb9f0efc50a8837492c9d6fd6004aed79", size = 313412, upload-time = "2026-08-04T18:15:28.737Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/63/34/ba1c580383c9eada3711951fef0795c80b829a078d72188184bcab9dd527/packaging-26.3-py3-none-any.whl", hash = "sha256:d7193f7c8e4e93f444fde0262bf90af30e16fa0ad0ad44cb553c87339b23cd1c", size = 129956, upload-time = "2026-08-04T18:15:27.159Z" }, +] + +[[package]] +name = "pluggy" +version = "1.6.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/f9/e2/3e91f31a7d2b083fe6ef3fa267035b518369d9511ffab804f839851d2779/pluggy-1.6.0.tar.gz", hash = "sha256:7dcc130b76258d33b90f61b658791dede3486c3e6bfb003ee5c9bfb396dd22f3", size = 69412, upload-time = "2025-05-15T12:30:07.975Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/54/20/4d324d65cc6d9205fabedc306948156824eb9f0ee1633355a8f7ec5c66bf/pluggy-1.6.0-py3-none-any.whl", hash = "sha256:e920276dd6813095e9377c0bc5566d94c932c33b27a3e3945d8389c374dd4746", size = 20538, upload-time = "2025-05-15T12:30:06.134Z" }, +] + [[package]] name = "pyasn1" version = "0.6.4" @@ -763,6 +951,31 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/4b/2d/69abac8f838090bbecd5df894befb2c2619e7996a98ddb949db9f3b93225/pydantic_core-2.46.4-pp311-pypy311_pp73-win_amd64.whl", hash = "sha256:d51026d73fcfd93610abc7b27789c26b313920fcfb20e27462d74a7f8b06e983", size = 2193071, upload-time = "2026-05-06T13:38:08.682Z" }, ] +[[package]] +name = "pygments" +version = "2.21.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/49/2e/ced460408999b33da6b31b0021b0f37d329e202d4169aeb164493778f25b/pygments-2.21.0.tar.gz", hash = "sha256:610ca751c9bc2492b38eb9a38a7fbc93edbbb2d7182edaf34e66ae493dee5c8c", size = 5005329, upload-time = "2026-08-17T08:02:48.824Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/71/46/17f022dd3e953bf20a04a028a21ec746d942f8d2af30fa0f124fa0e6a684/pygments-2.21.0-py3-none-any.whl", hash = "sha256:2363c69b61c4a97c838da3b130dcd6468f4848992b21a82f2a63ec34377137d9", size = 1250147, upload-time = "2026-08-17T08:02:44.912Z" }, +] + +[[package]] +name = "pytest" +version = "9.1.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "colorama", marker = "sys_platform == 'win32'" }, + { name = "iniconfig" }, + { name = "packaging" }, + { name = "pluggy" }, + { name = "pygments" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/e4/47/b9efed96c114afcfa3c9d3fe98a76a1d14c74a9e266d397cf6eb64be5e01/pytest-9.1.1.tar.gz", hash = "sha256:1088fbde8f2b49d95a549a195707afa7a76a3ce9bcadc26b6d71f0ffda5fe313", size = 1636369, upload-time = "2026-06-19T10:58:32.857Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/24/25/1de2678b631f5a49215c6c96fff41ba892b0a34df68d6d80292b1b48aa7f/pytest-9.1.1-py3-none-any.whl", hash = "sha256:37a86b45efb9a47a61a36449063e8e18d0cab3161329fc099eb21783169c4f0c", size = 386536, upload-time = "2026-06-19T10:58:31.347Z" }, +] + [[package]] name = "python-dotenv" version = "1.2.2" @@ -864,23 +1077,35 @@ dependencies = [ { name = "anthropic" }, { name = "beautifulsoup4" }, { name = "google-genai" }, + { name = "jinja2" }, { name = "lxml" }, { name = "openai" }, + { name = "openpyxl" }, { name = "python-dotenv" }, { name = "requests" }, ] +[package.dev-dependencies] +dev = [ + { name = "pytest" }, +] + [package.metadata] requires-dist = [ { name = "anthropic", specifier = "~=0.84.0" }, { name = "beautifulsoup4", specifier = ">=4.9.2" }, { name = "google-genai", specifier = ">=1.0.0" }, + { name = "jinja2", specifier = ">=3.1.6" }, { name = "lxml", specifier = ">=4.9.0" }, { name = "openai", specifier = ">=1.0.0" }, + { name = "openpyxl", specifier = ">=3.1.5" }, { name = "python-dotenv", specifier = "~=1.2.1" }, { name = "requests", specifier = ">=2.32.2" }, ] +[package.metadata.requires-dev] +dev = [{ name = "pytest", specifier = ">=8.3.5" }] + [[package]] name = "websockets" version = "16.1.1" diff --git a/vision_aid/evaluation/README.md b/vision_aid/evaluation/README.md new file mode 100644 index 0000000..1d030bc --- /dev/null +++ b/vision_aid/evaluation/README.md @@ -0,0 +1,602 @@ +# Evaluator guide + +The evaluator compares va-dat's programmatic and LLM findings with an existing +human accessibility audit. A reviewer selects complete matches; the evaluator +validates the evidence and calculates detection rates. Each invocation uses one +fixed model preset against saved HTML. + +Start with the [quick start in the repository README](../../README.md#evaluator-quick-start) +for the complete command sequence. **Run all commands in this guide from the +repository root**, the `va-dat/` directory containing `pyproject.toml` and the +root README. All relative paths in the examples start there, even though this +guide lives in `vision_aid/evaluation/`. + +- [Inputs and command reference](#inputs-and-command-reference) +- [Capture and reuse snapshots](#capture-and-reuse-snapshots) +- [Approve a human finding mapping](#approve-a-human-finding-mapping) +- [Model settings and execution](#model-settings-and-execution) +- [Evaluate one page](#evaluate-one-page) +- [Review LLM matches](#review-llm-matches) +- [How scoring works](#how-scoring-works) +- [How responses become findings](#how-responses-become-findings) +- [Reports and saved files](#reports-and-saved-files) +- [Troubleshooting and compatibility](#troubleshooting-and-compatibility) + +## Inputs and command reference + +An evaluation needs a human audit Excel file, saved page HTML, and an approved human +finding mapping. Live execution also needs the selected provider's API key and +a positive spending allowance. Preview requires the approved mapping and a +positive allowance, but needs no key and makes no model requests. + +Place the human audit Excel file, `Pristine Accessibility Defect Report.xlsx`, +directly in the repository root, alongside `pyproject.toml` and the root README. +The commands below expect it at `./Pristine Accessibility Defect Report.xlsx`. + +The file and mapping are private local inputs, excluded from Git. Mappings +are not installed as package data. A fresh checkout must supply an approved +mapping or create one through CSV review. Obtain the project's human audit Excel file from +your project team or use a compatible audit of your own. + +| Command | Purpose | Model requests | +| --- | --- | --- | +| `capture` | Download each URL listed in the human audit file and save `snapshots.json`. | None; downloads web pages. | +| `human-finding-mapping-prepare` | Create classification and programmatic-match review CSVs from saved inputs. | None. | +| `human-finding-mapping-approve` | Validate completed CSVs and export an approved mapping. | None. | +| `run` | Prepare one page and optionally execute its prompts. | Only with `--live` and approval. | +| `run-all` | Prepare every listed page and optionally execute them with one shared allowance. | Only with `--live` and approval. | +| `report` | Validate one completed page's review and generate its HTML report. | None. | +| `report-all` | Validate a completed full evaluation and generate combined and page reports. | None. | + +Use `uv run visionaid-evaluate COMMAND --help` for a command's options. + +### Human audit Excel file format + +The importer expects a human audit Excel file with a `Defect report` tab. +Its first row must begin with these exact headers, in this order: + +```text +Sr. # +element name +Browser Combination +Page name +Issue Title +Steps to Reproduce +actual result +expected result +Recommendation for Fix +WCAG Sc +Type of Change +comment +``` + +Each subsequent finding row needs an `Issue Title`. Original cell text, tab +name, and row number are preserved in the review evidence. Human finding IDs +derive from the entire file's SHA-256, tab, and source row. Changing the +file's bytes changes its identity and requires a new mapping review. + +Capture, full evaluation, and mapping authoring also require a `URL list` tab +whose first row includes a `URL` column. Non-empty URLs define page identity and +execution order. Full input loading rejects repeated target URLs. + +For each local finding, `Steps to Reproduce` must identify exactly one listed +target URL. Page names and recommendation URLs do not assign findings. Matching +uses the listed URL text; the known `/Contact/` alias maps to `/contact/`, while +other paths retain their case. Missing or ambiguous assignments stop preparation. +A finding whose `Page name` is `Global`, ignoring case and surrounding spaces, +applies to every listed page. Globals repeat in page reviews but retain one +original identity. + +The project's Pristine audit has 86 original findings across 17 pages. Repeated +Globals produce 198 page-review rows; the homepage has 16 Home or Global rows. +These counts describe that human audit file, rather than limits imposed by the +evaluator. + +The single-page importer retains Home and Global selection for older human audit +files without a `URL list`. The current mapping-authoring commands require that +tab; there is no CSV authoring route for those older files without adding it and +reviewing the resulting new human audit file identity. + +## Capture and reuse snapshots + +```bash +uv run visionaid-evaluate capture \ + --human-audit "Pristine Accessibility Defect Report.xlsx" \ + --output-dir .model-evaluation/pristine-snapshots +``` + +The output directory must be new. Capture downloads only listed URLs in source +order and follows redirects, retaining the originally listed URLs as page +identities. It saves `page-001.html`, `page-002.html`, and later files, then +writes `snapshots.json` with the audit checksum and relative HTML filenames. +Resolve filenames beside that mapping when reusing saved inputs. + +Capture saves server HTML. It does not render JavaScript, interact with the page, +or discover more URLs. Inspect the saved content; if important content requires +rendering, replace the relevant HTML with a rendered snapshot before mapping +review. Reuse those same snapshots across models. Downloading again can change +checker evidence and prevent an approved mapping from resolving. + +An HTTP or network error stops capture. Earlier HTML remains, but no snapshot +mapping is published until every download succeeds. Address the failure and +capture into a new directory. + +Full input loading requires the mapping's audit checksum and ordered URLs to +match the human audit file. HTML paths must be relative, remain within the mapping's +directory, and point to existing files. Later evaluation commands read these +saved files without downloading again. + +## Approve a human finding mapping + +A mapping records each original finding's classification and its approved +programmatic matches or explained absence of matches on every applicable page. +Approve this independently of model output, then reuse it across model runs. +LLM matches are reviewed separately after each execution. + +```bash +uv run visionaid-evaluate human-finding-mapping-prepare \ + --human-audit "Pristine Accessibility Defect Report.xlsx" \ + --snapshots .model-evaluation/pristine-snapshots/snapshots.json \ + --output-dir .model-evaluation/pristine-mapping-review +``` + +Preparation reads the human audit file and saved snapshots and runs existing +programmatic checks. It makes no network or model requests and requires no model +choice, credentials, or allowance. The output directory must be new. It contains +three CSVs and a README: + +| File | Review action | +| --- | --- | +| `classifications.csv` | Classify each original finding once, including each Global once. Fill `classification` and `classification_reason`. | +| `programmatic-review.csv` | Decide programmatic coverage for each applicable finding and page. Fill `programmatic_finding_refs` and `programmatic_reason`. | +| `programmatic-findings.csv` | Read-only checker catalog with references, page URLs, rule IDs, descriptions, elements, targets, and locations. | + +### Classifications + +| Value | Use it when | +| --- | --- | +| `llm_eligible` | Saved HTML contains enough evidence to judge the defect, and the issue needs semantic judgment or checks absent from existing rules. | +| `programmatic` | Existing checker findings fully cover the defect on every applicable page. Approval requires complete matches, rather than a rule that could theoretically detect it. | +| `unavailable_evidence` | Saved HTML lacks evidence needed to verify the defect, such as interaction or browser and assistive technology observations. | +| `ambiguous` | The human finding is too unclear to decide fairly whether tool findings cover it. | + +Explain each choice in `classification_reason`. A Global has one classification +shared across all its pages. The workflow does not infer classifications or +change checker rules. + +### Programmatic matches + +Copy references such as `P001-F003` from the catalog into the matching page row. +Separate joint matches with semicolons, such as `P001-F003;P001-F004`, and explain +how they cover the whole human defect. A shared WCAG criterion is insufficient. +Select only that page's findings and use each reference for at most one human +finding. For no complete match, leave references blank and explain the absence. +Every page row requires a deliberate decision. + +Programmatic classifications require complete matches on every applicable page. +LLM-eligible findings may have programmatic cross-catches. Unavailable and +ambiguous findings cannot claim matches. The [matching rules](#review-llm-matches) +also explain practical semantic coverage. + +Keep original evidence, identities, headers, and every row unchanged. Sorting +rows is allowed. Save the two edited files as CSV; keep the catalog unchanged. + +```bash +uv run visionaid-evaluate human-finding-mapping-approve \ + --human-audit "Pristine Accessibility Defect Report.xlsx" \ + --snapshots .model-evaluation/pristine-snapshots/snapshots.json \ + --review-dir .model-evaluation/pristine-mapping-review \ + --reviewer "Your name and review method" +``` + +Approval rereads the original inputs, reruns checks, and verifies that the +catalog and original CSV evidence still agree. Keep snapshots and checker code +unchanged during review. Changed checker results require a new review. +Approval validates completeness, classifications, page ownership, unique +matches, and finding reuse. The reviewer decides whether matches semantically +cover the defect; the command records that judgment. + +The output is `human-finding-mapping.json` in the review directory. Use `--output` +to choose another new filename. Approval refuses to overwrite an existing +mapping. Errors identify the affected file, finding, or page; edit the CSV and +retry. No reviewer needs to write JSON or calculate checksums. + +### Mapping identity and reuse + +Pass `--human-finding-mapping PATH` to `run` or `run-all`. Approval's default +output location is not automatically searched. Without an explicit path, the +CLI looks only for +`vision_aid/evaluation/human_finding_mappings/.json`, an ignored +local directory. Model and snapshot identities do not select a mapping. + +Catalog references are review aids. Approval converts selected findings into +rule and evidence expectations. New runs resolve each expectation against their +own checker output to obtain actual finding IDs. Missing or non-unique matches +stop preparation or reporting with the finding, page, rule, and evidence to +investigate. An empty expectation list records an approved absence of coverage. + +Each run copies the complete approved mapping to +`artifacts/human-finding-mapping.json` and records its checksum and approval +attribution. Full evaluations save the same mapping in the parent and every +child. Reporting uses those saved copies and rejects changes to fixed decisions. +Later edits to the original shared file do not rewrite historical runs. + +To change approved decisions or checker rules, prepare and approve a new review +and create new runs. An unchanged audit checksum does not guarantee that new +page HTML or changed checker evidence will satisfy old expectations. + +## Model settings and execution + +Each run accepts one of these exact IDs. Friendly names and aliases are rejected. + +| Model | `--model` | Endpoint | Thinking or reasoning | +| --- | --- | --- | --- | +| Haiku 5.5, default | `claude-haiku-5-5` | Anthropic Messages | Adaptive thinking, medium effort | +| Opus 5.5 | `claude-opus-5-5` | Anthropic Messages | Adaptive thinking, medium effort | +| Sonnet 5.5 | `claude-sonnet-5-5` | Anthropic Messages | Adaptive thinking, medium effort | +| GPT-6 Luna | `gpt-6-luna` | OpenAI Chat Completions | Medium reasoning effort | +| GPT-6.1 Sol | `gpt-6.1-sol` | OpenAI Chat Completions | Medium reasoning effort | + +All presets omit temperature, disable summary prompts, and cap output at 24,192 +tokens including thinking or reasoning and final text. Adaptive thinking does +not reserve a fixed final-text allowance. Equal effort labels and caps do not +establish equal computation between providers. Runs save the chosen model, +provider, endpoint, and settings. + +The [root quick start](../../README.md#evaluator-quick-start) shows `run-all` +preview and live commands. Preparation creates every child page before showing +the full scope and asking for one live approval. Preview shows ordered pages, +prompt counts, settings, saved token prices, and the shared allowance. + +Live Claude execution requires `ANTHROPIC_API_KEY`; live GPT execution requires +`OPENAI_API_KEY`, in the environment or repository-root `.env`. A missing key +stops live execution. Type exactly `yes` in an interactive terminal or pass +`--approve-live` for noninteractive approval. That flag requires `--live`. +Declined approval leaves prepared files and makes no model requests. + +Every invocation creates a fresh directory. A preview remains incomplete and +cannot produce a final report. Adding `--live` to a later invocation creates +another run; it does not execute the previous preview directory. + +### Page preparation and execution + +Preparation copies each snapshot, imports local findings and all Globals, runs +the programmatic checks, and saves extractor payloads. The pipeline filters +payloads using programmatic results before slicing them into prompts. For +example, it can remove empty headings or skip a missing-title prompt. Empty +slices can also skip requests. The model sees extracted and filtered data; +inspect saved prompts to determine whether relevant evidence reached it. + +Requests execute sequentially, in listed page order for full evaluations, +without retries or fallback models. Each request's responses and usage are saved +as execution progresses. A request failure stops later requests and pages. +Unreadable successful responses are recorded separately and execution continues. +Thus `complete: true` means execution completed, not that every response parsed +or every human finding was detected. Check `format_failures` in each `run.json`. + +A full evaluation saves ordered page membership, status, stopping reason, cost, +and usage in `full.json`. Preparation failures retain earlier and partial +evidence. Execution failures retain completed and partial page evidence, +including reported usage from failed requests. A completed child can still +produce an individual report if a later page fails. + +There is no resume or paid-output reuse command. After failure, a fresh live +invocation repeats all model requests, including pages completed previously. +Captured input HTML can be reused without downloading. + +### Cost accounting + +`--max-cost-usd` must be positive, including in previews. It is a limit checked +between requests, not a hard billing cap. A request underway can exceed it. +Execution completes after an overshoot if no work remains; otherwise it stops +before starting further requests. Full evaluation subtracts prior pages' saved +estimated costs and passes the remaining allowance to the next page. + +Cost uses provider-reported usage and a saved, versioned public price schedule. +Each request selects its own price tier. GPT models use the higher tier above +272,000 total input tokens; Haiku 5.5 uses it above 100,000. Equality uses the +lower tier. Input counts from separate requests are not combined for tier +selection. + +OpenAI cache reads and writes replace ordinary input charges. Completion usage +already includes reasoning. Anthropic cache categories are separate input +counts; the saved write rate is for five-minute caching. Raw responses retain +unchanged provider usage alongside disjoint billable categories and request +costs. Reports show saved estimated costs rather than repricing past runs. + +## Evaluate one page + +Use the same audit, approved mapping, and saved snapshot as the full workflow. +Find the desired URL's HTML filename in `snapshots.json`; `page-001.html` below +assumes the homepage is the first listed URL. + +```bash +uv run visionaid-evaluate run \ + --human-audit "Pristine Accessibility Defect Report.xlsx" \ + --html .model-evaluation/pristine-snapshots/page-001.html \ + --source-url https://pristineai.com/ \ + --human-finding-mapping .model-evaluation/pristine-mapping-review/human-finding-mapping.json \ + --model claude-haiku-5-5 --max-cost-usd 1.00 +``` + +This previews one page. Repeat with `--live` to start a fresh paid run. For a +human audit file with a `URL list`, the source URL selects its local findings and all +Globals. Unlisted URLs are rejected. Mapping authoring still reviews the +human audit file's full set of listed pages even if only one page will be executed. + +After execution completes, review the printed run directory's `review.csv`, then: + +```bash +uv run visionaid-evaluate report \ + --run-dir '.model-evaluation/runs/' \ + --reviewer "Your name and review method" +``` + +The same `report` command accepts a completed child directory under +`.model-evaluation/full-runs//pages/`, including after a +later page fails. Open that directory's `report.html`. + +## Review LLM matches + +After execution, compare `review.csv` with `normalized-llm-findings.json` and use +`snapshot.html` to verify the location and defect. Programmatic evidence is in +`artifacts/normalized-programmatic-findings.json`. Every applicable human finding +has one CSV row; review Globals separately on every page. + +Edit only these columns: + +| Column | What to enter | +| --- | --- | +| `llm_finding_id` | Finding IDs that jointly cover the whole human finding, separated by semicolons. Leave blank for a miss. | +| `match_reason` | Explain how the selected LLM findings cover the reported defect. Required when selecting LLM IDs. | +| `review_notes` | Optional qualifications, location details, or partial evidence. Notes alone award no credit. | + +Keep `classification`, `classification_reason`, `programmatic_finding_id`, and +`programmatic_match_reason` unchanged. Those fields come from the approved +mapping. Keep all original evidence, IDs, headers, and rows. `source_sheet` and +`source_row` point back to the human audit file; `source_element` copies its element +name, rather than an inferred page location. `page_scope` can be `home`, `page`, +or `global`. + +### Complete-match rules + +- Match the same underlying accessibility problem in the same element or + relevant section. Different wording, certainty, examples, or recommended fixes + can still describe a valid detection. +- Treat examples as illustrations unless they clearly describe distinct defects. + A different remediation recommendation does not invalidate detection. +- Conditional or tentative findings can count when they identify the reported + defect and the human audit supplies supporting context. Confirm the location + in the saved HTML and record qualifications in notes. +- Several findings from one source on one page can jointly cover a human + finding. Reject matches that only share a WCAG criterion, identify another + element, or leave a distinct part of the defect undetected. +- Each ID must exist in the correct source's findings file and can be assigned + to only one human finding within that page review. Do not repeat an ID within + a row. +- LLM and programmatic sources can each fully catch the same finding. Partial + coverage from different sources does not combine into a complete catch. +- Unavailable and ambiguous findings cannot claim either source's matches. + +A blank LLM ID is scored as a miss. Validation cannot distinguish a deliberate +miss from an unreviewed row, so inspect every row before reporting. It validates +structural rules and saved evidence; semantic coverage remains the reviewer's +judgment. + +### Using an agent for review + +Give an agent the specific live run directory and this prompt: + +```text +Review LLM matches in this run, fill its LLM decision columns in review.csv, and generate the report with attribution describing your review. For a full evaluation, review every page before using report-all. Keep approved classification and Programmatic fields unchanged. Use practical semantic matching. +Count a match when a finding identifies the same underlying accessibility problem in the same element or relevant section, even if its wording, certainty, examples, or recommended fix differ from the human audit. +Treat examples in a human finding as illustrations unless they clearly describe separate defects. A different remediation recommendation does not invalidate detection. +Conditional or tentative findings can count when they identify the reported defect and the human audit supplies supporting context. Use the saved HTML to confirm the location. +Allow multiple findings from one source on one page to cover a human finding together. Reject matches that merely share a WCAG criterion, concern another element, or leave a distinct part undetected. +Inspect every row. Leave the LLM ID blank for a miss and record qualifications or partial evidence in review_notes. Apply full coverage consistently. +``` + +## How scoring works + +Rates measure coverage of the supplied human audit. They do not measure +precision, false-positive rate, remediation quality, or every accessibility +defect on the site. Unmatched tool findings do not independently lower scores. +One homepage result cannot establish broad model equivalence. + +| Rate | Caught count | Denominator | +| --- | --- | --- | +| LLM | LLM catches among `llm_eligible` findings. | All `llm_eligible` findings. | +| Programmatic | Programmatic catches among `programmatic` findings. | All `programmatic` findings. | +| Overall | Human findings fully caught by either source, counting each once. | All imported human findings, including unavailable and ambiguous findings. | + +A cross-catch helps Overall but earns no credit in the expected source's rate. +For example, an LLM catch of a Programmatic-classified defect does not increase +the LLM rate. Findings with unavailable evidence or ambiguous descriptions stay +outside both source denominators and remain in Overall. A zero denominator +produces an undefined rate with an explanation and no bar. + +For ten findings with six LLM eligible, two Programmatic, and two not testable, +complete detection of all eight testable findings gives LLM 6/6, Programmatic +2/2, and Overall 8/10. Both source rates are 100%; Overall is 80%. + +The current mapping workflow requires complete checker matches for every +Programmatic-classified finding on every applicable page. Reporting rejects +missing expected checker evidence. Consequently, the Programmatic rate in a +successfully validated report is 100% whenever that category is non-empty. +It records approved checker coverage; it does not independently measure checker +recall against a set of potentially missed Programmatic findings. + +### Combined results and Globals + +Combined results count each original finding once. A Global earns source credit +when that source completely catches it on any single page. Multiple selected +findings on that page may jointly cover it. Partial evidence spread across +pages cannot earn credit. LLM and programmatic complete catches may come from +different pages, but Overall still counts the Global once. Repeated catches on +several pages also count once. + +The approved mapping fixes each Global's classification across pages. Each +original enters its source denominator once, including a miss by that source. +Combined rates use pooled original-finding counts, not averages of page rates. +Page charts repeat Globals and have different eligible counts; do not add page +counts or average percentages to reconstruct combined scores. Page comparisons +are not controlled rankings of page or model quality. + +## How responses become findings + +The evaluator uses va-dat's JSON parser, including its code-fence and repair +steps. A response that cannot be parsed, or whose shape the check cannot read, +is Unreadable. Examples include one object where a list is required, or text +where `issues`, `vague_headings`, or `header_clarity_issues` must be a list. +An Unreadable response contributes no findings, is listed under `format_failures` +in `run.json`, and does not stop subsequent checks. + +### Checks covered by the tool's CSV report + +For the 12 checks with CSV normalizers, each row those rules would produce +becomes one LLM finding: + +| Checks | Finding rule | +| --- | --- | +| Page title, landmarks | One finding per `issues` entry. Answers such as `is_descriptive` do not create findings. | +| Headings | One finding per `issues` entry and per `vague_headings` entry. | +| Link clarity, iframe titles, label quality, decorative verification | One finding per item whose `is_clear`, `is_descriptive`, or `likely_decorative` verdict is `false`. A missing verdict counts as a pass. | +| Required field indicators, informative alt quality, actionable image alt, SVGs, icon fonts | One finding per item with a non-empty `issues` list, joining its issues. A false yes/no answer alone does not create a finding. | + +Problem, element, and location text use what va-dat would write in its CSV row. +`raw_source` preserves the full item, or for page title, headings, and landmarks +the whole response plus its specific issue. + +Evaluation skips the tool's later false-positive filter and LLM deduplication. +The filter guesses from page HTML and can remove a finding other than the one +it matched. Deduplication costs extra model calls and is not repeatable. +Evaluation output therefore differs from the downloadable tool CSV. The +earlier payload filtering described under [execution](#page-preparation-and-execution) +still applies. + +### Checks shown on screen only + +Six checks have no CSV report normalizer: `table_semantics`, +`placeholder_as_label`, `group_labels`, `form_instructions`, +`complex_descriptions`, and `media_captions`. The evaluator still reads them. +It produces at most one finding per item with non-empty `issues`, or for tables, +`issues` or `header_clarity_issues`. Every `placeholder_as_label` item is a +finding because its input consists of placeholder-only fields. Yes/no answers +alone never create findings. Tables may return one object or a list. + +These findings carry `"screen_only": true`. Matched evidence in HTML reports +labels them as shown on screen only and absent from the tool's CSV report. + +## Reports and saved files + +`report-all` requires every listed page's execution and review to validate. +It checks ordered membership, evidence ownership, saved mappings, approved +decisions, and actual expected checker findings before producing combined +results. It writes the parent's `report.html` and every child's `report.html`. +Both reporting commands require reviewer attribution and make no model requests. + +Reports begin with cost, recorded usage, saved public price information, and +detection bars. Each bar has textual caught counts, eligible counts, and a +percentage. Full reports also compare page LLM rates in audit order. The human +findings table follows the human audit file's original order. + +Open **View details** for the original human evidence, then **View page decisions** +for classifications, recorded explanations, mapping approval attribution, +matched findings, check identities, LLM rationale, and notes. Globals in full +reports include a table of page decisions. Native disclosures work with a +keyboard and with JavaScript disabled. + +With JavaScript enabled, filter by detection outcome and, in full reports, page. +Caught filters include cross-catches; source-missed filters include only that +source's eligible findings. Selecting a page shows its outcomes beside the +combined results without changing detection rates. **Clear filters** restores +all rows. **Expand all dropdowns** and **Collapse all dropdowns** affect visible +rows and page panels. There is no built-in text search. + +Reports embed styling, controls, original human evidence, and selected matching +evidence for offline use. A copied HTML file needs no neighboring files or +internet. Keep the run directory for snapshots, prompts, raw responses, and +unmatched tool findings; these are not all embedded in the report. Unreadable +response counts remain in each child's `run.json`, rather than appearing as a +combined HTML summary. + +### Printing and regeneration + +**Print report** prints the header, cost, and detection results through the last +bar chart. The findings table and detailed evidence are omitted. The browser's +Print command uses the same summary layout, including without JavaScript. +Printing works offline and preserves filters and opened disclosures without +changing scores or reviews. Browser PDF output is not verified as an accessible +PDF. + +Regenerating from saved reviewed evidence makes no model calls. An individual +report is removed before its review is validated; the combined report is also +removed before full evidence validation. Invalid revised reviews therefore +cannot leave stale final results for that report. Preserve a separate copy +first if needed. Full validation failures leave existing child reports available. +Each new HTML file is published after rendering succeeds. Successful +regeneration removes superseded generated Markdown for those reports, leaving +unrelated historical runs alone. + +### Page run files + +Single-page runs live in `.model-evaluation/runs//`. Full evaluation +children use the same layout under +`.model-evaluation/full-runs//pages//`. + +| Path relative to the page run | Contents | +| --- | --- | +| `README.md` | Review instructions and report command. | +| `review.csv` | Original human evidence, approved decisions, and editable LLM review columns. | +| `snapshot.html` | Copied page HTML used for the evaluation. | +| `run.json` | Configuration, identities, prompts, skipped prompts, filters, completion, stopping reason, format failures, usage, and estimated cost. | +| `normalized-llm-findings.json` | LLM findings and their run-specific IDs, saved during execution. | +| `report.html` | Generated offline page report. | +| `artifacts/human-finding-mapping.json` | Frozen approved mapping, with checksum and attribution in the manifest. | +| `artifacts/human-findings.json` | Canonical imported local and Global human findings. | +| `artifacts/normalized-programmatic-findings.json` | Checker findings with run-specific IDs. | +| `artifacts/raw-programmatic-findings.json` | Original checker output. | +| `artifacts/raw-llm-responses.json` | Model replies, request outcomes, raw usage, billable categories, and request cost estimates. | +| `artifacts/pricing.json` | Saved model rates, version, and capture date. | +| `artifacts/prompts/*.json` | Exact `prompt_text` and filtered `payload_slice` for each prepared request. These contain no model replies. | +| `artifacts/payloads/cl01_payload.json` | Extracted semantic structure before filtering. | +| `artifacts/payloads/cl02_payload.json` | Extracted forms, fields, and labels before filtering. | +| `artifacts/payloads/cl03_payload.json` | Extracted images, SVGs, icons, and media before filtering. | + +Previews have no LLM response or normalized LLM findings files. Prompt files +show what is prepared for the model; raw response records show which requests +actually ran. Check prompts when relevant evidence seems missing and raw +responses when an answer is absent or malformed. The pipeline's temporary +`artifacts/manifest.json` is removed; evaluation metadata lives in `run.json`. + +A full parent also contains `README.md`, `full.json`, `human-findings.json`, and +`artifacts/human-finding-mapping.json`. `full.json` records ordered child +membership and execution totals. Parent human findings retain unique originals +and page assignments. Reporting adds the combined `report.html`. + +## Troubleshooting and compatibility + +| Symptom | Action | +| --- | --- | +| Approved mapping required | Complete mapping CSV preparation and approval, then pass its JSON path explicitly. Previews also require it. | +| Audit checksum or page assignment mismatch | Use the same human audit file and its matching snapshot mapping. For a changed human audit file, capture and approve new inputs. Check local findings' reproduction URLs. | +| Expected checker evidence resolves to zero or several findings | Inspect the named page, rule, and evidence. Reuse the reviewed snapshots and checker code, or prepare and approve a new mapping for changed evidence. | +| Capture or review directory already exists | Choose a new directory. These commands preserve previous work. | +| Approval output already exists | Choose a new output filename. Approval refuses to overwrite mappings. | +| Live key missing | Set the selected provider's key in the environment or repository-root `.env`. | +| Live execution declined in a script | Use `--live --approve-live` for explicit noninteractive approval. | +| Incomplete execution or exhausted allowance | Read the stopping reason in `run.json` or `full.json`. Address it and start a fresh live invocation, which repeats paid requests. Completed children can still be reported individually. | +| Complete run with missing LLM findings | Check `format_failures`, raw replies, skipped prompts, and prepared payload slices. Unreadable responses yield no findings but do not stop execution. | +| Approved column changed | Restore the saved decision. To change responsibilities, approve a new mapping and create new runs. | +| Unknown or reused finding ID | Copy IDs from the correct page and source file. Use each ID for at most one human finding in that page's review. | +| Report disappears after failed regeneration | Correct the reported validation error and rerun reporting. The prior report is removed to prevent stale final results. | + +Saved runs must have the current directory layout, mapping artifact, and manifest +fields. There is no automatic migration or compatibility fallback. Moving files +into `artifacts/` alone cannot update every older run format. Prepare new runs +when old evidence lacks required fields; retain earlier files as historical +artifacts. The former `correct-all` command is removed. + +Earlier private experiments may contain `preflight.json`, +`format-diagnostics.json`, or `review-validation.json`. Standard evaluator +commands do not create those files or perform those extra experimental checks. diff --git a/vision_aid/evaluation/__init__.py b/vision_aid/evaluation/__init__.py new file mode 100644 index 0000000..c734d16 --- /dev/null +++ b/vision_aid/evaluation/__init__.py @@ -0,0 +1 @@ +"""Private model-evaluation workflows.""" diff --git a/vision_aid/evaluation/cli.py b/vision_aid/evaluation/cli.py new file mode 100644 index 0000000..179e165 --- /dev/null +++ b/vision_aid/evaluation/cli.py @@ -0,0 +1,372 @@ +"""Prepare and execute fixed-model evaluations, then score human decisions.""" + +from __future__ import annotations + +import argparse +import json +import os +import sys +import uuid +from collections.abc import Sequence +from datetime import UTC, datetime +from decimal import Decimal +from pathlib import Path +from typing import Any + +from dotenv import load_dotenv + +from vision_aid.evaluation.execution.llm_audit import ( + DEFAULT_MODEL, + MODEL_PRESETS, + LLMRequestConfig, + evaluation_configuration, + execute, + positive_cost, +) +from vision_aid.evaluation.execution.pricing import PriceSchedule +from vision_aid.evaluation.full import ( + capture_snapshots, + execute_full_evaluation, + load_full_inputs, + prepare_full_evaluation, + report_full_evaluation, +) +from vision_aid.evaluation.human_finding_mapping import select_human_finding_mapping +from vision_aid.evaluation.human_finding_mapping_review import ( + approve_human_finding_mapping_review, + prepare_human_finding_mapping_review, +) +from vision_aid.evaluation.inputs.human_audit import import_page_human_findings +from vision_aid.evaluation.page import prepare_page, report_page + +PROJECT_ROOT = Path(__file__).resolve().parents[2] +DEFAULT_HUMAN_AUDIT = PROJECT_ROOT / "Pristine Accessibility Defect Report.xlsx" + + +def build_parser() -> argparse.ArgumentParser: + """Expose the supported evaluation operations.""" + parser = argparse.ArgumentParser(prog="visionaid-evaluate") + commands = parser.add_subparsers(dest="command", required=True) + mapping_prepare = commands.add_parser( + "human-finding-mapping-prepare", + help="prepare Human finding eligibility and Programmatic match review CSVs", + ) + mapping_prepare.add_argument("--human-audit", type=Path, default=DEFAULT_HUMAN_AUDIT) + mapping_prepare.add_argument("--snapshots", type=Path, required=True) + mapping_prepare.add_argument("--output-dir", type=Path, required=True) + mapping_approve = commands.add_parser( + "human-finding-mapping-approve", + help="validate reviewed CSVs and export an approved Human finding mapping", + ) + mapping_approve.add_argument("--human-audit", type=Path, default=DEFAULT_HUMAN_AUDIT) + mapping_approve.add_argument("--snapshots", type=Path, required=True) + mapping_approve.add_argument("--review-dir", type=Path, required=True) + mapping_approve.add_argument("--reviewer", required=True) + mapping_approve.add_argument( + "--output", type=Path, + help="new JSON filename; defaults to review-dir/human-finding-mapping.json", + ) + capture = commands.add_parser( + "capture", help="download each Human audit URL to a saved Benchmark snapshot", + description=( + "Download listed URLs in source order, following redirects. " + "No model key or approval is required." + ), + ) + capture.add_argument("--human-audit", type=Path, default=DEFAULT_HUMAN_AUDIT) + capture.add_argument( + "--output-dir", type=Path, required=True, help="new snapshot directory" + ) + run = commands.add_parser( + "run", help="preview or execute an Evaluation run with one selected model", + description=( + "Haiku 5.5 (default), Opus 5.5 " + "and Sonnet 5.5 use adaptive thinking with medium effort. GPT models " + "use Chat Completions with medium reasoning effort. All presets omit " + "temperature, disable summaries, and cap output at 24,192 tokens " + "including reasoning/thinking. Live GPT execution requires OPENAI_API_KEY; " + "live Claude execution requires ANTHROPIC_API_KEY. Previews need no key." + ), + ) + run.add_argument( + "--model", choices=MODEL_PRESETS, default=DEFAULT_MODEL, + help=f"fixed model preset (default: {DEFAULT_MODEL})", + ) + run.add_argument( + "--human-audit", type=Path, default=DEFAULT_HUMAN_AUDIT, help="Human audit" + ) + run.add_argument( + "--human-finding-mapping", type=Path, + help="approved eligibility classifications and Programmatic matches; defaults to this audit's local Human finding mapping", + ) + run.add_argument("--html", type=Path, required=True, help="local Benchmark HTML") + run.add_argument("--source-url", required=True, help="URL represented by the HTML") + run.add_argument( + "--max-cost-usd", required=True, + help="positive between-request limit; a request underway can exceed it", + ) + run.add_argument("--live", action="store_true", help="enable billable execution") + run.add_argument( + "--approve-live", + action="store_true", + help="explicitly approve billable execution without a prompt", + ) + run_all = commands.add_parser( + "run-all", help="preview or execute every listed page with one shared allowance", + description=run.description, + ) + run_all.add_argument( + "--model", choices=MODEL_PRESETS, default=DEFAULT_MODEL, + help=f"fixed model preset (default: {DEFAULT_MODEL})", + ) + run_all.add_argument( + "--human-audit", type=Path, default=DEFAULT_HUMAN_AUDIT, help="Human audit" + ) + run_all.add_argument( + "--human-finding-mapping", type=Path, + help="approved eligibility classifications and Programmatic matches; defaults to this audit's local Human finding mapping", + ) + run_all.add_argument( + "--snapshots", type=Path, required=True, + help="saved snapshots.json mapping from capture; resolves HTML beside it", + ) + run_all.add_argument( + "--max-cost-usd", required=True, + help=( + "positive total allowance shared across pages; checked between requests; " + "a request underway can exceed it" + ), + ) + run_all.add_argument("--live", action="store_true", help="enable billable execution") + run_all.add_argument( + "--approve-live", action="store_true", + help="explicitly approve all displayed pages without a prompt; requires --live", + ) + report = commands.add_parser( + "report", help="score completed run and edited review.csv" + ) + report.add_argument("--run-dir", type=Path, required=True) + report.add_argument("--reviewer", required=True, help="who made the review decisions") + report_all = commands.add_parser( + "report-all", help="validate every page review and score unique Human findings", + description=( + "Final combined reporting requires every listed execution to complete " + "and all page reviews to validate. Each original Human finding counts " + "once. A Global finding earns source credit through complete detection " + "on any single page; testable Global classifications must agree. " + "Individual completed page reports remain available after parent failure." + ), + ) + report_all.add_argument("--run-dir", type=Path, required=True, help="Full evaluation directory") + report_all.add_argument("--reviewer", required=True, help="who made all page review decisions") + return parser + + +def _new_run_directory(kind: str = "runs") -> Path: + """Create a unique private directory under the containing worktree root.""" + current = Path.cwd() + worktree = next( + ( + parent + for parent in (current, *current.parents) + if (parent / ".git").exists() + ), + current, + ) + root = worktree / ".model-evaluation" / kind + root.mkdir(parents=True, exist_ok=True) + run_id = datetime.now(UTC).strftime("%Y%m%dT%H%M%S") + "-" + uuid.uuid4().hex[:8] + run = root / run_id + run.mkdir() + return run + + +def _required_human_finding_mapping(path: Path | None, audit_sha256: str) -> dict[str, Any]: + """Require approved decisions before preparing any new Evaluation run.""" + mapping = select_human_finding_mapping(path, audit_sha256) + if mapping is None: + raise ValueError( + "New Evaluation runs require an approved Human finding mapping. Use " + "human-finding-mapping-prepare and human-finding-mapping-approve to review " + "this audit, then supply --human-finding-mapping PATH." + ) + return mapping + + +def _run(args: argparse.Namespace) -> int: + """Prepare a fresh run and execute only after explicit approval.""" + request_config = MODEL_PRESETS[args.model] + configuration = evaluation_configuration(request_config) + limit = positive_cost(args.max_cost_usd) + if args.approve_live and not args.live: + raise ValueError("--approve-live requires --live") + if not args.html.is_file(): + raise ValueError(f"Benchmark HTML not found: {args.html}") + human_audit = import_page_human_findings(args.human_audit, args.source_url) + mapping = _required_human_finding_mapping(args.human_finding_mapping, human_audit.human_audit_sha256) + run = _new_run_directory() + manifest = prepare_page( + run, args.html, args.source_url, human_audit, request_config, limit, mapping + ) + snapshot_hash = manifest["snapshot_sha256"] + schedule = PriceSchedule.load(run / "artifacts" / "pricing.json") + print(f"Evaluation run: {run}") + _display_model( + request_config, configuration, schedule, len(manifest["prompts"]), limit + ) + print( + f"Snapshot SHA-256: {snapshot_hash}; " + f"Human findings: {len(human_audit.human_findings)}" + ) + print(f"Review CSV: {run / 'review.csv'}") + if not args.live: + print("Preview complete; no paid requests made.") + return 0 + api_key = _approved_api_key(args, configuration) + if api_key is None: + return 0 + complete = execute(run, manifest, api_key, request_config) + print(f"Run {'complete' if complete else 'incomplete'}: {run}") + return 0 if complete else 1 + + +def _approved_api_key(args: argparse.Namespace, configuration: dict[str, Any]) -> str | None: + """Select the provider's credentials and ask once for the displayed scope.""" + load_dotenv(PROJECT_ROOT / ".env") + key_name = ( + "OPENAI_API_KEY" if configuration["provider"] == "openai" else "ANTHROPIC_API_KEY" + ) + api_key = os.getenv(key_name) + if not api_key: + raise ValueError(f"Live execution requires {key_name}") + if not args.approve_live and ( + not sys.stdin.isatty() + or input("Type 'yes' to approve paid execution: ") != "yes" + ): + print("Live execution declined; no paid requests made.") + return None + return api_key + + +def _display_model( + request_config: LLMRequestConfig, + configuration: dict[str, Any], + schedule: PriceSchedule, + prompt_count: int, + limit: Decimal, +) -> None: + """Display fixed request settings, recorded prices and the shared guardrail.""" + print( + f"Model: {request_config.model}; provider: {configuration['provider']}; " + f"endpoint: {configuration['endpoint']}; " + f"thinking: {configuration.get('thinking', {}).get('type', 'none')}; " + f"effort: {request_config.reasoning_effort or 'omitted'}; " + f"thinking budget: {request_config.thinking_budget_tokens or 'none'}; " + f"output cap: {request_config.max_output_tokens}" + ) + print( + "Temperature: omitted; summaries: disabled; " + f"prompts: {prompt_count}" + ) + print(f"Pricing: {schedule.version} ({schedule.identity})") + rates = schedule.data["models"][request_config.model] + tiers = [("", rates)] + if "long_context" in rates: + upper = rates["long_context"] + threshold = upper["above_input_tokens"] + tiers = [ + (f"up to {threshold:,} total input tokens per request", rates), + (f"above {threshold:,} total input tokens per request", upper), + ] + cache_write = ( + "5-minute cache write" if configuration["provider"] == "anthropic" else "cache write" + ) + for description, tier in tiers: + print( + f"Rates per million tokens (USD){f' ({description})' if description else ''}: " + f"input ${tier['input_per_million_usd']}; " + f"output ${tier['output_per_million_usd']}; " + f"cache read ${tier['cached_input_per_million_usd']}; " + f"{cache_write} ${tier['cache_creation_input_per_million_usd']}" + ) + print( + f"Cost guardrail: ${limit}; checked before each request; " + "a request underway can exceed the limit" + ) + + +def _run_all(args: argparse.Namespace) -> int: + """Prepare all listed pages and approve the entire scope once.""" + request_config = MODEL_PRESETS[args.model] + limit = positive_cost(args.max_cost_usd) + if args.approve_live and not args.live: + raise ValueError("--approve-live requires --live") + audits, snapshots = load_full_inputs(args.human_audit, args.snapshots) + mapping = _required_human_finding_mapping(args.human_finding_mapping, next(iter(audits.values())).human_audit_sha256) + run = _new_run_directory("full-runs") + print(f"Full evaluation: {run}") + manifest = prepare_full_evaluation(run, audits, snapshots, request_config, limit, mapping) + prompt_count = 0 + for page in manifest["pages"]: + child = run / page["run_dir"] + child_manifest = json.loads((child / "run.json").read_text()) + prompt_count += len(child_manifest["prompts"]) + print(f"Page: {page['source_url']}; Evaluation run: {child}") + print(f"Review CSV: {child / 'review.csv'}") + first_child = run / manifest["pages"][0]["run_dir"] + schedule = PriceSchedule.load(first_child / "artifacts/pricing.json") + _display_model( + request_config, manifest["configuration"], schedule, prompt_count, limit + ) + if not args.live: + print("Preview complete; no paid requests made.") + return 0 + api_key = _approved_api_key(args, manifest["configuration"]) + if api_key is None: + return 0 + complete = execute_full_evaluation(run, manifest, api_key, request_config) + print(f"Full evaluation {'complete' if complete else 'incomplete'}: {run}") + return 0 if complete else 1 + + +def _report(args: argparse.Namespace) -> int: + """Validate human decisions and report a completed run.""" + run = args.run_dir.resolve() + report_page(run, args.reviewer) + print(f"HTML report: {run / 'report.html'}") + return 0 + + +def main(argv: Sequence[str] | None = None) -> int: + """Dispatch the requested CLI operation.""" + parser = build_parser() + args = parser.parse_args(argv) + try: + if args.command == "human-finding-mapping-prepare": + prepare_human_finding_mapping_review(args.human_audit, args.snapshots, args.output_dir) + print(f"Human finding mapping review CSVs and instructions: {args.output_dir.resolve()}") + return 0 + if args.command == "human-finding-mapping-approve": + mapping = approve_human_finding_mapping_review( + args.review_dir, args.human_audit, args.snapshots, args.reviewer, args.output, + ) + print(f"Approved Human finding mapping: {mapping.resolve()}") + return 0 + if args.command == "capture": + mapping = capture_snapshots(args.human_audit, args.output_dir) + print(f"Snapshot mapping: {mapping}") + return 0 + if args.command == "run-all": + return _run_all(args) + if args.command == "report-all": + run = args.run_dir.resolve() + report_full_evaluation(run, args.reviewer) + print(f"Combined HTML report: {run / 'report.html'}") + return 0 + return _run(args) if args.command == "run" else _report(args) + except (OSError, ValueError, KeyError) as error: + parser.error(str(error)) + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/vision_aid/evaluation/execution/__init__.py b/vision_aid/evaluation/execution/__init__.py new file mode 100644 index 0000000..c6c6131 --- /dev/null +++ b/vision_aid/evaluation/execution/__init__.py @@ -0,0 +1 @@ +"""Execute LLM requests, normalize findings, and estimate run cost.""" diff --git a/vision_aid/evaluation/execution/llm_audit.py b/vision_aid/evaluation/execution/llm_audit.py new file mode 100644 index 0000000..6751cdc --- /dev/null +++ b/vision_aid/evaluation/execution/llm_audit.py @@ -0,0 +1,180 @@ +"""Fixed evaluation model presets and sequential paid execution.""" + +from __future__ import annotations + +import json +from dataclasses import asdict +from decimal import Decimal +from pathlib import Path +from typing import Any + +from processing_scripts.llm_client.audit import ( + AuditRequestClient as LLMRequestClient, +) +from processing_scripts.llm_client.audit import ( + AuditRequestConfig as LLMRequestConfig, +) +from vision_aid.evaluation.execution.normalization import normalize_prompt_response +from vision_aid.evaluation.execution.pricing import PriceSchedule + +DEFAULT_MODEL = "claude-haiku-5-5" +OPENAI_MODELS = ("gpt-6-luna", "gpt-6.1-sol") +MODEL_PRESETS = { + **{ + model: LLMRequestConfig( + model=model, + adaptive_thinking=True, + reasoning_effort="medium", + max_output_tokens=24192, + temperature=None, + ) + for model in (DEFAULT_MODEL, "claude-opus-5-5", "claude-sonnet-5-5") + }, + **{ + model: LLMRequestConfig( + model=model, + reasoning_effort="medium", + max_output_tokens=24192, + temperature=None, + ) + for model in OPENAI_MODELS + }, +} +USAGE_KEYS = ( + "input_tokens", + "cached_input_tokens", + "cache_creation_input_tokens", + "output_tokens", +) + + +def evaluation_configuration(config: LLMRequestConfig) -> dict[str, Any]: + """Record the selected preset and evaluation-only request settings.""" + provider = "openai" if config.model in OPENAI_MODELS else "anthropic" + metadata = config.as_metadata() + return { + "model": config.model, + "provider": provider, + "endpoint": "chat.completions" if provider == "openai" else "messages", + **({"thinking": metadata["thinking"]} if "thinking" in metadata else {}), + "thinking_budget_tokens": config.thinking_budget_tokens, + "reasoning_effort": config.reasoning_effort, + "max_output_tokens": config.max_output_tokens, + "temperature": "omitted", + "summaries": "disabled", + } + + +def write_json(path: Path, value: object) -> None: + """Save one readable private evidence artifact.""" + path.write_text( + json.dumps(value, indent=2, ensure_ascii=False, sort_keys=True) + "\n", + encoding="utf-8", + ) + + +def positive_cost(value: str) -> Decimal: + """Parse a strictly positive cost guardrail.""" + try: + amount = Decimal(value) + except Exception as error: + raise ValueError("Cost guardrail must be a positive decimal") from error + if not amount.is_finite() or amount <= 0: + raise ValueError("Cost guardrail must be a positive decimal") + return amount + + +def billable_usage(model: str, reported: dict[str, Any]) -> dict[str, int]: + """Separate OpenAI cached subsets without changing reported evidence.""" + usage = {key: int(reported.get(key) or 0) for key in USAGE_KEYS} + if model in OPENAI_MODELS: + raw = reported.get("provider_usage") or {} + details = raw.get("prompt_tokens_details") or {} + usage["cached_input_tokens"] = int(details.get("cached_tokens") or 0) + usage["cache_creation_input_tokens"] = int(details.get("cache_write_tokens") or 0) + usage["input_tokens"] -= ( + usage["cached_input_tokens"] + usage["cache_creation_input_tokens"] + ) + return usage + + +def execute( + run: Path, manifest: dict[str, Any], api_key: str, + request_config: LLMRequestConfig, +) -> bool: + """Make each paid request once, saving reported usage and responses as received.""" + artifacts = run / "artifacts" + schedule = PriceSchedule.load(artifacts / "pricing.json") + client = LLMRequestClient( + api_key=api_key, + request_config=request_config, + max_retries=0, + ) + usage: dict[str, int] = {key: 0 for key in USAGE_KEYS} + estimated_cost = Decimal(0) + responses: list[dict[str, Any]] = [] + findings: list[dict[str, Any]] = [] + format_failures: list[str] = [] + complete = True + reason = None + for prompt in manifest["prompts"]: + if estimated_cost >= Decimal(manifest["max_cost_usd"]): + complete, reason = False, "cost guardrail exhausted" + break + name = prompt["name"] + prompt_text = json.loads((artifacts / "prompts" / f"{name}.json").read_text())[ + "prompt_text" + ] + result: dict[str, Any] + try: + result = client.call(prompt_text) + except Exception as error: + # Retain an unexpected request failure as run evidence. + result = { + "success": False, + "response": None, + "usage": {}, + "error": str(error), + } + reported: dict[str, Any] = result.get("usage") or {} + for key in USAGE_KEYS: + usage[key] += int(reported.get(key) or 0) + billable = billable_usage(request_config.model, reported) + request_cost = schedule.estimate(request_config.model, billable) + estimated_cost += Decimal(request_cost) + responses.append({ + "prompt": name, "result": result, "billable_usage": billable, + "estimated_cost_usd": request_cost, + }) + if result.get("success"): + normalized = normalize_prompt_response( + name, + str(result.get("response") or ""), + run_id=manifest["run_id"], + model=request_config.model, + page_url=manifest["source_url"], + ) + findings.extend(asdict(finding) for finding in normalized.findings) + if normalized.parse_status == "malformed": + format_failures.append(name) + else: + complete, reason = False, str(result.get("error") or "request failed") + manifest.update( + usage=usage, + estimated_cost_usd=format(estimated_cost.normalize(), "f"), + complete=False, + incomplete_reason=reason, + format_failures=format_failures, + ) + write_json(artifacts / "raw-llm-responses.json", responses) + write_json(run / "normalized-llm-findings.json", findings) + write_json(run / "run.json", manifest) + if not complete: + break + manifest.update( + complete=complete, + incomplete_reason=reason, + estimated_cost_usd=format(estimated_cost.normalize(), "f"), + ) + write_json(run / "run.json", manifest) + return complete diff --git a/vision_aid/evaluation/execution/normalization.py b/vision_aid/evaluation/execution/normalization.py new file mode 100644 index 0000000..a0365be --- /dev/null +++ b/vision_aid/evaluation/execution/normalization.py @@ -0,0 +1,278 @@ +"""Lossless deterministic normalization of raw LLM outcomes.""" + +from __future__ import annotations + +import hashlib +import json +import re +from collections.abc import Callable, Iterable, Sequence +from dataclasses import dataclass +from typing import Any, NamedTuple + +from entry_points.generate_report import NORMALIZERS, ReportRow, safe_parse_json +from processing_scripts.llm.registry import PROMPT_REGISTRY, PromptSpec +from vision_aid.evaluation.schemas import CanonicalFinding + +_PROMPTS = {item.name: item for item in PROMPT_REGISTRY if not item.is_summary} +_PROBLEM_KEYS = ( + "problem", + "issue", + "issue_title", + "description", + "finding", + "reason", + "actual_result", + "summary", +) +_LOCATION_KEYS = ( + "location", + "location_hint", + "selector", + "xpath", + "text", + "link_text", + "src", +) +_ELEMENT_KEYS = ("element", "element_name", "tag", "html", "selector") +_WCAG_KEYS = ("wcag", "wcag_sc", "wcag_criteria", "criterion", "criteria") +# Checks without a tool normalizer only reach users on screen; these fields name their element. +_SCREEN_ONLY_ELEMENT_KEYS = ("field_id", "src", "legend", "placeholder") + + +@dataclass(frozen=True) +class NormalizationResult: + """Findings plus the parse outcome for one successful raw response.""" + + prompt: str + parse_status: str + raw_response: str + findings: tuple[CanonicalFinding, ...] + error: str | None = None + + +def _first_text(item: dict[str, Any], keys: Iterable[str]) -> str: + """Return the first populated evidence field as stable text.""" + for key in keys: + value = item.get(key) + if value is None or value == "": + continue + if isinstance(value, (dict, list)): + return json.dumps(value, ensure_ascii=False, sort_keys=True) + return str(value) + return "" + + +def _wcag(item: dict[str, Any], defaults: Sequence[str]) -> tuple[str, ...]: + """Extract ordered WCAG identifiers or use the prompt defaults.""" + values: list[str] = [] + for key in _WCAG_KEYS: + value = item.get(key) + if isinstance(value, list): + values.extend(str(part) for part in value) + elif value: + values.append(str(value)) + criteria = re.findall(r"\b\d+\.\d+\.\d+\b", " ".join(values)) + return tuple(dict.fromkeys(criteria or defaults)) + + +class _Source(NamedTuple): + """One LLM finding's text, with the response part it came from.""" + + position: int + source: Any + problem: str + element: str + location: str + + +def _issue_list(item: dict[str, Any], key: str) -> list[Any]: + """Copy a listed-problems field, rejecting text that would split per character.""" + value = item.get(key) + if value is None: + return [] + if not isinstance(value, list): + raise TypeError(f"Expected {key!r} to be a JSON array.") + return list(value) + + +def _item_list(parsed: Any) -> list[dict[str, Any]]: + """Return an array response's items, rejecting a shape no rule can read.""" + if not isinstance(parsed, list) or not all(isinstance(item, dict) for item in parsed): + raise ValueError("Expected a JSON array response.") + for item in parsed: + _issue_list(item, "issues") + return parsed + + +def _tool_findings( + normalizer: Callable[..., list[ReportRow]], parsed: Any, spec: PromptSpec +) -> list[_Source]: + """Read findings with the tool's report rule, pairing each row with its source.""" + wcag = ", ".join(spec.wcag_criteria) + if spec.output_type == "object": + if not isinstance(parsed, dict): + raise ValueError("Expected a JSON object response.") + # Page-level rules write one row per issue, then one per vague heading. + issues = _issue_list(parsed, "issues") + _issue_list(parsed, "vague_headings") + rows = normalizer(parsed, wcag=wcag) + sources = [{"response": parsed, "issue": issue} for issue in issues] + paired = list(enumerate(zip(sources, rows))) + else: + # One item at a time, so each row keeps the item it came from. + paired = [ + (index, (item, row)) + for index, item in enumerate(_item_list(parsed)) + for row in normalizer([item], wcag=wcag) + ] + return [ + _Source( + position, + source, + row.actual_result or row.issue_title, + row.element_name, + row.steps_to_reproduce, + ) + for position, (source, row) in paired + ] + + +def _screen_only_findings(prompt_name: str, parsed: Any) -> list[_Source]: + """Read a check the tool's report skips: a finding needs a listed problem.""" + if prompt_name == "table_semantics" and isinstance(parsed, dict): + # The table prompt asks for one object rather than a list. + parsed = [parsed] + findings = [] + for index, item in enumerate(_item_list(parsed)): + issues = _issue_list(item, "issues") + if prompt_name == "table_semantics": + issues += _issue_list(item, "header_clarity_issues") + # Placeholder-only fields are the problem; that check lists no issues. + if not issues and prompt_name != "placeholder_as_label": + continue + problem = "; ".join(str(issue) for issue in issues) + findings.append(_Source( + index, + item, + problem or str(item.get("reason") or "") or prompt_name, + _first_text(item, _SCREEN_ONLY_ELEMENT_KEYS), + str(item.get("location_hint") or ""), + )) + return findings + + +def normalize_prompt_response( + prompt_name: str, + raw_response: str, + *, + run_id: str, + model: str, + page_url: str, +) -> NormalizationResult: + """Read one response with the tool's rules, without its filter or deduplication.""" + spec = _PROMPTS.get(prompt_name) + if spec is None: + return NormalizationResult( + prompt=prompt_name, + parse_status="unknown_prompt", + raw_response=raw_response, + findings=(), + error=(f"No fixed evaluation normalizer exists for {prompt_name!r}."), + ) + try: + # Parse exactly as the API's report does, so only responses it skips are unreadable. + parsed = safe_parse_json(raw_response) + except (json.JSONDecodeError, ValueError) as error: + return NormalizationResult( + prompt=prompt_name, + parse_status="malformed", + raw_response=raw_response, + findings=(), + error=str(error), + ) + normalizer = NORMALIZERS.get(prompt_name) + try: + if normalizer is None: + sources = _screen_only_findings(prompt_name, parsed) + else: + sources = _tool_findings(normalizer, parsed, spec) + except Exception as error: + # A shape the check's rule cannot read is unreadable, not fatal to the run. + return NormalizationResult( + prompt=prompt_name, + parse_status="malformed", + raw_response=raw_response, + findings=(), + error=str(error) or type(error).__name__, + ) + findings = [] + for position, source, problem, element, location in sources: + canonical_source = json.dumps( + source, + ensure_ascii=False, + sort_keys=True, + separators=(",", ":"), + ) + identity = f"{run_id}\0{prompt_name}\0{position}\0{canonical_source}".encode() + findings.append( + CanonicalFinding( + finding_id="llm-" + hashlib.sha256(identity).hexdigest()[:20], + source="llm", + run_id=run_id, + model=model, + prompt=prompt_name, + checklist=spec.checklist, + page_url=page_url, + problem_family=prompt_name, + problem=problem, + element=element, + location=location, + wcag_evidence=tuple(spec.wcag_criteria), + raw_source=source, + parse_status="parsed", + screen_only=normalizer is None, + ) + ) + return NormalizationResult( + prompt=prompt_name, + parse_status="parsed" if findings else "empty", + raw_response=raw_response, + findings=tuple(findings), + ) + + +def normalize_programmatic_findings( + raw_findings: Sequence[dict[str, Any]], + *, + run_id: str, + page_url: str, +) -> tuple[CanonicalFinding, ...]: + """Normalize checker output while retaining each complete source object.""" + normalized = [] + for index, item in enumerate(raw_findings): + canonical_item = json.dumps( + item, + ensure_ascii=False, + sort_keys=True, + separators=(",", ":"), + ) + identity = f"{run_id}\0programmatic\0{index}\0{canonical_item}".encode() + rule = str(item.get("rule_id") or item.get("rule") or "programmatic") + normalized.append( + CanonicalFinding( + finding_id="programmatic-" + hashlib.sha256(identity).hexdigest()[:20], + source="programmatic", + run_id=run_id, + model=None, + prompt=rule, + checklist=str(item.get("checklist") or "programmatic"), + page_url=page_url, + problem_family=rule, + problem=_first_text(item, _PROBLEM_KEYS + ("message",)) or rule, + element=_first_text(item, _ELEMENT_KEYS), + location=_first_text(item, _LOCATION_KEYS), + wcag_evidence=_wcag(item, ()), + raw_source=item, + parse_status="parsed", + ) + ) + return tuple(normalized) diff --git a/vision_aid/evaluation/execution/pricing.json b/vision_aid/evaluation/execution/pricing.json new file mode 100644 index 0000000..170f8c4 --- /dev/null +++ b/vision_aid/evaluation/execution/pricing.json @@ -0,0 +1,61 @@ +{ + "models": { + "claude-opus-5-5": { + "input_per_million_usd": "4", + "output_per_million_usd": "20", + "cached_input_per_million_usd": "0.20", + "cache_creation_input_per_million_usd": "5" + }, + "claude-sonnet-5-5": { + "input_per_million_usd": "2", + "output_per_million_usd": "10", + "cached_input_per_million_usd": "0.20", + "cache_creation_input_per_million_usd": "2.50" + }, + "gpt-6-luna": { + "input_per_million_usd": "0.10", + "output_per_million_usd": "0.50", + "cached_input_per_million_usd": "0.01", + "cache_creation_input_per_million_usd": "0.125", + "long_context": { + "above_input_tokens": 272000, + "input_per_million_usd": "0.20", + "output_per_million_usd": "0.75", + "cached_input_per_million_usd": "0.02", + "cache_creation_input_per_million_usd": "0.25" + } + }, + "gpt-6.1-sol": { + "input_per_million_usd": "2.00", + "output_per_million_usd": "10.00", + "cached_input_per_million_usd": "0.10", + "cache_creation_input_per_million_usd": "2.50", + "long_context": { + "above_input_tokens": 272000, + "input_per_million_usd": "4.00", + "output_per_million_usd": "15.00", + "cached_input_per_million_usd": "0.20", + "cache_creation_input_per_million_usd": "5.00" + } + }, + "claude-haiku-5-5": { + "input_per_million_usd": "0.10", + "output_per_million_usd": "0.50", + "cached_input_per_million_usd": "0.01", + "cache_creation_input_per_million_usd": "0.125", + "long_context": { + "above_input_tokens": 100000, + "input_per_million_usd": "0.50", + "output_per_million_usd": "2.50", + "cached_input_per_million_usd": "0.05", + "cache_creation_input_per_million_usd": "0.625" + } + } + }, + "captured_at": "2026-10-09", + "sources": [ + "https://developers.openai.com/api/docs/pricing", + "https://platform.claude.com/docs/en/about-claude/pricing" + ], + "version": "public-standard-2026-10-09" +} diff --git a/vision_aid/evaluation/execution/pricing.py b/vision_aid/evaluation/execution/pricing.py new file mode 100644 index 0000000..d4850c4 --- /dev/null +++ b/vision_aid/evaluation/execution/pricing.py @@ -0,0 +1,69 @@ +"""Reproducible cost estimates from a versioned public price schedule.""" + +from __future__ import annotations + +import hashlib +import json +from collections.abc import Mapping +from dataclasses import dataclass +from decimal import Decimal +from pathlib import Path +from typing import Any + +DEFAULT_SCHEDULE = Path(__file__).with_name("pricing.json") +MILLION = Decimal(1_000_000) + + +@dataclass(frozen=True) +class PriceSchedule: + """Immutable model rates and their content identity.""" + + data: dict[str, Any] + identity: str + + @classmethod + def load(cls, path: Path) -> PriceSchedule: + """Load and content-address one JSON price schedule.""" + raw = path.read_bytes() + data = json.loads(raw) + if not isinstance(data, dict) or not isinstance(data.get("models"), dict): + raise TypeError("Price schedule must contain a models object.") + return cls(data=data, identity=hashlib.sha256(raw).hexdigest()) + + @property + def version(self) -> str: + """Return the curator-assigned price-schedule version.""" + return str(self.data["version"]) + + @property + def price_description(self) -> str: + """Distinguish captured public rates from older publication metadata.""" + if "captured_at" in self.data: + return f"published prices captured on {self.data['captured_at']}" + return f"published prices dated {self.data['published_at']}" + + def estimate(self, model: str, usage: Mapping[str, int | None]) -> str: + """Price one request's disjoint categories at its input-size tier.""" + try: + rates = self.data["models"][model] + except KeyError as error: + raise ValueError( + f"No versioned pricing exists for model {model!r}." + ) from error + input_tokens = Decimal(usage.get("input_tokens", 0) or 0) + cached_tokens = Decimal(usage.get("cached_input_tokens", 0) or 0) + cache_creation_tokens = Decimal( + usage.get("cache_creation_input_tokens", 0) or 0 + ) + output_tokens = Decimal(usage.get("output_tokens", 0) or 0) + upper = rates.get("long_context") + if upper and input_tokens + cached_tokens + cache_creation_tokens > upper["above_input_tokens"]: + rates = upper + cost = ( + input_tokens * Decimal(rates["input_per_million_usd"]) + + cached_tokens * Decimal(rates["cached_input_per_million_usd"]) + + cache_creation_tokens + * Decimal(rates["cache_creation_input_per_million_usd"]) + + output_tokens * Decimal(rates["output_per_million_usd"]) + ) / MILLION + return format(cost.normalize(), "f") diff --git a/vision_aid/evaluation/full.py b/vision_aid/evaluation/full.py new file mode 100644 index 0000000..0b4830b --- /dev/null +++ b/vision_aid/evaluation/full.py @@ -0,0 +1,398 @@ +"""Coordinate Full evaluation preparation and saved Benchmark snapshots.""" + +from __future__ import annotations + +import hashlib +import json +import shlex +import uuid +from decimal import Decimal +from pathlib import Path +from typing import Any + +import requests + +from vision_aid.evaluation.execution.llm_audit import ( + USAGE_KEYS, + LLMRequestConfig, + evaluation_configuration, + execute, + write_json, +) +from vision_aid.evaluation.human_finding_mapping import ( + load_human_finding_mapping_snapshot, + save_human_finding_mapping_snapshot, + validate_human_finding_mapping, +) +from vision_aid.evaluation.inputs.human_audit import ( + import_all_human_findings, + list_audit_urls, +) +from vision_aid.evaluation.page import PageReview, prepare_page, validated_page_review +from vision_aid.evaluation.reporting.html_report import ( + write_full_report, + write_page_report, +) +from vision_aid.evaluation.schemas import HumanAudit + + +def capture_snapshots(human_audit: Path, output_dir: Path) -> Path: + """Download only listed pages and publish their mapping after all succeed.""" + urls = list_audit_urls(human_audit) + audit_checksum = hashlib.sha256(human_audit.read_bytes()).hexdigest() + output_dir.mkdir(parents=True) + snapshots = [] + for number, source_url in enumerate(urls, start=1): + response = requests.get(source_url, allow_redirects=True) + response.raise_for_status() + html = f"page-{number:03d}.html" + (output_dir / html).write_bytes(response.content) + snapshots.append({"source_url": source_url, "html": html}) + mapping = output_dir / "snapshots.json" + write_json(mapping, { + "human_audit_filename": human_audit.name, + "human_audit_sha256": audit_checksum, + "snapshots": snapshots, + }) + return mapping + + +def load_full_inputs( + human_audit: Path, mapping_path: Path, +) -> tuple[dict[str, HumanAudit], dict[str, Path]]: + """Load listed page assignments and saved snapshots without downloading.""" + audits = import_all_human_findings(human_audit) + mapping = json.loads(mapping_path.read_text(encoding="utf-8")) + checksum = next(iter(audits.values())).human_audit_sha256 + if not isinstance(mapping, dict) or mapping.get("human_audit_sha256") != checksum: + raise ValueError( + "Snapshot mapping Human audit identity differs from the supplied audit" + ) + entries = mapping.get("snapshots") + if not isinstance(entries, list) or any( + not isinstance(entry, dict) + or not isinstance(entry.get("source_url"), str) + or not isinstance(entry.get("html"), str) + for entry in entries + ): + raise ValueError( + "Snapshot mapping must contain source URLs and relative HTML filenames" + ) + if [entry["source_url"] for entry in entries] != list(audits): + raise ValueError( + "Snapshot mapping must match every listed target once in source order" + ) + snapshots = {} + directory = mapping_path.parent.resolve() + for entry in entries: + filename = Path(entry["html"]) + html = (directory / filename).resolve() + if filename.is_absolute() or not html.is_relative_to(directory): + raise ValueError( + "Snapshot mapping HTML files must be relative files beside the mapping" + ) + if not html.is_file(): + raise ValueError(f"Snapshot mapping Benchmark HTML not found: {filename}") + snapshots[entry["source_url"]] = html + return audits, snapshots + + +def prepare_full_evaluation( + run: Path, + audits: dict[str, HumanAudit], + snapshots: dict[str, Path], + request_config: LLMRequestConfig, + limit: Decimal, + mapping: dict[str, Any], +) -> dict[str, Any]: + """Prepare all listed pages, retaining membership and progress on failure.""" + first_audit = next(iter(audits.values())) + pages = [] + originals = {} + assignments = {} + for number, (source_url, audit) in enumerate(audits.items(), start=1): + child_id = f"{run.name}-page-{number:03d}-{uuid.uuid4().hex[:8]}" + human_ids = [] + for finding in audit.to_dict()["human_findings"]: + identity = finding["human_finding_id"] + originals[identity] = finding + human_ids.append(identity) + if finding["page_scope"] != "global": + assignments[identity] = source_url + pages.append({ + "source_url": source_url, + "run_id": child_id, + "run_dir": f"pages/{child_id}", + "human_finding_ids": human_ids, + "status": "unprepared", + "complete": False, + }) + manifest = { + "run_id": run.name, + "human_audit_filename": first_audit.human_audit_filename, + "human_audit_sha256": first_audit.human_audit_sha256, + "configuration": evaluation_configuration(request_config), + "max_cost_usd": str(limit), + "estimated_cost_usd": "0", + "usage": {key: 0 for key in USAGE_KEYS}, + "complete": False, + "incomplete_reason": "preparing pages", + "pages": pages, + } + write_json(run / "human-findings.json", { + "human_audit_filename": first_audit.human_audit_filename, + "human_audit_sha256": first_audit.human_audit_sha256, + "human_findings": sorted(originals.values(), key=lambda item: item["source_row"]), + "page_assignments": assignments, + "source_urls": list(audits), + }) + write_json(run / "full.json", manifest) + (run / "README.md").write_text( + "# Review this Full evaluation\n\n" + "Check `full.json` for page directories, completion, and any stopping reason. " + "Add `--live` to `run-all` to call the LLMs. Without `--live`, the command " + "runs in preview mode, generating artifacts without calling the LLMs.\n\n" + "1. After every page completes, open each directory under `pages/`. Follow its " + "README to review LLM matches and save `review.csv`. Review Global findings on " + "every page. Keep approved classifications and Programmatic fields unchanged.\n" + "2. Generate all page reports and the combined report from the repository root:\n\n" + "```bash\n" + f"uv run visionaid-evaluate report-all --run-dir {shlex.quote(str(run.resolve()))} " + '--reviewer "Your name and review method"\n' + "```\n\n" + "Open this directory's `report.html` for combined results. Each page also has " + "its own `report.html`. Incomplete runs cannot produce a combined report.\n", + encoding="utf-8", + ) + save_full_human_finding_mapping(run, mapping) + manifest = json.loads((run / "full.json").read_text()) + pages = manifest["pages"] + for page in pages: + child = run / page["run_dir"] + try: + child.mkdir(parents=True) + child_manifest = prepare_page( + child, snapshots[page["source_url"]], page["source_url"], + audits[page["source_url"]], request_config, limit, mapping, + ) + except (OSError, ValueError, KeyError) as error: + page["status"] = "preparation_failed" + manifest["incomplete_reason"] = ( + f"page preparation failed for {page['source_url']}: {error}" + ) + write_json(run / "full.json", manifest) + raise + page.update( + status="prepared", snapshot_sha256=child_manifest["snapshot_sha256"] + ) + write_json(run / "full.json", manifest) + manifest["incomplete_reason"] = "preview" + write_json(run / "full.json", manifest) + return manifest + + +def save_full_human_finding_mapping(run: Path, mapping: dict[str, Any]) -> None: + """Pin the approved source shared by all children during preparation.""" + manifest = json.loads((run / "full.json").read_text()) + validate_human_finding_mapping(mapping, manifest["human_audit_sha256"]) + original = json.loads((run / "human-findings.json").read_text()) + if {item["human_finding_id"] for item in mapping["decisions"]} != { + item["human_finding_id"] for item in original["human_findings"] + }: + raise ValueError("Full evaluation Human finding mapping must contain each original Human finding exactly once") + save_human_finding_mapping_snapshot(run, manifest, mapping) + write_json(run / "full.json", manifest) + + +def execute_full_evaluation( + run: Path, manifest: dict[str, Any], api_key: str, + request_config: LLMRequestConfig, +) -> bool: + """Execute prepared pages sequentially with one remaining cost allowance.""" + total_cost = Decimal(0) + usage = {key: 0 for key in USAGE_KEYS} + limit = Decimal(manifest["max_cost_usd"]) + manifest["incomplete_reason"] = None + for number, page in enumerate(manifest["pages"]): + remaining = limit - total_cost + if remaining <= 0: + manifest["incomplete_reason"] = "cost guardrail exhausted before next page" + for unfinished in manifest["pages"][number:]: + unfinished["status"] = "not_executed" + write_json(run / "full.json", manifest) + return False + child = run / page["run_dir"] + child_manifest = json.loads((child / "run.json").read_text()) + child_manifest["max_cost_usd"] = str(remaining) + write_json(child / "run.json", child_manifest) + try: + complete = execute(child, child_manifest, api_key, request_config) + except Exception as error: + # Provider setup can fail before the executor saves a request outcome. + child_manifest.update(complete=False, incomplete_reason=str(error)) + write_json(child / "run.json", child_manifest) + complete = False + total_cost += Decimal(child_manifest["estimated_cost_usd"]) + for key in USAGE_KEYS: + usage[key] += child_manifest["usage"][key] + page.update(status="complete" if complete else "incomplete", complete=complete) + manifest.update(estimated_cost_usd=str(total_cost), usage=usage) + if not complete: + manifest["incomplete_reason"] = ( + f"page execution failed for {page['source_url']}: " + f"{child_manifest['incomplete_reason']}" + ) + for unfinished in manifest["pages"][number + 1:]: + unfinished["status"] = "not_executed" + write_json(run / "full.json", manifest) + if not complete: + return False + manifest.update(complete=True, incomplete_reason=None) + write_json(run / "full.json", manifest) + return True + + +def _pooled_rows( + originals: list[dict[str, Any]], + pages: list[dict[str, Any]], + reviews: list[PageReview], +) -> list[dict[str, Any]]: + """Keep each original once, using complete source matches on any one page.""" + grouped: dict[str, list[dict[str, Any]]] = {} + for page, review in zip(pages, reviews, strict=True): + for row in review.rows: + grouped.setdefault(row["human_finding"]["human_finding_id"], []).append({ + "source_url": page["source_url"], + "decision": row["decision"], + "matches": row["matches"], + }) + pooled = [] + for finding in originals: + page_rows = grouped[finding["human_finding_id"]] + classifications = { + row["decision"]["classification"].strip() for row in page_rows + if row["decision"]["classification"].strip() in {"llm_eligible", "programmatic"} + } + if len(classifications) > 1: + raise ValueError( + f"Global Human finding {finding['human_finding_id']} has conflicting " + "testable classifications; all testable pages must agree" + ) + classification = ( + next(iter(classifications)) if classifications + else page_rows[0]["decision"]["classification"].strip() + ) + pooled.append({ + "human_finding": finding, + "decision": {"classification": classification}, + "matches": { + kind: tuple(identity for row in page_rows for identity in row["matches"][kind]) + for kind in ("llm", "programmatic") + }, + "page_reviews": page_rows, + }) + return pooled + + +def _validated_full_reviews( + run: Path, manifest: dict[str, Any], original: dict[str, Any], +) -> list[PageReview]: + """Check ordered membership, original evidence and ownership for every child.""" + if manifest["run_id"] != run.name: + raise ValueError("Full evaluation evidence does not belong to this directory") + if not manifest["complete"]: + raise ValueError("Incomplete Full evaluations cannot produce a final report") + urls = original["source_urls"] + if not urls or len(urls) != len(set(urls)) or [ + page["source_url"] for page in manifest["pages"] + ] != urls: + raise ValueError("Full evaluation membership differs from the original ordered targets") + if original["human_audit_sha256"] != manifest["human_audit_sha256"]: + raise ValueError("Human audit identity differs from the Full evaluation") + findings = original["human_findings"] + identities = [item["human_finding_id"] for item in findings] + if not identities or len(identities) != len(set(identities)): + raise ValueError("Full evaluation must preserve each original Human finding once") + assignments = original["page_assignments"] + locals_ = {item["human_finding_id"] for item in findings if item["page_scope"] != "global"} + if set(assignments) != locals_ or any(url not in urls for url in assignments.values()): + raise ValueError("Full evaluation has invalid original page assignments") + mapping = load_human_finding_mapping_snapshot(run, manifest, "Saved Full evaluation Human finding mapping") + mapping_digest = manifest["human_finding_mapping_sha256"] + if {item["human_finding_id"] for item in mapping["decisions"]} != set(identities): + raise ValueError("Full evaluation Human finding mapping differs from original Human findings") + reviews = [] + child_ids: set[str] = set() + tool_ids: set[str] = set() + for page in manifest["pages"]: + child = (run / page["run_dir"]).resolve() + if ( + page["run_id"] in child_ids + or page["run_dir"] != f"pages/{page['run_id']}" + or child.parent != run / "pages" + or child.name != page["run_id"] + ): + raise ValueError("Child Evaluation run identity or directory ownership differs") + child_ids.add(page["run_id"]) + if not page["complete"] or page["status"] != "complete": + raise ValueError("Every listed page must complete execution before combined reporting") + expected = [ + item for item in findings if item["page_scope"] == "global" + or assignments[item["human_finding_id"]] == page["source_url"] + ] + if page["human_finding_ids"] != [item["human_finding_id"] for item in expected]: + raise ValueError("Child Human finding membership differs from its original assignments") + evidence = json.loads((child / "artifacts/human-findings.json").read_text()) + if evidence["human_findings"] != expected or evidence["homepage_url"] != page["source_url"]: + raise ValueError("Child Human finding evidence differs from the original page selection") + review = validated_page_review(child) + saved = review.manifest + if saved.get("human_finding_mapping_sha256") != mapping_digest: + raise ValueError(f"Child Human finding mapping differs from the Full evaluation: {page['source_url']}") + for field, expected_value in ( + ("run_id", page["run_id"]), ("source_url", page["source_url"]), + ("human_audit_sha256", manifest["human_audit_sha256"]), + ("configuration", manifest["configuration"]), + ("snapshot_sha256", page["snapshot_sha256"]), + ): + if saved[field] != expected_value: + raise ValueError(f"Child {field} differs from the Full evaluation") + if hashlib.sha256((child / "snapshot.html").read_bytes()).hexdigest() != saved["snapshot_sha256"]: + raise ValueError("Child Benchmark snapshot checksum differs from the saved evidence") + if review.schedule.identity != saved["pricing_sha256"] or review.schedule.version != saved["pricing_version"]: + raise ValueError("Child pricing evidence differs from its recorded schedule") + for filename in ("raw-llm-responses.json", "raw-programmatic-findings.json"): + json.loads((child / "artifacts" / filename).read_text()) + for prompt in saved["prompts"]: + json.loads((child / "artifacts/prompts" / f"{prompt['name']}.json").read_text()) + if any(item["model"] != manifest["configuration"]["model"] for item in review.llm): + raise ValueError("LLM findings do not belong to the selected model") + ids = [item["finding_id"] for item in review.llm + review.programmatic] + if len(ids) != len(set(ids)) or tool_ids.intersection(ids): + raise ValueError("Tool finding identities must be unique across all page runs") + tool_ids.update(ids) + reviews.append(review) + cost = sum((Decimal(review.manifest["estimated_cost_usd"]) for review in reviews), Decimal(0)) + usage = {key: sum(review.manifest["usage"][key] for review in reviews) for key in USAGE_KEYS} + if cost != Decimal(manifest["estimated_cost_usd"]) or usage != manifest["usage"]: + raise ValueError("Full evaluation totals differ from authoritative child cost and usage") + return reviews + + +def report_full_evaluation(run: Path, reviewer: str) -> None: + """Validate every page review before writing page and combined reports.""" + if not reviewer.strip(): + raise ValueError("A reviewer description is required") + run = run.resolve() + # Regeneration cannot leave a previous final report beside invalid decisions. + (run / "report.html").unlink(missing_ok=True) + (run / "report.md").unlink(missing_ok=True) + manifest = json.loads((run / "full.json").read_text()) + original = json.loads((run / "human-findings.json").read_text()) + reviews = _validated_full_reviews(run, manifest, original) + rows = _pooled_rows(original["human_findings"], manifest["pages"], reviews) + for page, review in zip(manifest["pages"], reviews, strict=True): + child = run / page["run_dir"] + write_page_report(child / "report.html", review, reviewer) + (child / "report.md").unlink(missing_ok=True) + write_full_report(run / "report.html", manifest, rows, reviews, reviewer) diff --git a/vision_aid/evaluation/human_finding_mapping.py b/vision_aid/evaluation/human_finding_mapping.py new file mode 100644 index 0000000..fb1d956 --- /dev/null +++ b/vision_aid/evaluation/human_finding_mapping.py @@ -0,0 +1,218 @@ +"""Approved audit decisions, resolved against each page's actual checker evidence.""" + +from __future__ import annotations + +import csv +import hashlib +import json +from pathlib import Path +from typing import Any + +from vision_aid.evaluation.inputs.human_audit import ELIGIBILITY_VALUES + +HUMAN_FINDING_MAPPING_DIRECTORY = Path(__file__).with_name("human_finding_mappings") +HUMAN_FINDING_MAPPING_ARTIFACT = "artifacts/human-finding-mapping.json" +LOCKED_COLUMNS = ( + "classification", "classification_reason", "programmatic_finding_id", + "programmatic_match_reason", +) + + +def _text(value: Any) -> bool: + """Identify populated human-authored text without coercing invalid values.""" + return isinstance(value, str) and bool(value.strip()) + + +def validate_human_finding_mapping(mapping: dict[str, Any], audit_sha256: str) -> None: + """Reject unrelated audits and incomplete or conflicting approved decisions.""" + if not isinstance(mapping, dict) or mapping.get("schema_version") != 1: + raise ValueError("Human finding mapping must use schema_version 1") + if mapping.get("human_audit_sha256") != audit_sha256: + raise ValueError("Human finding mapping Human audit identity differs from the run") + if not _text(mapping.get("approved_by")): + raise ValueError("Human finding mapping needs approved_by attribution") + decisions = mapping.get("decisions") + if not isinstance(decisions, list) or not decisions: + raise ValueError("Human finding mapping needs approved decisions") + seen = set() + for decision in decisions: + if not isinstance(decision, dict): + raise ValueError("Human finding mapping decision must be an object") # CLI input validation. + identity = decision.get("human_finding_id") + if not _text(identity) or identity in seen: + raise ValueError(f"Human finding mapping has unknown or repeated Human finding: {identity}") + seen.add(identity) + prefix = f"Human finding mapping Human finding {identity}" + if decision.get("classification") not in ELIGIBILITY_VALUES: + raise ValueError(f"{prefix}: invalid classification") + if not _text(decision.get("classification_reason")): + raise ValueError(f"{prefix}: needs approved classification_reason") + pages = decision.get("programmatic_by_page") + if not isinstance(pages, dict) or not pages: + raise ValueError(f"{prefix}: needs explicit Programmatic decisions per page") + for url, page in pages.items(): + if not _text(url) or not isinstance(page, dict): + raise ValueError(f"{prefix}: invalid Programmatic page decision") + expectations = page.get("expectations") + if not isinstance(expectations, list) or not _text(page.get("reason")): + raise ValueError(f"{prefix}, {url}: needs expectations and approved reason") + classification = decision["classification"] + if classification == "programmatic" and not expectations: + raise ValueError(f"{prefix}, {url}: Programmatic responsibility requires expected coverage") + if classification in {"unavailable_evidence", "ambiguous"} and expectations: + raise ValueError(f"{prefix}, {url}: not testable; cannot claim Programmatic coverage") + for expectation in expectations: + if not isinstance(expectation, dict) or not _text(expectation.get("rule")): + raise ValueError(f"{prefix}, {url}: expectation needs an existing rule and evidence") + evidence = expectation.get("evidence") + if ( + not isinstance(evidence, dict) + or not evidence + or set(evidence) - {"element", "location", "problem", "raw_source"} + or not any(value for value in evidence.values()) + ): + raise ValueError(f"{prefix}, {url}: expectation needs relevant element/page evidence") + for field, value in evidence.items(): + if field == "raw_source": + if not isinstance(value, dict) or not value: + raise ValueError(f"{prefix}, {url}: raw_source evidence must be a nonempty object") + elif not isinstance(value, str): + raise ValueError(f"{prefix}, {url}: {field} evidence must be text") + + +def load_human_finding_mapping(path: Path, audit_sha256: str) -> dict[str, Any]: + """Read explicitly selected approved decisions for exactly this Human audit.""" + mapping = json.loads(path.read_text(encoding="utf-8")) + validate_human_finding_mapping(mapping, audit_sha256) + return mapping + + +def select_human_finding_mapping(path: Path | None, audit_sha256: str) -> dict[str, Any] | None: + """Select explicit decisions or a locally stored Human finding mapping for this audit checksum.""" + if path is None: + path = HUMAN_FINDING_MAPPING_DIRECTORY / f"{audit_sha256}.json" + if not path.is_file(): + return None + return load_human_finding_mapping(path, audit_sha256) + + +def _contains(actual: Any, expected: Any) -> bool: + """Match exact evidence values, allowing a selected subset of raw object fields.""" + if isinstance(expected, dict): + return isinstance(actual, dict) and all( + key in actual and _contains(actual[key], value) + for key, value in expected.items() + ) + return actual == expected + + +def resolved_decisions( + mapping: dict[str, Any], human_audit: dict[str, Any], source_url: str, + programmatic: list[dict[str, Any]], +) -> dict[str, dict[str, str]]: + """Resolve each approved complete match uniquely without run-dependent keys.""" + validate_human_finding_mapping(mapping, human_audit["human_audit_sha256"]) + decisions = {item["human_finding_id"]: item for item in mapping["decisions"]} + resolved = {} + used = set() + for human in human_audit["human_findings"]: + identity = human["human_finding_id"] + prefix = f"Human finding mapping Human finding {identity}, {source_url}" + if identity not in decisions: + raise ValueError(f"{prefix}: missing approved decision") + decision = decisions[identity] + if source_url not in decision["programmatic_by_page"]: + raise ValueError(f"{prefix}: missing explicit Programmatic page decision") + page = decision["programmatic_by_page"][source_url] + ids = [] + for expectation in page["expectations"]: + candidates = [ + item for item in programmatic + if item["problem_family"] == expectation["rule"] + and _contains(item, expectation["evidence"]) + ] + if len(candidates) != 1: + raise ValueError( + f"{prefix}: expected Programmatic rule {expectation['rule']} " + f"with evidence {json.dumps(expectation['evidence'], ensure_ascii=False, sort_keys=True)} " + f"resolved to {len(candidates)} findings; expected exactly one. " + "Inspect the saved checker evidence and approved expectation." + ) + finding_id = candidates[0]["finding_id"] + if finding_id in used: + raise ValueError(f"{prefix}: conflicting Programmatic finding ID reuse: {finding_id}") + used.add(finding_id) + ids.append(finding_id) + resolved[identity] = { + "classification": decision["classification"], + "classification_reason": decision["classification_reason"], + "programmatic_finding_id": ";".join(ids), + "programmatic_match_reason": page["reason"], + } + return resolved + + +def save_human_finding_mapping_snapshot( + run: Path, manifest: dict[str, Any], mapping: dict[str, Any], +) -> None: + """Save approved decisions and pin their bytes and attribution in a manifest.""" + path = run / HUMAN_FINDING_MAPPING_ARTIFACT + path.parent.mkdir(exist_ok=True) + path.write_text(json.dumps(mapping, ensure_ascii=False, indent=2) + "\n", encoding="utf-8") + manifest["human_finding_mapping_sha256"] = hashlib.sha256(path.read_bytes()).hexdigest() + manifest["human_finding_mapping_approved_by"] = mapping["approved_by"] + + +def load_human_finding_mapping_snapshot( + run: Path, manifest: dict[str, Any], label: str = "Saved Human finding mapping", +) -> dict[str, Any]: + """Load a pinned approved source only after verifying its bytes and attribution.""" + path = run / HUMAN_FINDING_MAPPING_ARTIFACT + digest = manifest.get("human_finding_mapping_sha256") + if not digest: + raise ValueError(f"{label} is missing its run identity; prepare a new run with --human-finding-mapping PATH") + if not path.is_file() or hashlib.sha256(path.read_bytes()).hexdigest() != digest: + raise ValueError(f"{label} is missing or its checksum changed") + mapping = load_human_finding_mapping(path, manifest["human_audit_sha256"]) + if manifest.get("human_finding_mapping_approved_by") != mapping["approved_by"]: + raise ValueError(f"{label} approval attribution changed") + return mapping + + +def apply_human_finding_mapping(run: Path, mapping: dict[str, Any]) -> None: + """Populate fixed decisions and save their source, preserving per-run LLM review.""" + manifest = json.loads((run / "run.json").read_text()) + human = json.loads((run / "artifacts/human-findings.json").read_text()) + programmatic = json.loads((run / "artifacts/normalized-programmatic-findings.json").read_text()) + if human["human_audit_sha256"] != manifest["human_audit_sha256"]: + raise ValueError("Human audit identity differs from the run") + decisions = resolved_decisions(mapping, human, manifest["source_url"], programmatic) + path = run / "review.csv" + with path.open(encoding="utf-8-sig", newline="") as stream: + reader = csv.DictReader(stream) + columns = list(reader.fieldnames or ()) + rows = list(reader) + if ( + len(rows) != len(decisions) + or {row.get("human_finding_id") for row in rows} != set(decisions) + ): + raise ValueError("Review CSV must contain each Human finding exactly once") + if "programmatic_match_reason" not in columns: + columns.append("programmatic_match_reason") + for row in rows: + row.update(decisions[row["human_finding_id"]]) + with path.open("w", encoding="utf-8", newline="") as stream: + writer = csv.DictWriter(stream, fieldnames=columns) + writer.writeheader() + writer.writerows(rows) + save_human_finding_mapping_snapshot(run, manifest, mapping) + (run / "run.json").write_text(json.dumps(manifest, ensure_ascii=False, indent=2) + "\n", encoding="utf-8") + + +def saved_human_finding_mapping_decisions( + run: Path, manifest: dict[str, Any], human: dict[str, Any], + programmatic: list[dict[str, Any]], +) -> dict[str, dict[str, str]]: + """Load the run's required frozen decisions and verify their saved identity.""" + mapping = load_human_finding_mapping_snapshot(run, manifest) + return resolved_decisions(mapping, human, manifest["source_url"], programmatic) diff --git a/vision_aid/evaluation/human_finding_mapping_review.py b/vision_aid/evaluation/human_finding_mapping_review.py new file mode 100644 index 0000000..5bc2f16 --- /dev/null +++ b/vision_aid/evaluation/human_finding_mapping_review.py @@ -0,0 +1,316 @@ +"""Prepare readable audit decisions and export an approved Human finding mapping.""" + +from __future__ import annotations + +import csv +import json +from dataclasses import asdict +from pathlib import Path +from typing import Any + +from processing_scripts.programmatic.forms_checklist_02 import audit_forms +from processing_scripts.programmatic.nontext_checklist_03 import audit_nontext +from processing_scripts.programmatic.semantic_checklist_01 import audit_html_file +from vision_aid.evaluation.execution.normalization import ( + normalize_programmatic_findings, +) +from vision_aid.evaluation.full import load_full_inputs +from vision_aid.evaluation.human_finding_mapping import ( + resolved_decisions, + validate_human_finding_mapping, +) +from vision_aid.evaluation.inputs.human_audit import ELIGIBILITY_VALUES, HEADERS + +IDENTITY_COLUMNS = ( + "human_finding_id", + "source_sheet", + "source_row", + "page_scope", + "source_element", + *HEADERS, +) +CLASSIFICATION_COLUMNS = (*IDENTITY_COLUMNS, "classification", "classification_reason") +MATCH_COLUMNS = ( + "source_url", + *IDENTITY_COLUMNS, + "programmatic_finding_refs", + "programmatic_reason", +) +FINDING_COLUMNS = ( + "finding_ref", + "source_url", + "rule", + "problem", + "element", + "text", + "target", + "location", +) + + +def _write_csv( + path: Path, columns: tuple[str, ...], rows: list[dict[str, Any]] +) -> None: + """Write a UTF-8 CSV that Excel can open with multiline evidence intact.""" + with path.open("w", encoding="utf-8-sig", newline="") as stream: + writer = csv.DictWriter(stream, fieldnames=columns, restval="") + writer.writeheader() + writer.writerows(rows) + + +def _human_columns(finding: dict[str, Any]) -> dict[str, Any]: + """Flatten original evidence beside generated identities for human review.""" + return { + key: finding[key] for key in IDENTITY_COLUMNS if key not in HEADERS + } | finding["raw_evidence"] + + +def _review_sources( + human_audit: Path, snapshots: Path, +) -> tuple[dict[str, Any], list[dict[str, Any]]]: + """Rebuild original findings and the checker catalog from the supplied saved inputs.""" + audits, html_files = load_full_inputs(human_audit, snapshots) + originals = {} + pages = [] + catalog = [] + for number, (url, audit) in enumerate(audits.items(), start=1): + saved = html_files[url] + raw = ( + audit_html_file(str(saved)) + + audit_forms(str(saved)) + + audit_nontext(str(saved)) + ) + findings = normalize_programmatic_findings( + raw, run_id="human-finding-mapping-review", page_url=url + ) + evidence = {} + for position, item in enumerate(findings, start=1): + ref = f"P{number:03d}-F{position:03d}" + finding = asdict(item) + evidence[ref] = finding + location = finding["raw_source"].get("location") or {} + attributes = location.get("attributes") or {} + catalog.append( + { + "finding_ref": ref, + "source_url": url, + "rule": finding["problem_family"], + "problem": finding["problem"], + "element": location.get("tag", finding["element"]), + "text": location.get("text_preview") + or attributes.get("aria-label") + or attributes.get("alt") + or attributes.get("title", ""), + "target": location.get("href") + or location.get("src") + or attributes.get("id", ""), + "location": location.get("css_path", finding["location"]), + } + ) + humans = list(audit.to_dict()["human_findings"]) + for finding in humans: + originals[finding["human_finding_id"]] = finding + pages.append( + { + "source_url": url, + "human_finding_ids": [ + finding["human_finding_id"] for finding in humans + ], + "findings": evidence, + } + ) + humans = sorted(originals.values(), key=lambda finding: finding["source_row"]) + evidence = { + "human_audit_sha256": next(iter(audits.values())).human_audit_sha256, + "human_findings": humans, + "pages": pages, + } + return evidence, catalog + + +def _match_rows(evidence: dict[str, Any]) -> list[dict[str, Any]]: + """List every original Human finding on each applicable page.""" + humans = {finding["human_finding_id"]: finding for finding in evidence["human_findings"]} + return [ + {"source_url": page["source_url"], **_human_columns(humans[identity])} + for page in evidence["pages"] + for identity in page["human_finding_ids"] + ] + + +def prepare_human_finding_mapping_review(human_audit: Path, snapshots: Path, output: Path) -> None: + """Prepare two editable CSVs and a checker catalog using the original saved inputs.""" + if output.exists(): + raise FileExistsError(f"Review directory already exists: {output}") + evidence, catalog = _review_sources(human_audit, snapshots) + output.mkdir(parents=True) + _write_csv( + output / "classifications.csv", CLASSIFICATION_COLUMNS, + [_human_columns(finding) for finding in evidence["human_findings"]], + ) + _write_csv(output / "programmatic-review.csv", MATCH_COLUMNS, _match_rows(evidence)) + _write_csv(output / "programmatic-findings.csv", FINDING_COLUMNS, catalog) + (output / "README.md").write_text( + "# Review the Human finding mapping\n\n" + "1. Open classifications.csv in Excel or another CSV editor. For every original Human finding, " + "fill classification and classification_reason. Classify each original once, including Globals. " + "Use programmatic only when existing checker findings fully cover the reported defect; " + "llm_eligible for semantic judgment or checks absent from existing rules; unavailable_evidence " + "when saved HTML cannot establish the defect; ambiguous when the Human finding is unclear.\n\n" + "2. Open programmatic-review.csv and programmatic-findings.csv side by side. " + "For each Human finding and page, enter complete matching finding_ref values in " + "programmatic_finding_refs, separated by semicolons, and explain them in programmatic_reason. " + "Multiple selected findings must jointly cover the full defect. A shared WCAG criterion alone " + "does not establish a match. Select only findings from that page, and use each reference once. " + "Leave references blank for no complete match and explain the absence. Programmatic " + "classifications require complete matches on every applicable page. Not-testable classifications " + "cannot claim matches. LLM classifications may have Programmatic cross-catches.\n\n" + "3. Save both edited files as CSV. Keep original evidence, IDs, headers, and every row unchanged. " + "Row order may change. Inspect the original saved snapshots when needed. The findings catalog is " + "read-only reference material; approval compares it with fresh checker results from those snapshots.\n\n" + "4. Run visionaid-evaluate human-finding-mapping-approve --human-audit ORIGINAL_AUDIT.xlsx " + "--snapshots ORIGINAL_SNAPSHOTS.json --review-dir THIS_DIRECTORY " + "--reviewer 'Your name'. Keep the original human audit Excel file " + "and captured snapshots available. " + "The CLI reruns the checks, validates the decisions, and writes human-finding-mapping.json. " + "Use that file with run or run-all --human-finding-mapping PATH. Approval records your decisions; it does " + "not automatically judge whether the selected evidence semantically covers a defect.\n", + encoding="utf-8", + ) + + +def _read_review( + path: Path, + columns: tuple[str, ...], + expected: list[dict[str, Any]], + keys: tuple[str, ...], + label: str = "Human finding", +) -> dict[tuple[str, ...], dict[str, str]]: + """Accept reordered rows while rejecting missing rows and altered original evidence.""" + with path.open(encoding="utf-8-sig", newline="") as stream: + reader = csv.DictReader(stream) + if len(reader.fieldnames or []) != len(columns) or set( + reader.fieldnames or [] + ) != set(columns): + raise ValueError(f"{path.name}: keep every original column exactly once") + entered = list(reader) + originals = {tuple(str(row[key]) for key in keys): row for row in expected} + if len(entered) != len(originals): + raise ValueError(f"{path.name}: keep every original review row exactly once") + result = {} + for line, row in enumerate(entered, start=2): + key = tuple(row[column] for column in keys) + if key not in originals or key in result: + raise ValueError( + f"{path.name} row {line}: unknown or repeated {label}/page {key}" + ) + if any( + row.get(column) != str(value) for column, value in originals[key].items() + ): + raise ValueError( + f"{path.name} row {line}: original {label} evidence changed for {key}" + ) + if None in row or any(value is None for value in row.values()): + raise ValueError( + f"{path.name} row {line}: invalid CSV row; preserve all columns" + ) + result[key] = row + return result + + +def approve_human_finding_mapping_review( + directory: Path, human_audit: Path, snapshots: Path, + reviewer: str, output: Path | None = None, +) -> Path: + """Validate human CSV decisions and generate portable approved evidence expectations.""" + if not reviewer.strip(): + raise ValueError("A Human finding mapping reviewer description is required") + evidence, catalog = _review_sources(human_audit, snapshots) + _read_review( + directory / "programmatic-findings.csv", FINDING_COLUMNS, catalog, + ("finding_ref",), label="Programmatic finding", + ) + humans = evidence["human_findings"] + by_id = {finding["human_finding_id"]: finding for finding in humans} + classifications = _read_review( + directory / "classifications.csv", + CLASSIFICATION_COLUMNS, + [_human_columns(finding) for finding in humans], + ("human_finding_id",), + ) + matches = _read_review( + directory / "programmatic-review.csv", + MATCH_COLUMNS, + _match_rows(evidence), + ("source_url", "human_finding_id"), + ) + decisions = {} + for identity in by_id: + row = classifications[(identity,)] + if ( + row["classification"].strip() not in ELIGIBILITY_VALUES + or not row["classification_reason"].strip() + ): + raise ValueError( + f"classifications.csv Human finding {identity}: fill a valid classification and classification_reason" + ) + decisions[identity] = { + "human_finding_id": identity, + "classification": row["classification"].strip(), + "classification_reason": row["classification_reason"].strip(), + "programmatic_by_page": {}, + } + for page in evidence["pages"]: + for identity in page["human_finding_ids"]: + row = matches[(page["source_url"], identity)] + prefix = f"programmatic-review.csv Human finding {identity} on {page['source_url']}" + reason = row["programmatic_reason"].strip() + if not reason: + raise ValueError( + f"{prefix}: explain the complete match or deliberate absence in programmatic_reason" + ) + expectations = [] + for ref in ( + part.strip() + for part in row["programmatic_finding_refs"].split(";") + if part.strip() + ): + if ref not in page["findings"]: + raise ValueError( + f"{prefix}: unknown or wrong-page Programmatic finding reference {ref}" + ) + finding = page["findings"][ref] + expectations.append( + { + "rule": finding["problem_family"], + "evidence": {"raw_source": finding["raw_source"]}, + } + ) + decisions[identity]["programmatic_by_page"][page["source_url"]] = { + "expectations": expectations, + "reason": reason, + } + mapping = { + "schema_version": 1, + "human_audit_sha256": evidence["human_audit_sha256"], + "approved_by": reviewer.strip(), + "decisions": list(decisions.values()), + } + validate_human_finding_mapping(mapping, evidence["human_audit_sha256"]) + for page in evidence["pages"]: + resolved_decisions( + mapping, + { + "human_audit_sha256": evidence["human_audit_sha256"], + "human_findings": [ + by_id[identity] for identity in page["human_finding_ids"] + ], + }, + page["source_url"], + list(page["findings"].values()), + ) + output = output or directory / "human-finding-mapping.json" + output.parent.mkdir(parents=True, exist_ok=True) + with output.open("x", encoding="utf-8") as stream: + stream.write(json.dumps(mapping, ensure_ascii=False, indent=2) + "\n") + return output diff --git a/vision_aid/evaluation/inputs/__init__.py b/vision_aid/evaluation/inputs/__init__.py new file mode 100644 index 0000000..57a4d3c --- /dev/null +++ b/vision_aid/evaluation/inputs/__init__.py @@ -0,0 +1 @@ +"""Import Human audit evidence for evaluation workflows.""" diff --git a/vision_aid/evaluation/inputs/human_audit.py b/vision_aid/evaluation/inputs/human_audit.py new file mode 100644 index 0000000..1ae550a --- /dev/null +++ b/vision_aid/evaluation/inputs/human_audit.py @@ -0,0 +1,167 @@ +"""Preserve Human findings and select them for one page or every listed page.""" + +from __future__ import annotations + +import hashlib +import re +from pathlib import Path +from typing import Literal +from urllib.parse import urlsplit + +import openpyxl + +from vision_aid.evaluation.schemas import HumanAudit, HumanFinding + +SHEET = "Defect report" +HOMEPAGE_SCOPES: set[Literal["home", "global"]] = {"home", "global"} +HEADERS = ( + "Sr. #", + "element name", + "Browser Combination", + "Page name", + "Issue Title", + "Steps to Reproduce", + "actual result", + "expected result", + "Recommendation for Fix", + "WCAG Sc", + "Type of Change", + "comment", +) +ELIGIBILITY_VALUES = ( + "llm_eligible", + "programmatic", + "unavailable_evidence", + "ambiguous", +) + + +def _source_findings( + path: Path, *, homepage_only: bool +) -> tuple[str, tuple[HumanFinding, ...]]: + """Read original Human finding evidence with stable source identities.""" + checksum = hashlib.sha256(path.read_bytes()).hexdigest() + human_audit_excel = openpyxl.load_workbook(path, read_only=True, data_only=True) + try: + sheet = human_audit_excel[SHEET] + rows = sheet.iter_rows(values_only=True) + headers = tuple(str(value or "") for value in next(rows)) + if headers[: len(HEADERS)] != HEADERS: + raise ValueError("Human audit has unexpected Defect report columns") + human_findings = [] + for number, values in enumerate(rows, start=2): + raw = { + header: "" if value is None else str(value) + for header, value in zip(headers, values, strict=False) + if header + } + scope = raw["Page name"].strip().casefold() + if homepage_only and scope not in HOMEPAGE_SCOPES: + continue + if not raw["Issue Title"].strip(): + raise ValueError(f"Defect report row {number} has no Issue Title") + identity = f"{checksum}:{SHEET}:{number}".encode() + human_findings.append( + HumanFinding( + human_finding_id="human-" + hashlib.sha256(identity).hexdigest()[:16], + source_sheet=SHEET, + source_row=number, + page_scope=scope if scope in HOMEPAGE_SCOPES else "page", + source_element=raw["element name"], + problem=raw["Issue Title"], + raw_evidence=raw, + wcag_evidence=tuple( + dict.fromkeys(re.findall(r"\b\d+\.\d+\.\d+\b", raw["WCAG Sc"])) + ), + ) + ) + finally: + human_audit_excel.close() + return checksum, tuple(human_findings) + + +def import_homepage_human_findings(path: Path, homepage_url: str) -> HumanAudit: + """Preserve the one-page command's Home and Global source selection.""" + checksum, human_findings = _source_findings(path, homepage_only=True) + if not human_findings: + raise ValueError("Human audit has no Home or Global defects") + return HumanAudit(path.name, checksum, homepage_url, human_findings) + + +def import_all_human_findings(path: Path) -> dict[str, HumanAudit]: + """Assign local findings by reproduction URL and repeat Globals per page.""" + urls = list_audit_urls(path) + if len(urls) != len(set(urls)): + raise ValueError("Human audit URL list contains repeated target URLs") + checksum, findings = _source_findings(path, homepage_only=False) + if not findings: + raise ValueError("Human audit has no Human findings") + by_page: dict[str, list[HumanFinding]] = {url: [] for url in urls} + for finding in findings: + if finding.page_scope == "global": + for page_findings in by_page.values(): + page_findings.append(finding) + continue + steps = finding.raw_evidence["Steps to Reproduce"] + reproduced = { + match.rstrip(".,;:)]}") + for match in re.findall(r'https?://[^\s<>"\']+', steps) + } + assigned = set() + for url in reproduced: + if url in by_page: + assigned.add(url) + continue + parsed = urlsplit(url) + # This is the audit's known alias, not general path case folding. + if parsed.path == "/Contact/": + url = parsed._replace(path="/contact/").geturl() + if url in by_page: + assigned.add(url) + if len(assigned) != 1: + raise ValueError( + f"Defect report row {finding.source_row} must map to exactly " + "one listed target URL in Steps to Reproduce" + ) + by_page[assigned.pop()].append(finding) + return { + url: HumanAudit(path.name, checksum, url, tuple(page_findings)) + for url, page_findings in by_page.items() + } + + +def list_audit_urls(path: Path) -> tuple[str, ...]: + """Read the Human audit's explicit target URLs in source order.""" + human_audit_excel = openpyxl.load_workbook(path, read_only=True, data_only=True) + try: + sheet = human_audit_excel["URL list"] + rows = sheet.iter_rows(values_only=True) + headers = tuple(str(value or "").strip() for value in next(rows)) + if "URL" not in headers: + raise ValueError("Human audit URL list has no URL column") + index = headers.index("URL") + urls = tuple( + str(row[index]).strip() + for row in rows + if len(row) > index and row[index] is not None and str(row[index]).strip() + ) + finally: + human_audit_excel.close() + if not urls: + raise ValueError("Human audit URL list has no target URLs") + return urls + + +def import_page_human_findings(path: Path, source_url: str) -> HumanAudit: + """Select a listed page's local findings and Globals, retaining older Home audits.""" + human_audit_excel = openpyxl.load_workbook(path, read_only=True, data_only=True) + try: + has_urls = "URL list" in human_audit_excel.sheetnames + finally: + human_audit_excel.close() + if not has_urls: + return import_homepage_human_findings(path, source_url) + pages = import_all_human_findings(path) + if source_url not in pages: + raise ValueError(f"Source URL is not listed in the Human audit: {source_url}") + return pages[source_url] diff --git a/vision_aid/evaluation/page.py b/vision_aid/evaluation/page.py new file mode 100644 index 0000000..d0c2b8e --- /dev/null +++ b/vision_aid/evaluation/page.py @@ -0,0 +1,181 @@ +"""Shared preparation and human-reviewed evidence for one evaluated page.""" + +from __future__ import annotations + +import hashlib +import json +import shlex +import shutil +from dataclasses import asdict, dataclass +from decimal import Decimal +from pathlib import Path +from typing import Any + +from entry_points.run_pipeline import run_pipeline +from vision_aid.evaluation.execution.llm_audit import ( + USAGE_KEYS, + LLMRequestConfig, + evaluation_configuration, + write_json, +) +from vision_aid.evaluation.execution.normalization import ( + normalize_programmatic_findings, +) +from vision_aid.evaluation.execution.pricing import DEFAULT_SCHEDULE, PriceSchedule +from vision_aid.evaluation.human_finding_mapping import ( + apply_human_finding_mapping, + saved_human_finding_mapping_decisions, +) +from vision_aid.evaluation.reporting.html_report import write_page_report +from vision_aid.evaluation.review import create_review, reviewed_rows +from vision_aid.evaluation.schemas import HumanAudit + + +@dataclass(frozen=True) +class PageReview: + """A completed page's validated review and authoritative saved evidence.""" + + manifest: dict[str, Any] + rows: list[dict[str, Any]] + llm: list[dict[str, Any]] + programmatic: list[dict[str, Any]] + schedule: PriceSchedule + + +def prepare_page( + run: Path, + html: Path, + source_url: str, + human_audit: HumanAudit, + request_config: LLMRequestConfig, + limit: Decimal, + mapping: dict[str, Any], +) -> dict[str, Any]: + """Save a page's inputs, checks, prompts and blank review without paid requests. + + The caller supplies an existing fresh run directory and applicable Human + findings, retaining the same operations for one-page and Full evaluations. + """ + if not html.is_file(): + raise ValueError(f"Benchmark HTML not found: {html}") + configuration = evaluation_configuration(request_config) + artifacts = run / "artifacts" + artifacts.mkdir() + shutil.copyfile(html, run / "snapshot.html") + shutil.copyfile(DEFAULT_SCHEDULE, artifacts / "pricing.json") + snapshot_hash = hashlib.sha256((run / "snapshot.html").read_bytes()).hexdigest() + write_json(artifacts / "human-findings.json", human_audit.to_dict()) + production = run_pipeline( + html_path=str(run / "snapshot.html"), + output_dir=artifacts, + api_key=None, + model=request_config.model, + dry_run=True, + include_summaries=False, + request_config=request_config, + ) + (artifacts / "manifest.json").unlink() + raw_path = artifacts / "programmatic_findings.json" + raw_programmatic = json.loads(raw_path.read_text()) + raw_path.rename(artifacts / "raw-programmatic-findings.json") + findings = normalize_programmatic_findings( + raw_programmatic, + run_id=run.name, + page_url=source_url, + ) + write_json( + artifacts / "normalized-programmatic-findings.json", + [asdict(item) for item in findings], + ) + create_review( + run / "review.csv", list(human_audit.to_dict()["human_findings"]) + ) + schedule = PriceSchedule.load(artifacts / "pricing.json") + manifest = { + "run_id": run.name, + "source_url": source_url, + "human_audit_filename": human_audit.human_audit_filename, + "human_audit_sha256": human_audit.human_audit_sha256, + "snapshot_sha256": snapshot_hash, + "configuration": configuration, + "pricing_version": schedule.version, + "pricing_sha256": schedule.identity, + "max_cost_usd": str(limit), + "complete": False, + "incomplete_reason": "preview", + "estimated_cost_usd": "0", + "usage": {key: 0 for key in USAGE_KEYS}, + "format_failures": [], + "prompts": [ + {"name": entry["name"], "checklist": entry["checklist"]} + for entry in production["prompts_dry_run"] + ], + "filters": production["pass1_filters_active"], + "skipped_prompts": production["prompts_skipped"], + } + write_json(run / "run.json", manifest) + apply_human_finding_mapping(run, mapping) + (run / "README.md").write_text( + "# Review this page run\n\n" + "Check `run.json` for completion and any stopping reason. " + "Add `--live` to `run` or `run-all` to call the LLMs. Without `--live`, the command " + "runs in preview mode, generating artifacts without calling the LLMs.\n\n" + "1. After execution completes, compare `normalized-llm-findings.json` with the Human " + "findings in `review.csv`. Use `snapshot.html` to check the page evidence.\n" + "2. Edit only `llm_finding_id`, `match_reason`, and `review_notes`. Separate joint " + "matches with semicolons. Select findings only when they fully cover the Human " + "finding; otherwise leave the LLM ID blank. Keep approved fields unchanged.\n" + "3. Save the CSV, then run from the repository root:\n\n" + "```bash\n" + f"uv run visionaid-evaluate report --run-dir {shlex.quote(str(run.resolve()))} " + '--reviewer "Your name and review method"\n' + "```\n\n" + "Open `report.html` in a browser. Supporting evidence, including " + "`normalized-programmatic-findings.json`, is in `artifacts/`.\n", + encoding="utf-8", + ) + manifest = json.loads((run / "run.json").read_text()) + return manifest + + +def validated_page_review(run: Path) -> PageReview: + """Load and validate a completed page's evidence and human decisions.""" + run = run.resolve() + artifacts = run / "artifacts" + manifest = json.loads((run / "run.json").read_text()) + if manifest["run_id"] != run.name: + raise ValueError("Run evidence does not belong to this directory") + if not manifest["complete"]: + raise ValueError("Incomplete Evaluation runs cannot produce a final report") + human_audit = json.loads((artifacts / "human-findings.json").read_text()) + if human_audit["human_audit_sha256"] != manifest["human_audit_sha256"]: + raise ValueError("Human audit identity differs from the run") + llm = json.loads((run / "normalized-llm-findings.json").read_text()) + programmatic_file = artifacts / "normalized-programmatic-findings.json" + programmatic = json.loads(programmatic_file.read_text()) + for kind, collection in (("llm", llm), ("programmatic", programmatic)): + if any( + item["run_id"] != run.name + or item["source"] != kind + or item["page_url"] != manifest["source_url"] + for item in collection + ): + label = kind.upper() if kind == "llm" else kind.title() + raise ValueError(f"{label} findings do not belong to this run") + approved = saved_human_finding_mapping_decisions(run, manifest, human_audit, programmatic) + rows = reviewed_rows( + run / "review.csv", human_audit["human_findings"], llm, programmatic, approved + ) + schedule = PriceSchedule.load(artifacts / "pricing.json") + return PageReview(manifest, rows, llm, programmatic, schedule) + + +def report_page(run: Path, reviewer: str) -> PageReview: + """Validate human decisions and write the page report with attribution.""" + (run / "report.html").unlink(missing_ok=True) + if not reviewer.strip(): + raise ValueError("A reviewer description is required") + review = validated_page_review(run) + write_page_report(run / "report.html", review, reviewer) + (run / "report.md").unlink(missing_ok=True) + return review diff --git a/vision_aid/evaluation/reporting/__init__.py b/vision_aid/evaluation/reporting/__init__.py new file mode 100644 index 0000000..46ce280 --- /dev/null +++ b/vision_aid/evaluation/reporting/__init__.py @@ -0,0 +1 @@ +"""Render self-contained Evaluation reports and their packaged assets.""" diff --git a/vision_aid/evaluation/reporting/css/charts.css b/vision_aid/evaluation/reporting/css/charts.css new file mode 100644 index 0000000..d401ae5 --- /dev/null +++ b/vision_aid/evaluation/reporting/css/charts.css @@ -0,0 +1,27 @@ +.performance-heading { margin-bottom: 1.5rem; } +.performance-heading h2 { margin: 0 0 1rem; } +.cost-overview { display: grid; grid-template-columns: minmax(0, .8fr) minmax(0, 1.2fr); gap: 1.5rem; align-items: start; } +.cost-overview > * { min-width: 0; } +@media (min-width: 60rem) { .cost-overview { align-items: stretch; } } +.run-cost { margin: 0; padding: .75rem 1rem; max-width: 100%; border: 3px solid #164e88; background: #edf2f6; } +.cost-label { display: block; margin-bottom: .4rem; font-weight: 650; } +.cost-value { display: block; font-size: clamp(2rem, 4vw, 3rem); line-height: 1.1; font-variant-numeric: tabular-nums; overflow-wrap: anywhere; } +.cost-currency { font-size: 1rem; font-weight: 650; } +.token-usage { display: grid; grid-template-columns: minmax(0, 1fr) auto; gap: .4rem 1rem; margin: 1.25rem 0 0; padding-top: .75rem; border-top: 1px solid #687f93; } +.token-usage dd { margin: 0; text-align: right; font-variant-numeric: tabular-nums; } +.model-pricing h3 { margin: 0; } +.model-pricing table { min-width: 26rem; margin-bottom: 0; } +.model-pricing th { overflow-wrap: normal; } +.model-pricing td { white-space: nowrap; } +@media (max-width: 60rem) { .cost-overview { grid-template-columns: minmax(0, 1fr); } } +.detection-chart { margin: 1rem 0; } +.chart-bars { list-style: none; padding: 0; } +.chart-bars li { margin-block: 1rem; padding: .75rem; border: 1px solid #687f93; overflow-wrap: anywhere; } +.chart-bars .chart-primary { border: 3px solid #164e88; background: #edf2f6; } +.chart-primary .chart-value { font-size: 1.2rem; } +.chart-value { margin: 0 0 .5rem; } +.chart-percentage { text-decoration: underline; text-underline-offset: .15em; } +.chart-track { width: 100%; height: 1.4rem; background: #edf2f6; border: 1px solid #687f93; } +.chart-fill { display: block; height: 100%; background: #164e88; } +@media (forced-colors: active) { .chart-fill { background: Highlight; } } +@media print { .chart-track, .chart-fill { print-color-adjust: exact; } .chart-bars li { break-inside: avoid; } } diff --git a/vision_aid/evaluation/reporting/css/print.css b/vision_aid/evaluation/reporting/css/print.css new file mode 100644 index 0000000..5c4f3db --- /dev/null +++ b/vision_aid/evaluation/reporting/css/print.css @@ -0,0 +1,11 @@ +#print-report { font: inherit; color: #164e88; background: #edf2f6; border: 1px solid #687f93; border-radius: .2rem; padding: .5rem .75rem; margin-bottom: 1rem; cursor: pointer; } +@page { margin: 12mm; } +@media print { + body { max-width: none; padding: 0; color: #000; font-size: 11pt; line-height: 1.4; } + .skip, nav, #print-report, main > :not(#performance) { display: none !important; } + .table-scroll { overflow: visible; } + table { table-layout: fixed; } + thead { display: table-header-group; } + th, td { padding: .35rem; } + h1, h2, h3, caption { break-after: avoid; } +} diff --git a/vision_aid/evaluation/reporting/css/report.css b/vision_aid/evaluation/reporting/css/report.css new file mode 100644 index 0000000..405c193 --- /dev/null +++ b/vision_aid/evaluation/reporting/css/report.css @@ -0,0 +1,57 @@ +:root { color-scheme: light; font-family: system-ui, sans-serif; color: #182838; background: #fff; line-height: 1.6; } +* { box-sizing: border-box; } +body { margin: 0 auto; max-width: 90rem; padding: 1.25rem; } +h1, h2, h3 { line-height: 1.25; } +h1 { font-size: clamp(1.8rem, 5vw, 2.7rem); } +a { color: #164e88; text-underline-offset: .18em; } +a:focus-visible, summary:focus-visible, button:focus-visible, input:focus-visible, select:focus-visible { outline: 3px solid #8c3500; outline-offset: 4px; } +.skip { position: absolute; top: -10rem; padding: .7rem; background: #fff; } +.skip:focus { top: .5rem; } +nav { display: flex; flex-wrap: wrap; gap: 1.25rem; border-block: 1px solid #8090a0; padding-block: .8rem; } +section { margin-block: 2rem; } +.table-scroll { overflow-x: auto; } +table { width: 100%; border-collapse: collapse; margin-block: 1rem; } +caption { font-weight: 700; text-align: left; margin-bottom: .5rem; } +th, td { text-align: left; vertical-align: top; border: 1px solid #8a99a6; padding: .65rem; overflow-wrap: anywhere; } +thead { background: #edf2f6; } +summary { cursor: pointer; color: #164e88; font-weight: 600; } +details[open] > summary { margin-bottom: 1rem; } +.evidence { min-width: 16rem; max-width: 50rem; } +dl { margin-block: 1rem; } +dt { font-weight: 650; } +dd { margin: 0 0 .8rem; white-space: pre-wrap; overflow-wrap: anywhere; } +.matched { border-left: 3px solid #687f93; padding-left: 1rem; margin-block: 1rem; } +@media (max-width: 40rem) { body { padding: .8rem; } .evidence { min-width: 13rem; } } +@media print { body { max-width: none; } .skip, nav { display: none; } .table-scroll { overflow: visible; } } + +#finding-controls fieldset { border: 1px solid #8a99a6; padding: 1rem; } +#finding-controls label { display: block; font-weight: 650; margin-top: .8rem; } +#finding-controls select { display: block; width: 100%; max-width: 42rem; font: inherit; padding: .5rem; color: inherit; background: #fff; border: 1px solid #687f93; } +#finding-controls button { font: inherit; color: #164e88; background: #fff; border: 1px solid #687f93; border-radius: .2rem; padding: .5rem .75rem; margin: .5rem .5rem .5rem 0; cursor: pointer; } +[data-selected-page-result] { border-left: 3px solid #687f93; padding-left: .7rem; } + +@media screen { + .table-scroll > table { min-width: 32rem; } + .table-scroll thead th { overflow-wrap: normal; } + #findings { min-width: 64rem; } + #findings > tbody > tr > th, #findings > tbody > tr > td:last-child { min-width: 16rem; } + .findings-panel { border: 1px solid #8a99a6; border-bottom: 0; } + .findings-panel > .table-scroll { margin-inline: -1px; } + .findings-panel > .table-scroll > table { margin: 0; } + #finding-controls { background: #edf2f6; padding-top: 1rem; } + #finding-controls fieldset { min-width: 0; margin: 0; padding: 0 1rem 1rem; border: 0; } + #finding-controls legend { padding: 0; font-weight: 650; margin-bottom: .75rem; } + .finding-filter-fields { display: grid; grid-template-columns: minmax(0, 1fr); gap: 1rem; } + .finding-filter { min-width: 0; } + #finding-controls label { margin-top: 0; } + #finding-controls select { max-width: none; } + .finding-filter-actions { display: flex; flex-wrap: wrap; gap: .75rem; margin-top: .75rem; } + #finding-controls button { margin: 0; } + #finding-count { margin: 0; padding: .75rem 1rem; background: #fff; border-top: 1px solid #8a99a6; } +} +@media screen and (min-width: 64rem) { + .finding-filter-fields { grid-template-columns: repeat(auto-fit, minmax(18rem, 1fr)); } +} +@media screen and (max-width: 40rem) { + .evidence { width: 16rem; } +} diff --git a/vision_aid/evaluation/reporting/html_report.py b/vision_aid/evaluation/reporting/html_report.py new file mode 100644 index 0000000..cce4fbc --- /dev/null +++ b/vision_aid/evaluation/reporting/html_report.py @@ -0,0 +1,236 @@ +"""Prepare reader-facing data and publish self-contained evaluation reports.""" + +from __future__ import annotations + +import os +import tempfile +from decimal import Decimal +from importlib.resources import files +from pathlib import Path +from typing import TYPE_CHECKING, Any + +from jinja2 import ( + Environment, + PackageLoader, + StrictUndefined, + TemplateError, + select_autoescape, +) + +from vision_aid.evaluation.review import detection_counts + +if TYPE_CHECKING: + from vision_aid.evaluation.execution.pricing import PriceSchedule + from vision_aid.evaluation.page import PageReview + +EXPECTED = { + "llm_eligible": "LLM", + "programmatic": "Programmatic checks", + "unavailable_evidence": "Not testable from the page file", + "ambiguous": "Not testable from the page file", +} + + +def _display_model(model: str) -> str: + """Use uppercase GPT in report labels while preserving saved model IDs.""" + return "GPT-" + model[4:].replace("-", " ").title() if model.startswith("gpt-") else model + + +def _cost_details(manifest: dict[str, Any], schedules: list[PriceSchedule]) -> dict[str, Any]: + """Present saved usage and model prices without recalculating the run cost.""" + usage = manifest["usage"] + model = manifest["configuration"].get("model") + if model is None: + raise ValueError("Cannot render complete report: model is missing") + token_usage = [ + {"label": label, "count": f"{usage[key]:,}"} + for key, label in ( + ("input_tokens", "Input tokens"), + ("output_tokens", "Output tokens"), + ("cached_input_tokens", "Cached input tokens"), + ("cache_creation_input_tokens", "Cache-write input tokens"), + ) + if key in {"input_tokens", "output_tokens"} or usage[key] + ] + model_prices = [] + seen = set() + for schedule in schedules: + if schedule.identity in seen: + continue + seen.add(schedule.identity) + rates = schedule.data["models"][model] + upper = rates.get("long_context") + tiers = [{"label": "All requests", "rates": rates}] + if upper: + threshold = f"{upper['above_input_tokens']:,}" + tiers = [ + {"label": f"Up to {threshold}", "rates": rates}, + {"label": f"Over {threshold}", "rates": upper}, + ] + model_prices.append({ + "model": _display_model(model), "description": schedule.price_description, + "tiers": tiers, + "show_cache_read": bool(usage["cached_input_tokens"]), + "show_cache_write": bool(usage["cache_creation_input_tokens"]), + }) + return { + "cost": f"{Decimal(manifest['estimated_cost_usd']):.6f}", + "model_name": _display_model(model), + "token_usage": token_usage, "model_prices": model_prices, + } + + +def _chart_rates(counts: dict[str, dict[str, int]]) -> list[dict[str, Any]]: + """Prepare chart labels and widths from the existing deterministic counts.""" + rates = [] + for kind, label in ( + ("llm", "LLM"), + ("programmatic", "Programmatic"), + ("overall", "Overall"), + ): + caught, eligible = counts[kind]["caught"], counts[kind]["eligible"] + percentage = f"{caught / eligible * 100:.1f}" if eligible else None + rates.append({ + "label": label, **counts[kind], + "bar_percent": percentage, + "percentage": percentage + "%" if percentage is not None else ( + "Not defined: no imported Human findings" if kind == "overall" + else "Not defined: no eligible Human findings" + ), + }) + return rates + + +def page_data(review: PageReview, reviewer: str) -> dict[str, Any]: + """Resolve validated evidence and deterministic scores before rendering.""" + manifest = review.manifest + findings = {item["finding_id"]: item for item in review.llm + review.programmatic} + counts = detection_counts(review.rows) + rows = [] + for row in review.rows: + human, decision = row["human_finding"], row["decision"] + classification = decision["classification"].strip() + matched = [] + for kind in ("llm", "programmatic"): + for identity in row["matches"][kind]: + finding = findings[identity] + check = finding["prompt"].replace("_", " ") + " check" + if finding.get("screen_only"): + check += "; shown on screen only, not in the tool's CSV report" + matched.append( + { + "id": finding["finding_id"], + "source": "LLM" if kind == "llm" else "Programmatic checks", + "check": check, + "check_id": finding["prompt"], + "checklist": finding["checklist"], + "problem": finding["problem"], + "element": finding["element"], + "location": finding["location"], + } + ) + rows.append( + { + "human": human, + "human_finding_mapping_approved_by": manifest["human_finding_mapping_approved_by"], + "decision": decision, + "source_url": manifest["source_url"], + "run_id": manifest["run_id"], + "expected": EXPECTED[classification], + "llm_outcome": "Caught" if row["matches"]["llm"] else "Not caught", + "programmatic_outcome": "Caught" + if row["matches"]["programmatic"] + else "Not caught", + "matched": matched, + } + ) + rates = _chart_rates(counts) + return { + "title": "LLM Accessibility Evaluation", + "manifest": manifest, + "reviewer": reviewer, + "rows": rows, + "rates": rates, + **_cost_details(manifest, [review.schedule]), + } + + +def full_data( + manifest: dict[str, Any], pooled: list[dict[str, Any]], + reviews: list[PageReview], reviewer: str, +) -> dict[str, Any]: + """Combine validated presentation rows without changing pooled scoring.""" + pages = [page_data(review, reviewer) for review in reviews] + grouped: dict[str, list[dict[str, Any]]] = {} + for page in pages: + for row in page["rows"]: + grouped.setdefault(row["human"]["human_finding_id"], []).append({ + **row, "source_url": page["manifest"]["source_url"], + "run_id": page["manifest"]["run_id"], + }) + counts = detection_counts(pooled) + rows = [] + for row in pooled: + rows.append({ + "human": row["human_finding"], + "expected": EXPECTED[row["decision"]["classification"]], + "llm_outcome": "Caught" if row["matches"]["llm"] else "Not caught", + "programmatic_outcome": "Caught" if row["matches"]["programmatic"] else "Not caught", + "pages": grouped[row["human_finding"]["human_finding_id"]], + }) + return { + "title": "Full Accessibility Evaluation", "full": True, + "manifest": manifest, "reviewer": reviewer, "rows": rows, "pages": pages, + "rates": _chart_rates(counts), + **_cost_details(manifest, [review.schedule for review in reviews]), + } + + +def write_full_report( + path: Path, manifest: dict[str, Any], pooled: list[dict[str, Any]], + reviews: list[PageReview], reviewer: str, +) -> None: + """Render every accepted page decision into the standalone parent.""" + _write_html(path, "full.html.jinja", full_data(manifest, pooled, reviews, reviewer)) + + +def write_page_report(path: Path, review: PageReview, reviewer: str) -> None: + """Publish a standalone individual Evaluation report.""" + _write_html(path, "page.html.jinja", page_data(review, reviewer)) + + +def _write_html(path: Path, template: str, data: dict[str, Any]) -> None: + """Render completely, then atomically replace the report in its directory.""" + environment = Environment( + loader=PackageLoader("vision_aid.evaluation.reporting", "templates"), + autoescape=select_autoescape(("html.jinja",)), + undefined=StrictUndefined, + ) + # Only trusted package assets bypass escaping; review data stays escaped. + assets = files("vision_aid.evaluation.reporting") + styles = "".join( + assets.joinpath("css", name).read_text(encoding="utf-8") + for name in ("report.css", "charts.css", "print.css") + ) + scripts = { + name: assets.joinpath("js", f"{name}.js").read_text(encoding="utf-8") + for name in ("filters", "print") + } + try: + content = environment.get_template(template).render( + **data, styles=styles, scripts=scripts, + ) + except TemplateError as error: + raise ValueError(f"Cannot render complete report: {error}") from error + temporary = None + try: + with tempfile.NamedTemporaryFile( + mode="w", encoding="utf-8", dir=path.parent, + prefix=f".{path.name}.", suffix=".tmp", delete=False, + ) as stream: + temporary = Path(stream.name) + stream.write(content) + os.replace(temporary, path) + finally: + if temporary is not None: + temporary.unlink(missing_ok=True) diff --git a/vision_aid/evaluation/reporting/js/filters.js b/vision_aid/evaluation/reporting/js/filters.js new file mode 100644 index 0000000..31bd093 --- /dev/null +++ b/vision_aid/evaluation/reporting/js/filters.js @@ -0,0 +1,60 @@ +/* All evidence is already in this document; no network or runtime dependencies. */ +(() => { + 'use strict'; + const controls = document.getElementById('finding-controls'); + const pageFilter = document.getElementById('finding-page'); + const outcome = document.getElementById('finding-outcome'); + const status = document.getElementById('finding-count'); + const rows = [...document.querySelectorAll('#findings > tbody > tr')].map(row => { + return {row, decisions: [...row.querySelectorAll('[data-page-decision]')], + selected: row.querySelector('[data-selected-page-result]')}; + }); + function matchesOutcome(data) { + const llm = data.llm === 'Caught'; + const programmatic = data.programmatic === 'Caught'; + switch (outcome.value) { + case 'llm-caught': return llm; + case 'llm-missed': return data.expected === 'LLM' && !llm; + case 'programmatic-caught': return programmatic; + case 'programmatic-missed': return data.expected === 'Programmatic checks' && !programmatic; + case 'overall-caught': return llm || programmatic; + case 'overall-missed': return !llm && !programmatic; + case 'not-testable': return data.expected === 'Not testable from the page file'; + default: return true; + } + } + function update() { + const selectedPage = pageFilter?.value || ''; + let count = 0; + for (const item of rows) { + const pageDecision = selectedPage ? item.decisions.find(panel => panel.dataset.page === selectedPage) : null; + item.row.hidden = Boolean((selectedPage && !pageDecision) || + !matchesOutcome(pageDecision?.dataset || item.row.dataset)); + item.selected.hidden = !selectedPage || !pageDecision; + item.selected.textContent = pageDecision ? `Selected page: ${selectedPage}. Expected source: ${pageDecision.dataset.expected}. LLM: ${pageDecision.dataset.llm}. Programmatic checks: ${pageDecision.dataset.programmatic}. Combined results remain in the overview columns.` : ''; + for (const panel of item.decisions) panel.hidden = Boolean(selectedPage && panel.dataset.page !== selectedPage); + for (const tr of item.row.querySelectorAll('[data-page-table-row]')) tr.hidden = Boolean(selectedPage && tr.dataset.pageTableRow !== selectedPage); + if (!item.row.hidden) count++; + } + status.textContent = `Filtered findings: ${count} of ${rows.length} original Human findings.${count ? '' : ' No matching findings.'}`; + } + pageFilter?.addEventListener('change', update); + outcome.addEventListener('change', update); + document.getElementById('clear-filters').addEventListener('click', () => { + if (pageFilter) pageFilter.value = ''; + outcome.value = 'all'; + update(); + }); + for (const [id, open] of [['expand-visible', true], ['collapse-visible', false]]) { + document.getElementById(id).addEventListener('click', () => { + for (const {row} of rows) { + if (row.hidden) continue; + for (const panel of row.querySelectorAll('details')) { + if (!panel.closest('[hidden]')) panel.open = open; + } + } + }); + } + update(); + controls.hidden = false; +})(); diff --git a/vision_aid/evaluation/reporting/js/print.js b/vision_aid/evaluation/reporting/js/print.js new file mode 100644 index 0000000..24a1a61 --- /dev/null +++ b/vision_aid/evaluation/reporting/js/print.js @@ -0,0 +1,29 @@ +/* Print styling limits output to the report header, cost and detection charts. */ +(() => { + 'use strict'; + const button = document.getElementById('print-report'); + const printMedia = window.matchMedia('print'); + let readingState = null; + function restoreFocus() { + if (!readingState || printMedia.matches) return; + const {focused, scrollX, scrollY} = readingState; + readingState = null; + if (focused?.isConnected) focused.focus({preventScroll: true}); + window.scrollTo(scrollX, scrollY); + } + printMedia.addEventListener('change', restoreFocus); + window.addEventListener('afterprint', restoreFocus); + button.addEventListener('click', () => { + if (readingState) return; + readingState = { + focused: document.activeElement, scrollX: window.scrollX, scrollY: window.scrollY, + }; + try { + window.print(); + } catch (error) { + restoreFocus(); + throw error; + } + }); + button.hidden = false; +})(); diff --git a/vision_aid/evaluation/reporting/templates/base.html.jinja b/vision_aid/evaluation/reporting/templates/base.html.jinja new file mode 100644 index 0000000..5ddbbef --- /dev/null +++ b/vision_aid/evaluation/reporting/templates/base.html.jinja @@ -0,0 +1,25 @@ + + + + + +{% block document_title %}{% endblock %} + + + + +
+

{{ title }}: {{ model_name }}

+{% block scope %}{% endblock %} +

Run ID: {{ manifest.run_id }}.

+

Review decisions by: {{ reviewer }}

+ + +
+
+{% block content %}{% endblock %} +
+ + + + diff --git a/vision_aid/evaluation/reporting/templates/charts.html.jinja b/vision_aid/evaluation/reporting/templates/charts.html.jinja new file mode 100644 index 0000000..f7831ee --- /dev/null +++ b/vision_aid/evaluation/reporting/templates/charts.html.jinja @@ -0,0 +1,44 @@ +{% macro performance_header(cost, token_usage, model_prices) %} +
+

Cost and detection results

+
+
Estimated run cost:${{ cost }} USD +
{% for item in token_usage %}
{{ item.label }}
{{ item.count }}
{% endfor %}
+
+

Listed model prices

+{% for prices in model_prices %} +

{{ prices.model }}. {{ prices.description | capitalize }}.

+
+ +{% if prices.show_cache_read %}{% endif %}{% if prices.show_cache_write %}{% endif %} +{% for tier in prices.tiers %}{% if prices.show_cache_read %}{% endif %}{% if prices.show_cache_write %}{% endif %}{% endfor %} +
USD per 1 million tokens
Input tokens per requestInputOutputCache readCache write
{{ tier.label }}${{ tier.rates.input_per_million_usd }}${{ tier.rates.output_per_million_usd }}${{ tier.rates.cached_input_per_million_usd }}${{ tier.rates.cache_creation_input_per_million_usd }}
+{% endfor %} +
+
+
+{% endmacro %} + +{% macro rate_bar(label, rate, primary=false) %} + +

{{ label }}: {{ rate.caught }} caught / {{ rate.eligible }} eligible; {{ rate.percentage }}

+{% if rate.bar_percent is not none %}{% endif %} + +{% endmacro %} + +{% macro detection_overview(rates) %} +
+

Detection overview

+
    {% for rate in rates %}{{ rate_bar(rate.label, rate, loop.first) }}{% endfor %}
+

Overall includes all Human findings, including those the tool cannot detect from the saved HTML and those too ambiguous to evaluate. These findings are left out of the LLM and Programmatic eligible counts, so the Overall count can exceed their combined total.

+
+{% endmacro %} + +{% macro page_comparison(pages) %} +
+

LLM detection by page

+

Repeated Global findings contribute to each page's results but only once to combined results.

+
+
    {% for page in pages %}{{ rate_bar(page.manifest.source_url, page.rates[0]) }}{% endfor %}
+
+{% endmacro %} diff --git a/vision_aid/evaluation/reporting/templates/controls.html.jinja b/vision_aid/evaluation/reporting/templates/controls.html.jinja new file mode 100644 index 0000000..884e39a --- /dev/null +++ b/vision_aid/evaluation/reporting/templates/controls.html.jinja @@ -0,0 +1,32 @@ + diff --git a/vision_aid/evaluation/reporting/templates/evidence.html.jinja b/vision_aid/evaluation/reporting/templates/evidence.html.jinja new file mode 100644 index 0000000..9ebd0d5 --- /dev/null +++ b/vision_aid/evaluation/reporting/templates/evidence.html.jinja @@ -0,0 +1,16 @@ +{% macro human_evidence(row) %} +

Original Human finding

+
Human finding ID
{{ row.human.human_finding_id }}
Source
{{ row.human.source_sheet }}, row {{ row.human.source_row }}; scope: {{ row.human.page_scope }}; element: {{ row.human.source_element }}
+{% for label, value in row.human.raw_evidence.items() %}
{{ label }}
{{ value }}
{% endfor %}
+{% endmacro %} +{% macro decision_evidence(row) %} +

{{ row.source_url }}

+

Run ID: {{ row.run_id }}

+
Eligibility classification
{{ row.expected }}
Recorded explanation
{{ row.decision.classification_reason }}
Human finding mapping approved by
{{ row.human_finding_mapping_approved_by }}
Approved Programmatic explanation
{{ row.decision.programmatic_match_reason }}
{% if row.decision.llm_finding_id %}
LLM matching rationale
{{ row.decision.match_reason }}
{% endif %}{% if row.matched or row.decision.review_notes %}
VA-DAT Matched Findings
{{ row.decision.review_notes }}
{% endif %}
+{% for finding in row.matched %}

{{ 'LLM Finding' if finding.source == 'LLM' else 'Programmatic Finding' }}

{{ finding.source }} ({{ finding.check }}): {{ finding.problem }}

Finding ID
{{ finding.id }}
Check
{{ finding.check_id }}; {{ finding.checklist }}
{% if finding.source == 'LLM' %}
Element
{{ finding.element }}
{% endif %}
Location
{{ finding.location }}
{% endfor %} +{% endmacro %} +{% macro page_evidence(row) %} +
View page decisions: {{ row.source_url }} +{{ decision_evidence(row) }} +
+{% endmacro %} diff --git a/vision_aid/evaluation/reporting/templates/full.html.jinja b/vision_aid/evaluation/reporting/templates/full.html.jinja new file mode 100644 index 0000000..9965ede --- /dev/null +++ b/vision_aid/evaluation/reporting/templates/full.html.jinja @@ -0,0 +1,39 @@ +{% extends 'base.html.jinja' %} +{% from 'evidence.html.jinja' import human_evidence, page_evidence %} +{% from 'charts.html.jinja' import performance_header, detection_overview, page_comparison %} + +{% block document_title %}{{ title }}: {{ model_name }}; {{ manifest.run_id }}{% endblock %} +{% block content %} +
+{{ performance_header(cost, token_usage, model_prices) }} +{{ detection_overview(rates) }} +{{ page_comparison(pages) }} +
+
+

Tool Findings and Human Finding Matches

+

Each original Human finding appears once in source order. Open "View details" to read the original Human finding, then "View page decisions" to read the review decision and matched tool findings. Global findings include a separate page decisions dropdown for every page. Details and page decisions start collapsed.

+
+{% include 'controls.html.jinja' %} +
+ + + +{% for row in rows %} + + + +{% endfor %} +
Issue titleWCAGScopeExpected sourceLLMProgrammatic checksDetails
{{ row.human.problem }}{{ row.human.raw_evidence['WCAG Sc'] }}{% if row.human.page_scope == 'global' %}Global{% else %}Page-local{% endif %}{{ row.expected }}{{ row.llm_outcome }}{{ row.programmatic_outcome }}
View details
+{{ human_evidence(row) }} +{% if row.human.page_scope == 'global' %} +

Combined result

+
+ +{% for page in row.pages %}{% endfor %} +
Page decisions
PageExpected sourceLLMProgrammatic checks
{{ page.source_url }}{{ page.expected }}{{ page.llm_outcome }}{{ page.programmatic_outcome }}
+{% endif %} +{% for page in row.pages %}{{ page_evidence(page) }}{% endfor %} +
+
+
+{% endblock %} diff --git a/vision_aid/evaluation/reporting/templates/page.html.jinja b/vision_aid/evaluation/reporting/templates/page.html.jinja new file mode 100644 index 0000000..c4e076b --- /dev/null +++ b/vision_aid/evaluation/reporting/templates/page.html.jinja @@ -0,0 +1,35 @@ +{% extends 'base.html.jinja' %} +{% from 'evidence.html.jinja' import human_evidence, page_evidence %} +{% from 'charts.html.jinja' import performance_header, detection_overview, page_comparison %} + +{% block document_title %}{{ title }}: {{ model_name }}; {{ manifest.source_url }}{% endblock %} +{% block scope %}

{{ manifest.source_url }}

{% endblock %} +{% block content %} +
+{{ performance_header(cost, token_usage, model_prices) }} +{{ detection_overview(rates) }} +
+
+

Tool Findings and Human Finding Matches

+

Open View details to read the original Human finding, then View page decisions to read the review decision and matched tool findings. Details and page decisions start collapsed.

+
+{% include 'controls.html.jinja' %} +
+ + + +{% for row in rows %} + + + + +{% endfor %} +
Issue titleWCAGExpected sourceLLMProgrammatic checksDetails
{{ row.human.problem }}{{ row.human.raw_evidence['WCAG Sc'] }}{{ row.expected }}{{ row.llm_outcome }}{{ row.programmatic_outcome }}
View details +
+{{ human_evidence(row) }} +{{ page_evidence(row) }} +
+
+{% if not rows %}

No Human findings were imported for this page.

{% endif %} +
+{% endblock %} diff --git a/vision_aid/evaluation/review.py b/vision_aid/evaluation/review.py new file mode 100644 index 0000000..40f421c --- /dev/null +++ b/vision_aid/evaluation/review.py @@ -0,0 +1,166 @@ +"""Validate editable Human review decisions and calculate deterministic counts.""" + +from __future__ import annotations + +import csv +from pathlib import Path +from typing import Any + +from vision_aid.evaluation.human_finding_mapping import LOCKED_COLUMNS +from vision_aid.evaluation.inputs.human_audit import ELIGIBILITY_VALUES, HEADERS + +DECISION_COLUMNS = ( + "classification", + "classification_reason", + "llm_finding_id", + "programmatic_finding_id", + "match_reason", + "review_notes", +) +REVIEW_COLUMNS = ( + "human_finding_id", + "source_sheet", + "source_row", + "page_scope", + "source_element", + *HEADERS, + *DECISION_COLUMNS, +) + + +def create_review(path: Path, human_findings: list[dict[str, Any]]) -> None: + """Create one editable row per Human finding with blank decisions.""" + with path.open("w", encoding="utf-8", newline="") as stream: + writer = csv.DictWriter(stream, fieldnames=REVIEW_COLUMNS) + writer.writeheader() + for item in human_findings: + writer.writerow( + { + "human_finding_id": item["human_finding_id"], + "source_sheet": item["source_sheet"], + "source_row": item["source_row"], + "page_scope": item["page_scope"], + "source_element": item["source_element"], + **item["raw_evidence"], + **{column: "" for column in DECISION_COLUMNS}, + } + ) + + +def _ids(value: str) -> tuple[str, ...]: + """Semicolons join multiple finding IDs in either finding column.""" + parts = tuple(part.strip() for part in value.split(";") if part.strip()) + if len(parts) != len(set(parts)): + raise ValueError("A finding ID is repeated within one review row") + return parts + + +def reviewed_rows( + path: Path, + human_findings: list[dict[str, Any]], + llm: list[dict[str, Any]], + programmatic: list[dict[str, Any]], + approved: dict[str, dict[str, str]], +) -> list[dict[str, Any]]: + """Validate completeness and finding kind before granting any row credit.""" + with path.open(encoding="utf-8-sig", newline="") as stream: + reader = csv.DictReader(stream) + if not set(REVIEW_COLUMNS).issubset(reader.fieldnames or ()): + raise ValueError("Review CSV is missing required columns") + if not set(LOCKED_COLUMNS).issubset(reader.fieldnames or ()): + raise ValueError("Review CSV is missing approved Human finding mapping columns") + entered = list(reader) + expected = {item["human_finding_id"]: item for item in human_findings} + if len(entered) != len(expected): + raise ValueError("Review CSV must contain each Human finding exactly once") + by_kind = { + "llm": {item["finding_id"]: item for item in llm}, + "programmatic": {item["finding_id"]: item for item in programmatic}, + } + used: dict[str, set[str]] = {"llm": set(), "programmatic": set()} + result: dict[str, dict[str, Any]] = {} + for row in entered: + human_finding_id = row["human_finding_id"] + if human_finding_id not in expected or human_finding_id in result: + raise ValueError(f"Unknown or repeated Human finding: {human_finding_id}") + human_finding = expected[human_finding_id] + for field in ("source_sheet", "source_row", "source_element"): + if str(row[field]) != str(human_finding[field]): + raise ValueError( + f"Human finding identity changed for {human_finding_id}: {field}" + ) + if ( + row["page_scope"] != human_finding["page_scope"] or any( + row[field] != human_finding["raw_evidence"].get(field, "") for field in HEADERS + ) + ): + raise ValueError(f"Human finding evidence changed for {human_finding_id}") + for column in LOCKED_COLUMNS: + actual, required = row[column], approved[human_finding_id][column] + if column == "programmatic_finding_id": + unchanged = set(_ids(actual)) == set(_ids(required)) + else: + unchanged = actual == required + if not unchanged: + raise ValueError( + f"Human finding {human_finding_id}: approved Human finding mapping " + f"decision changed: {column}. Restore the saved decision; " + "review LLM matches in this run, or approve a new Human finding mapping CSV review for future runs." + ) + classification = row["classification"].strip() + if classification not in ELIGIBILITY_VALUES: + raise ValueError(f"Human finding {human_finding_id}: invalid classification") + matched: dict[str, tuple[str, ...]] = {} + for kind, column, label in ( + ("llm", "llm_finding_id", "LLM"), + ("programmatic", "programmatic_finding_id", "Programmatic"), + ): + ids = _ids(row[column]) + unknown = set(ids) - set(by_kind[kind]) + if unknown: + raise ValueError( + f"Unknown {label} finding ID: {', '.join(sorted(unknown))}" + ) + reused = set(ids) & used[kind] + if reused: + raise ValueError( + f"Conflicting {label} finding ID: {', '.join(sorted(reused))}" + ) + used[kind].update(ids) + matched[kind] = ids + if matched["llm"] and not row["match_reason"].strip(): + raise ValueError(f"Human finding {human_finding_id} needs a match reason") + if classification in {"unavailable_evidence", "ambiguous"} and any( + matched.values() + ): + raise ValueError(f"Human finding {human_finding_id} is not testable; cannot claim coverage") + result[human_finding_id] = { + "human_finding": human_finding, "decision": row, "matches": matched, + } + return [result[identity] for identity in expected] + + +def detection_counts(rows: list[dict[str, Any]]) -> dict[str, dict[str, int]]: + """Count validated catches and denominators without averaging page rates. + + Each row represents one Human finding. Cross-catches earn Overall credit, + while source rates include only catches by the expected source. Not-testable + findings remain in the Overall denominator. + """ + counts = {} + for classification, kind in ( + ("llm_eligible", "llm"), ("programmatic", "programmatic") + ): + eligible = [ + row for row in rows + if row["decision"]["classification"].strip() == classification + ] + counts[kind] = { + "caught": sum(bool(row["matches"][kind]) for row in eligible), + "eligible": len(eligible), + } + counts["overall"] = { + "caught": sum(any(row["matches"].values()) for row in rows), + "eligible": len(rows), + } + return counts diff --git a/vision_aid/evaluation/schemas.py b/vision_aid/evaluation/schemas.py new file mode 100644 index 0000000..646a9af --- /dev/null +++ b/vision_aid/evaluation/schemas.py @@ -0,0 +1,50 @@ +"""Evidence records for page evaluations against the Human audit.""" + +from __future__ import annotations + +from dataclasses import asdict, dataclass +from typing import Any, Literal + + +@dataclass(frozen=True) +class HumanFinding: + human_finding_id: str + source_sheet: str + source_row: int + page_scope: Literal["home", "global", "page"] + source_element: str + problem: str + raw_evidence: dict[str, str] + wcag_evidence: tuple[str, ...] + + +@dataclass(frozen=True) +class HumanAudit: + human_audit_filename: str + human_audit_sha256: str + homepage_url: str + human_findings: tuple[HumanFinding, ...] + + def to_dict(self) -> dict[str, Any]: + """Return JSON-compatible source evidence.""" + return asdict(self) + + +@dataclass(frozen=True) +class CanonicalFinding: + finding_id: str + source: Literal["llm", "programmatic"] + run_id: str + model: str | None + prompt: str + checklist: str + page_url: str + problem_family: str + problem: str + element: str + location: str + wcag_evidence: tuple[str, ...] + raw_source: Any + parse_status: str + # True when the tool shows this finding on screen but leaves it out of its CSV report. + screen_only: bool = False