diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index a118b3e..f3cac24 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -25,6 +25,22 @@ jobs: - name: Test run: cargo test --workspace + site: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: dtolnay/rust-toolchain@stable + - uses: Swatinem/rust-cache@v2 + # pages.yml only builds on push, so without this a change that breaks site + # generation lands on main and is discovered after deploy. + - name: Generate site + run: cargo run -q -p willitcall -- site --results results/ --out site/ + - name: Check expected pages exist + run: | + for page in index.html submit.html style.css site.js; do + test -s "site/$page" || { echo "site/$page missing or empty"; exit 1; } + done + results: runs-on: ubuntu-latest steps: @@ -48,38 +64,28 @@ jobs: from pathlib import Path from jsonschema import Draft202012Validator - # Pre-amendment-6 rows scheduled for re-measurement. - PRE_AMENDMENT_6_RESULTS = { - "llamacpp-granite3.1-dense-8b.json", - "llamacpp-phi4-mini.json", - "llamacpp-qwen2.5-1.5b-instruct-q4_k_m.json", - "llamacpp-watt-tool-8b-q4_k_m.json", - "ollama-gemma3-12b.json", - "ollama-gemma3-4b.json", - "ollama-granite3.1-dense-8b.json", - "ollama-hermes3-8b.json", - "ollama-llama3-groq-tool-use-8b.json", - "ollama-llama3.1-8b.json", - "ollama-mistral-7b.json", - "ollama-phi4-mini.json", - "ollama-qwen2.5-7b-instruct.json", - "ollama-qwen3-0.6b.json", - "ollama-qwen3-1.7b.json", - "ollama-qwen3-14b.json", - "ollama-qwen3-4b.json", - "ollama-qwen3-8b.json", - } + # Rows exempt from the amendment-6 environment requirement, which only + # ever applied to schema v2. Every row it covered is now v3, so it is + # empty; the staleness check below keeps it that way. + PRE_AMENDMENT_6_RESULTS = set() + # Every schema the repository ships must be loaded here. Pinning this + # list behind the current schema version is what kept CI red for two + # milestones once already; a version we cannot validate is reported as + # a failure, so the list going stale is loud rather than silent. validators = {} - for version in (1, 2): + for version in (1, 2, 3): with open(f"schemas/result-v{version}.schema.json") as handle: validators[version] = Draft202012Validator(json.load(handle)) failed = False + exemptions_used = set() for path in sys.argv[1:]: with open(path) as handle: document = json.load(handle) version = document.get("schema_version", 1) + if version == 2 and Path(path).name in PRE_AMENDMENT_6_RESULTS: + exemptions_used.add(Path(path).name) validator = validators.get(version) if validator is None: failed = True @@ -98,5 +104,15 @@ jobs: print(f"{path}: {location}: {error.message}") else: print(f"{path}: ok") + + # The exemption list is hand maintained, so it has to say when it stops + # describing reality. A name that no longer needs exempting is debt that + # would otherwise sit here indefinitely looking load bearing. + stale = sorted(PRE_AMENDMENT_6_RESULTS - exemptions_used) + if stale: + failed = True + print("PRE_AMENDMENT_6_RESULTS is stale; remove these entries:") + for name in stale: + print(f" {name}") sys.exit(1 if failed else 0) PY diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 21dade7..7a046b4 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -59,8 +59,14 @@ record_id = "rec-17" ## Submitting a result file 1. Run the full scenario corpus against one loaded model at a time. -2. Run `willitcall validate results/.json`; it must pass against `schemas/result-v1.schema.json`. -3. Open a pull request adding the file under `results/`, and state the hardware and server version used. +2. Add the exact model selector to `registry/models-v1.json`. Every claimed + identity field needs a provenance reference. If the provenance cannot be + recovered, an explicit unresolved entry is acceptable and will be displayed + as `unresolved`; do not infer identity from the selector or result filename. +3. Run `willitcall validate results/.json`; new runs must pass as schema v3 + against `schemas/result-v3.schema.json`. The CLI continues to accept schema v1 + and v2 files without upgrading them during `annotate` or `rescore`. +4. Open a pull request adding the file under `results/`, and state the hardware and server version used. Never hand-edit a result file. Each scenario record carries an evidence hash, so edited results are not comparable. diff --git a/README.md b/README.md index 25bcdcf..149c3c9 100644 --- a/README.md +++ b/README.md @@ -56,6 +56,12 @@ Check a result file against the published schema: cargo run -p willitcall -- validate willitcall-result.json ``` +New runs use result schema v3 (`schemas/result-v3.schema.json`). The CLI resolves +the model selector through `registry/models-v1.json` and embeds the resolved +model and artifact metadata in the result. Legacy schema v1 and v2 files remain +valid inputs to `validate`, `site`, `annotate`, and `rescore`; editing a legacy +file preserves its original schema version. + `--server` selects a preset (`llamacpp`, `ollama`, `mlx-lm`, `lmstudio`, `vllm`, `custom`). The preset only supplies request defaults; the preset name is recorded in the result file so results stay comparable. @@ -126,11 +132,13 @@ model. So: nearly right that the parser then rejected. The `unparsed_tool_call` failure class exists to mark exactly that case, and the transcript shows the bytes. -Each result records which side of this line its server sits on, in -`server.quirk_flags`: `grammar_constrained_decoding` for llama.cpp, -`unconstrained_post_hoc_parse` for Ollama and mlx-lm. LM Studio and vLLM are -unflagged because their decode path has not been verified here; absence of a -flag means unverified, not unconstrained. +V3 results can record which side of this line their server sits on in +`metadata.server.decode_mode`, while retaining any corresponding +`server.quirk_flags`: `grammar_constrained_decoding` for llama.cpp and +`unconstrained_post_hoc_parse` for Ollama and mlx-lm. When a historical run did +not record the mode, the site consults the cited preset mapping in +`registry/decode-modes-v1.json`; presets absent from that mapping remain +`unknown` rather than being guessed. This was established the hard way. An earlier version of this project published a claim that Ollama discarded valid tool calls. Recovering the discarded bytes @@ -180,10 +188,13 @@ that back it. Seed results so far cover `qwen3` at 0.6b/1.7b/4b/8b, 1. Run the full corpus against your endpoint, one model loaded at a time. Running two models at once produces spurious `error` outcomes from resource contention, not real measurements. -2. Run `willitcall validate` on the output. Current results are schema - version 2 (`schemas/result-v2.schema.json`); version 1 files are still - accepted. -3. Open a pull request adding the result file **and its `evidence/` directory** +2. Ensure the exact model selector has an entry in `registry/models-v1.json`, + with evidence references for every claimed identity field. An unresolved + entry is acceptable when provenance cannot be recovered; the result and site + show that status as `unresolved` rather than inferring identity from a name. +3. Run `willitcall validate` on the schema v3 output. Legacy schema v1 and v2 + files remain accepted, but must not be upgraded by editing them. +4. Open a pull request adding the result file **and its `evidence/` directory** under `results/`, and say what hardware and server version produced it. Every scenario writes a full request/response transcript to diff --git a/crates/wic-core/catalogs/wic-50-v1.json b/crates/wic-core/catalogs/wic-50-v1.json new file mode 100644 index 0000000..f4b8a82 --- /dev/null +++ b/crates/wic-core/catalogs/wic-50-v1.json @@ -0,0 +1,2921 @@ +{ + "id": "wic-50", + "revision": "v1", + "scenario_count": 50, + "sha256": "sha256:042f625c2f898787834524323862c42ab1dffc66fe012d27927944705159f9d8", + "scenarios": [ + { + "id": "multi-turn-calendar-followup", + "category": "multi_turn", + "description": "Use a calendar lookup result in a follow-up event call.", + "rationale": "This asserts using a calendar result in a later create_event call. The date and title are literal in the prompt, and the time is copied verbatim from the prior tool-result JSON.", + "stream": false, + "arguments_match": "exact", + "tools": [ + { + "name": "get_calendar", + "description": "Get calendar availability for a date.", + "parameters": { + "properties": { + "date": { + "type": "string" + } + }, + "required": [ + "date" + ], + "type": "object" + } + }, + { + "name": "create_event", + "description": "Create an event at an exact date and time.", + "parameters": { + "properties": { + "date": { + "type": "string" + }, + "time": { + "type": "string" + }, + "title": { + "type": "string" + } + }, + "required": [ + "title", + "date", + "time" + ], + "type": "object" + } + } + ], + "tool_choice": { + "mode": "auto" + }, + "turns": [ + { + "messages": [ + { + "role": "user", + "content": "Check my calendar on 2026-08-11, then create event Review at the available time." + } + ], + "expected_calls": [ + { + "name": "get_calendar", + "arguments": { + "date": "2026-08-11" + } + } + ] + }, + { + "messages": [ + { + "role": "tool", + "content": "{\"available_time\":\"15:30\"}", + "tool_call_ref": 0 + } + ], + "expected_calls": [ + { + "name": "create_event", + "arguments": { + "date": "2026-08-11", + "time": "15:30", + "title": "Review" + } + } + ] + } + ] + }, + { + "id": "multi-turn-flight-booking", + "category": "multi_turn", + "description": "Use a flight search result in a follow-up booking call.", + "rationale": "This asserts chaining a flight search into a booking. The route and date are literal in the prompt, while the opaque flight id is copied verbatim from the prior tool-result JSON.", + "stream": false, + "arguments_match": "exact", + "tools": [ + { + "name": "find_flight", + "description": "Find a flight for an exact route and date.", + "parameters": { + "properties": { + "date": { + "type": "string" + }, + "destination": { + "type": "string" + }, + "origin": { + "type": "string" + } + }, + "required": [ + "origin", + "destination", + "date" + ], + "type": "object" + } + }, + { + "name": "book_flight", + "description": "Book a flight by exact id.", + "parameters": { + "properties": { + "flight_id": { + "type": "string" + } + }, + "required": [ + "flight_id" + ], + "type": "object" + } + } + ], + "tool_choice": { + "mode": "auto" + }, + "turns": [ + { + "messages": [ + { + "role": "user", + "content": "Find a flight from BOS to DEN on 2026-10-04, then book the returned flight." + } + ], + "expected_calls": [ + { + "name": "find_flight", + "arguments": { + "date": "2026-10-04", + "destination": "DEN", + "origin": "BOS" + } + } + ] + }, + { + "messages": [ + { + "role": "tool", + "content": "{\"flight_id\":\"FL-808\"}", + "tool_call_ref": 0 + } + ], + "expected_calls": [ + { + "name": "book_flight", + "arguments": { + "flight_id": "FL-808" + } + } + ] + } + ] + }, + { + "id": "multi-turn-inventory-order", + "category": "multi_turn", + "description": "Use an inventory result in a follow-up order call.", + "rationale": "This asserts checking availability before placing an order. The opaque SKU and exact quantity are literal in the prompt, and the prior tool-result JSON establishes that stock is available.", + "stream": false, + "arguments_match": "exact", + "tools": [ + { + "name": "check_inventory", + "description": "Check inventory for a SKU.", + "parameters": { + "properties": { + "sku": { + "type": "string" + } + }, + "required": [ + "sku" + ], + "type": "object" + } + }, + { + "name": "create_order", + "description": "Create an order for a SKU and quantity.", + "parameters": { + "properties": { + "quantity": { + "type": "integer" + }, + "sku": { + "type": "string" + } + }, + "required": [ + "sku", + "quantity" + ], + "type": "object" + } + } + ], + "tool_choice": { + "mode": "auto" + }, + "turns": [ + { + "messages": [ + { + "role": "user", + "content": "Check inventory for SKU QZ-4 and order exactly 3 if available." + } + ], + "expected_calls": [ + { + "name": "check_inventory", + "arguments": { + "sku": "QZ-4" + } + } + ] + }, + { + "messages": [ + { + "role": "tool", + "content": "{\"sku\":\"QZ-4\",\"available\":12}", + "tool_call_ref": 0 + } + ], + "expected_calls": [ + { + "name": "create_order", + "arguments": { + "quantity": 3, + "sku": "QZ-4" + } + } + ] + } + ] + }, + { + "id": "multi-turn-parallel-followup", + "category": "multi_turn", + "description": "Feed two parallel results into one follow-up call.", + "rationale": "This asserts combining two parallel lookup results in one follow-up call; the city order is defined by the prompt and schema descriptions, and the values come verbatim from tool-result JSON. Turn 1 expects only lookups because the temperatures are not known yet, so an early comparison would invent them.", + "stream": false, + "arguments_match": "exact", + "tools": [ + { + "name": "get_temperature", + "description": "Get the temperature for a city.", + "parameters": { + "properties": { + "city": { + "type": "string" + } + }, + "required": [ + "city" + ], + "type": "object" + } + }, + { + "name": "compare_temperatures", + "description": "Compare two named city temperatures.", + "parameters": { + "properties": { + "first_city": { + "description": "The city the user mentioned first.", + "type": "string" + }, + "first_value": { + "type": "number" + }, + "second_city": { + "description": "The city the user mentioned second.", + "type": "string" + }, + "second_value": { + "type": "number" + } + }, + "required": [ + "first_city", + "first_value", + "second_city", + "second_value" + ], + "type": "object" + } + } + ], + "tool_choice": { + "mode": "auto" + }, + "turns": [ + { + "messages": [ + { + "role": "user", + "content": "Get temperatures for Boston and Tokyo, then compare the returned values." + } + ], + "expected_calls": [ + { + "name": "get_temperature", + "arguments": { + "city": "Boston" + } + }, + { + "name": "get_temperature", + "arguments": { + "city": "Tokyo" + } + } + ] + }, + { + "messages": [ + { + "role": "tool", + "content": "{\"city\":\"Boston\",\"temperature\":21}", + "tool_call_ref": 0 + }, + { + "role": "tool", + "content": "{\"city\":\"Tokyo\",\"temperature\":28}", + "tool_call_ref": 1 + } + ], + "expected_calls": [ + { + "name": "compare_temperatures", + "arguments": { + "first_city": "Boston", + "first_value": 21, + "second_city": "Tokyo", + "second_value": 28 + } + } + ] + } + ] + }, + { + "id": "multi-turn-route", + "category": "multi_turn", + "description": "Use a geocode result in a follow-up route call.", + "rationale": "This asserts passing exact geocode coordinates into a follow-up route call. Turn 1 ignores arguments because a correct model may geocode Fenway Park, Boston, MA, while turn 2 is pinned verbatim by the tool-result JSON.", + "stream": false, + "arguments_match": "exact", + "tools": [ + { + "name": "geocode", + "description": "Resolve a place to coordinates.", + "parameters": { + "properties": { + "place": { + "type": "string" + } + }, + "required": [ + "place" + ], + "type": "object" + } + }, + { + "name": "get_route", + "description": "Get a route to coordinates.", + "parameters": { + "properties": { + "latitude": { + "type": "number" + }, + "longitude": { + "type": "number" + } + }, + "required": [ + "latitude", + "longitude" + ], + "type": "object" + } + } + ], + "tool_choice": { + "mode": "auto" + }, + "turns": [ + { + "messages": [ + { + "role": "user", + "content": "Find a route to Fenway Park." + } + ], + "expected_calls": [ + { + "name": "geocode", + "arguments": { + "place": "Fenway Park" + }, + "arguments_match": "ignore" + } + ] + }, + { + "messages": [ + { + "role": "tool", + "content": "{\"latitude\":42.3467,\"longitude\":-71.0972}", + "tool_call_ref": 0 + } + ], + "expected_calls": [ + { + "name": "get_route", + "arguments": { + "latitude": 42.3467, + "longitude": -71.0972 + } + } + ] + } + ] + }, + { + "id": "multi-turn-ticket-update", + "category": "multi_turn", + "description": "Use a ticket lookup result in a follow-up update call.", + "rationale": "This asserts looking up a ticket before updating its priority; EXT-55 and high are literal in the prompt, and ticket-902 comes verbatim from the tool-result JSON. Turn 1 expects only the lookup because emitting the update before receiving the internal id would require inventing it.", + "stream": false, + "arguments_match": "exact", + "tools": [ + { + "name": "find_ticket", + "description": "Find a ticket by exact external reference.", + "parameters": { + "properties": { + "external_ref": { + "type": "string" + } + }, + "required": [ + "external_ref" + ], + "type": "object" + } + }, + { + "name": "set_ticket_priority", + "description": "Set ticket priority by internal id.", + "parameters": { + "properties": { + "priority": { + "enum": [ + "low", + "normal", + "high" + ], + "type": "string" + }, + "ticket_id": { + "type": "string" + } + }, + "required": [ + "ticket_id", + "priority" + ], + "type": "object" + } + } + ], + "tool_choice": { + "mode": "auto" + }, + "turns": [ + { + "messages": [ + { + "role": "user", + "content": "Find ticket EXT-55 and set its priority to high." + } + ], + "expected_calls": [ + { + "name": "find_ticket", + "arguments": { + "external_ref": "EXT-55" + } + } + ] + }, + { + "messages": [ + { + "role": "tool", + "content": "{\"ticket_id\":\"ticket-902\"}", + "tool_call_ref": 0 + } + ], + "expected_calls": [ + { + "name": "set_ticket_priority", + "arguments": { + "priority": "high", + "ticket_id": "ticket-902" + } + } + ] + } + ] + }, + { + "id": "multi-turn-two-hop-chain", + "category": "multi_turn", + "description": "Complete a two-hop chain across three tool calls.", + "rationale": "This asserts a three-call dependency chain. The email is literal in the prompt, and each opaque downstream id must be copied verbatim from the immediately preceding tool-result JSON, so no correct alternative value exists.", + "stream": false, + "arguments_match": "exact", + "tools": [ + { + "name": "find_user", + "description": "Find a user id by exact email.", + "parameters": { + "properties": { + "email": { + "type": "string" + } + }, + "required": [ + "email" + ], + "type": "object" + } + }, + { + "name": "find_account", + "description": "Find an account id for a user id.", + "parameters": { + "properties": { + "user_id": { + "type": "string" + } + }, + "required": [ + "user_id" + ], + "type": "object" + } + }, + { + "name": "archive_account", + "description": "Archive an account by exact id.", + "parameters": { + "properties": { + "account_id": { + "type": "string" + } + }, + "required": [ + "account_id" + ], + "type": "object" + } + } + ], + "tool_choice": { + "mode": "auto" + }, + "turns": [ + { + "messages": [ + { + "role": "user", + "content": "Archive the account owned by sam@example.test. Resolve each required id first." + } + ], + "expected_calls": [ + { + "name": "find_user", + "arguments": { + "email": "sam@example.test" + } + } + ] + }, + { + "messages": [ + { + "role": "tool", + "content": "{\"user_id\":\"user-41\"}", + "tool_call_ref": 0 + } + ], + "expected_calls": [ + { + "name": "find_account", + "arguments": { + "user_id": "user-41" + } + } + ] + }, + { + "messages": [ + { + "role": "tool", + "content": "{\"account_id\":\"account-73\"}", + "tool_call_ref": 0 + } + ], + "expected_calls": [ + { + "name": "archive_account", + "arguments": { + "account_id": "account-73" + } + } + ] + } + ] + }, + { + "id": "negative-auto-arithmetic", + "category": "negative_trap", + "description": "Do not call an irrelevant tool for simple arithmetic.", + "rationale": "This asserts abstaining from tool use for arithmetic. The prompt explicitly says to answer without a tool, and the only offered tool is topically irrelevant weather lookup, so every correct response has no tool call.", + "stream": false, + "arguments_match": "exact", + "tools": [ + { + "name": "get_weather", + "description": "Get weather for a city.", + "parameters": { + "properties": { + "city": { + "type": "string" + } + }, + "required": [ + "city" + ], + "type": "object" + } + } + ], + "tool_choice": { + "mode": "auto" + }, + "turns": [ + { + "messages": [ + { + "role": "user", + "content": "What is 19 plus 23? Answer without using a tool." + } + ], + "expected_calls": [] + } + ] + }, + { + "id": "negative-auto-knowledge", + "category": "negative_trap", + "description": "Do not call an irrelevant tool for a general knowledge reply.", + "rationale": "This asserts abstaining from tool use for general knowledge. The prompt explicitly says not to use a tool, and the only offered order lookup cannot answer the ocean question, so every correct response has no call.", + "stream": false, + "arguments_match": "exact", + "tools": [ + { + "name": "lookup_order", + "description": "Look up an order by exact id.", + "parameters": { + "properties": { + "order_id": { + "type": "string" + } + }, + "required": [ + "order_id" + ], + "type": "object" + } + } + ], + "tool_choice": { + "mode": "auto" + }, + "turns": [ + { + "messages": [ + { + "role": "user", + "content": "Name the largest ocean on Earth. Do not use a tool." + } + ], + "expected_calls": [] + } + ] + }, + { + "id": "negative-greeting", + "category": "negative_trap", + "description": "Do not call a tool for a simple greeting.", + "rationale": "This asserts honoring a no-tools request for a greeting. The API request enforces tool_choice none, so tool calls are forbidden and the unscored wording of the short text reply cannot cause a false red.", + "stream": false, + "arguments_match": "exact", + "tools": [ + { + "name": "get_weather", + "description": "Get the current weather for a city.", + "parameters": { + "properties": { + "city": { + "type": "string" + } + }, + "required": [ + "city" + ], + "type": "object" + } + } + ], + "tool_choice": { + "mode": "none" + }, + "turns": [ + { + "messages": [ + { + "role": "user", + "content": "Say hello in one short sentence." + } + ], + "expected_calls": [] + } + ] + }, + { + "id": "negative-invalid-schema", + "category": "negative_trap", + "description": "Refuse a tool call when the requested value is outside its schema.", + "rationale": "This asserts refusing to fabricate a schema-valid call for an unsupported value. Kelvin is absent from the celsius-or-fahrenheit enum, and the prompt explicitly requires plain text without a call when the schema cannot express it.", + "stream": false, + "arguments_match": "exact", + "tools": [ + { + "name": "set_temperature_unit", + "description": "Set one supported temperature unit.", + "parameters": { + "properties": { + "unit": { + "enum": [ + "celsius", + "fahrenheit" + ], + "type": "string" + } + }, + "required": [ + "unit" + ], + "type": "object" + } + } + ], + "tool_choice": { + "mode": "auto" + }, + "turns": [ + { + "messages": [ + { + "role": "user", + "content": "Set the temperature unit to kelvin. If the schema cannot express it, explain that in plain text without calling a tool." + } + ], + "expected_calls": [] + } + ] + }, + { + "id": "negative-long-argument", + "category": "negative_trap", + "description": "Preserve one long ASCII argument exactly.", + "rationale": "This asserts exact transport of the 256-character token supplied verbatim in the prompt. The named tool choice fixes the tool, and the run-path max_tokens default is large enough that a correct model is not truncated.", + "stream": false, + "arguments_match": "exact", + "tools": [ + { + "name": "store_token", + "description": "Store one exact token.", + "parameters": { + "properties": { + "token": { + "type": "string" + } + }, + "required": [ + "token" + ], + "type": "object" + } + } + ], + "tool_choice": { + "mode": "named", + "name": "store_token" + }, + "turns": [ + { + "messages": [ + { + "role": "user", + "content": "Store this exact token: abcdefghijklmnopqrstuvwxyz0123456789-ABCDEFGHIJKLMNOPQRSTUVWXYZ-abcdefghijklmnopqrstuvwxyz0123456789-ABCDEFGHIJKLMNOPQRSTUVWXYZ-abcdefghijklmnopqrstuvwxyz0123456789-ABCDEFGHIJKLMNOPQRSTUVWXYZ-abcdefghijklmnopqrstuvwxyz0123456789-ABCDEFGHIJKLMNOPQRSTUVWXYZ" + } + ], + "expected_calls": [ + { + "name": "store_token", + "arguments": { + "token": "abcdefghijklmnopqrstuvwxyz0123456789-ABCDEFGHIJKLMNOPQRSTUVWXYZ-abcdefghijklmnopqrstuvwxyz0123456789-ABCDEFGHIJKLMNOPQRSTUVWXYZ-abcdefghijklmnopqrstuvwxyz0123456789-ABCDEFGHIJKLMNOPQRSTUVWXYZ-abcdefghijklmnopqrstuvwxyz0123456789-ABCDEFGHIJKLMNOPQRSTUVWXYZ" + } + } + ] + } + ] + }, + { + "id": "negative-plain-text", + "category": "negative_trap", + "description": "Use a plain text reply instead of an unrelated tool.", + "rationale": "This asserts choosing plain text for an explanatory question. The prompt explicitly says not to send email, and the only offered tool sends email rather than answering the question, so a correct response cannot call it.", + "stream": false, + "arguments_match": "exact", + "tools": [ + { + "name": "send_email", + "description": "Send an email to an exact address.", + "parameters": { + "properties": { + "address": { + "type": "string" + }, + "body": { + "type": "string" + } + }, + "required": [ + "address", + "body" + ], + "type": "object" + } + } + ], + "tool_choice": { + "mode": "auto" + }, + "turns": [ + { + "messages": [ + { + "role": "user", + "content": "In one plain text sentence, explain why leaves are green. Do not send email." + } + ], + "expected_calls": [] + } + ] + }, + { + "id": "negative-unicode-argument", + "category": "negative_trap", + "description": "Preserve a non-ASCII city argument.", + "rationale": "This asserts copying the exact city spelling supplied verbatim in the prompt. The scorer compares strings in NFC and uses subset matching for the optional country code, so an NFD-emitting model or one adding that optional field is not failed.", + "stream": false, + "arguments_match": "subset", + "tools": [ + { + "name": "get_weather", + "description": "Get weather for a city with an optional country code.", + "parameters": { + "properties": { + "city": { + "type": "string" + }, + "country_code": { + "type": "string" + } + }, + "required": [ + "city" + ], + "type": "object" + } + } + ], + "tool_choice": { + "mode": "auto" + }, + "turns": [ + { + "messages": [ + { + "role": "user", + "content": "Get the weather for the exact city name München." + } + ], + "expected_calls": [ + { + "name": "get_weather", + "arguments": { + "city": "München" + } + } + ] + } + ] + }, + { + "id": "parallel-city-time", + "category": "parallel_calls", + "description": "Call the time tool for two cities in parallel.", + "rationale": "This asserts emitting both city-time calls in one response, so a model that can only call sequentially genuinely lacks the measured capability. Boston and Tokyo are literal prompt values that pin both arguments.", + "stream": false, + "arguments_match": "exact", + "tools": [ + { + "name": "get_time", + "description": "Get the local time for a city.", + "parameters": { + "properties": { + "city": { + "type": "string" + } + }, + "required": [ + "city" + ], + "type": "object" + } + } + ], + "tool_choice": { + "mode": "required" + }, + "turns": [ + { + "messages": [ + { + "role": "user", + "content": "What time is it in Boston and Tokyo?" + } + ], + "expected_calls": [ + { + "name": "get_time", + "arguments": { + "city": "Boston" + } + }, + { + "name": "get_time", + "arguments": { + "city": "Tokyo" + } + } + ] + } + ] + }, + { + "id": "parallel-different-tools", + "category": "parallel_calls", + "description": "Call two different tools in parallel.", + "rationale": "This asserts emitting weather and time calls together in one response, so sequential-only calling is a true red. The prompt maps the literal Oslo and Cairo values to distinct named tools, leaving no argument ambiguity.", + "stream": false, + "arguments_match": "exact", + "tools": [ + { + "name": "get_weather", + "description": "Get weather for a city.", + "parameters": { + "properties": { + "city": { + "type": "string" + } + }, + "required": [ + "city" + ], + "type": "object" + } + }, + { + "name": "get_time", + "description": "Get local time for a city.", + "parameters": { + "properties": { + "city": { + "type": "string" + } + }, + "required": [ + "city" + ], + "type": "object" + } + } + ], + "tool_choice": { + "mode": "required" + }, + "turns": [ + { + "messages": [ + { + "role": "user", + "content": "Get the weather in Oslo and the local time in Cairo." + } + ], + "expected_calls": [ + { + "name": "get_weather", + "arguments": { + "city": "Oslo" + } + }, + { + "name": "get_time", + "arguments": { + "city": "Cairo" + } + } + ] + } + ] + }, + { + "id": "parallel-three-cities", + "category": "parallel_calls", + "description": "Call the same tool three times in parallel.", + "rationale": "This asserts emitting three weather calls in one response, so a model that calls them only sequentially genuinely lacks the capability. Austin, Dublin, and Seoul are supplied verbatim and the prompt explicitly requests three calls.", + "stream": false, + "arguments_match": "exact", + "tools": [ + { + "name": "get_weather", + "description": "Get weather for a city.", + "parameters": { + "properties": { + "city": { + "type": "string" + } + }, + "required": [ + "city" + ], + "type": "object" + } + } + ], + "tool_choice": { + "mode": "required" + }, + "turns": [ + { + "messages": [ + { + "role": "user", + "content": "Get weather for Austin, Dublin, and Seoul with three calls." + } + ], + "expected_calls": [ + { + "name": "get_weather", + "arguments": { + "city": "Austin" + } + }, + { + "name": "get_weather", + "arguments": { + "city": "Dublin" + } + }, + { + "name": "get_weather", + "arguments": { + "city": "Seoul" + } + } + ] + } + ] + }, + { + "id": "parallel-three-different-tools", + "category": "parallel_calls", + "description": "Call three different tools in parallel.", + "rationale": "This asserts emitting three distinct tool calls in one response, so sequential-only calling is a true red. The prompt explicitly pairs Rome with weather, Nairobi with time, and the literal amount and currency codes with conversion.", + "stream": false, + "arguments_match": "exact", + "tools": [ + { + "name": "get_weather", + "description": "Get weather for a city.", + "parameters": { + "properties": { + "city": { + "type": "string" + } + }, + "required": [ + "city" + ], + "type": "object" + } + }, + { + "name": "get_time", + "description": "Get local time for a city.", + "parameters": { + "properties": { + "city": { + "type": "string" + } + }, + "required": [ + "city" + ], + "type": "object" + } + }, + { + "name": "convert_currency", + "description": "Convert an exact currency amount.", + "parameters": { + "properties": { + "amount": { + "type": "number" + }, + "from": { + "type": "string" + }, + "to": { + "type": "string" + } + }, + "required": [ + "amount", + "from", + "to" + ], + "type": "object" + } + } + ], + "tool_choice": { + "mode": "required" + }, + "turns": [ + { + "messages": [ + { + "role": "user", + "content": "Get weather in Rome, time in Nairobi, and convert 25 USD to EUR." + } + ], + "expected_calls": [ + { + "name": "get_weather", + "arguments": { + "city": "Rome" + } + }, + { + "name": "get_time", + "arguments": { + "city": "Nairobi" + } + }, + { + "name": "convert_currency", + "arguments": { + "amount": 25, + "from": "USD", + "to": "EUR" + } + } + ] + } + ] + }, + { + "id": "parallel-two-calculations", + "category": "parallel_calls", + "description": "Call the same arithmetic tool twice in parallel.", + "rationale": "This asserts emitting both division calls in one response, so sequential-only calling is a true red. Division is non-commutative, and the named dividend and divisor parameters plus the literal prompt wording force operand position.", + "stream": false, + "arguments_match": "exact", + "tools": [ + { + "name": "divide", + "description": "Divide one number by another.", + "parameters": { + "properties": { + "dividend": { + "type": "number" + }, + "divisor": { + "type": "number" + } + }, + "required": [ + "dividend", + "divisor" + ], + "type": "object" + } + } + ], + "tool_choice": { + "mode": "required" + }, + "turns": [ + { + "messages": [ + { + "role": "user", + "content": "Use separate calls to divide 42 by 6 and divide 10 by 4." + } + ], + "expected_calls": [ + { + "name": "divide", + "arguments": { + "dividend": 42, + "divisor": 6 + } + }, + { + "name": "divide", + "arguments": { + "dividend": 10, + "divisor": 4 + } + } + ] + } + ] + }, + { + "id": "parallel-two-multiple-args", + "category": "parallel_calls", + "description": "Call two different tools with multiple arguments.", + "rationale": "This asserts emitting the room and car calls together in one response, so sequential-only calling is a true red. Each room, city, date, and hour is supplied verbatim and mapped to a distinct named tool by the prompt.", + "stream": false, + "arguments_match": "exact", + "tools": [ + { + "name": "book_room", + "description": "Book a room for an exact date and hour.", + "parameters": { + "properties": { + "date": { + "type": "string" + }, + "hour": { + "type": "integer" + }, + "room": { + "type": "string" + } + }, + "required": [ + "room", + "date", + "hour" + ], + "type": "object" + } + }, + { + "name": "reserve_car", + "description": "Reserve a car in a city for an exact date.", + "parameters": { + "properties": { + "city": { + "type": "string" + }, + "date": { + "type": "string" + } + }, + "required": [ + "city", + "date" + ], + "type": "object" + } + } + ], + "tool_choice": { + "mode": "required" + }, + "turns": [ + { + "messages": [ + { + "role": "user", + "content": "Book room Cedar on 2026-09-02 at hour 14 and reserve a car in Boise on 2026-09-03." + } + ], + "expected_calls": [ + { + "name": "book_room", + "arguments": { + "date": "2026-09-02", + "hour": 14, + "room": "Cedar" + } + }, + { + "name": "reserve_car", + "arguments": { + "city": "Boise", + "date": "2026-09-03" + } + } + ] + } + ] + }, + { + "id": "parallel-two-searches", + "category": "parallel_calls", + "description": "Call the same search tool twice with distinct queries.", + "rationale": "This asserts emitting two search calls in one response, so a model that only calls sequentially genuinely lacks the capability. Both exact query strings appear verbatim in quotes and the prompt explicitly requests separate searches.", + "stream": false, + "arguments_match": "exact", + "tools": [ + { + "name": "search_docs", + "description": "Search documents for an exact query.", + "parameters": { + "properties": { + "query": { + "type": "string" + } + }, + "required": [ + "query" + ], + "type": "object" + } + } + ], + "tool_choice": { + "mode": "required" + }, + "turns": [ + { + "messages": [ + { + "role": "user", + "content": "Run separate document searches for 'alpha protocol' and 'beta protocol'." + } + ], + "expected_calls": [ + { + "name": "search_docs", + "arguments": { + "query": "alpha protocol" + } + }, + { + "name": "search_docs", + "arguments": { + "query": "beta protocol" + } + } + ] + } + ] + }, + { + "id": "parallel-weather-and-time", + "category": "parallel_calls", + "description": "Call weather and time tools for the same city.", + "rationale": "This asserts emitting weather and time calls together in one response, so sequential-only calling is a true red even under auto choice. The prompt explicitly requests both tools and supplies Denver verbatim for each argument.", + "stream": false, + "arguments_match": "exact", + "tools": [ + { + "name": "get_weather", + "description": "Get weather for a city.", + "parameters": { + "properties": { + "city": { + "type": "string" + } + }, + "required": [ + "city" + ], + "type": "object" + } + }, + { + "name": "get_time", + "description": "Get local time for a city.", + "parameters": { + "properties": { + "city": { + "type": "string" + } + }, + "required": [ + "city" + ], + "type": "object" + } + } + ], + "tool_choice": { + "mode": "auto" + }, + "turns": [ + { + "messages": [ + { + "role": "user", + "content": "Get both the weather and local time for Denver." + } + ], + "expected_calls": [ + { + "name": "get_weather", + "arguments": { + "city": "Denver" + } + }, + { + "name": "get_time", + "arguments": { + "city": "Denver" + } + } + ] + } + ] + }, + { + "id": "single-array-tags", + "category": "single_call", + "description": "Call one tool with an array of strings.", + "rationale": "This asserts an exact string-array argument for document doc-17. Array comparison is positional, and the prompt supplies every literal tag and says in that order, so the expected sequence is fully pinned.", + "stream": false, + "arguments_match": "exact", + "tools": [ + { + "name": "tag_document", + "description": "Apply tags to a document.", + "parameters": { + "properties": { + "document_id": { + "type": "string" + }, + "tags": { + "items": { + "type": "string" + }, + "type": "array" + } + }, + "required": [ + "document_id", + "tags" + ], + "type": "object" + } + } + ], + "tool_choice": { + "mode": "auto" + }, + "turns": [ + { + "messages": [ + { + "role": "user", + "content": "Tag document doc-17 with exactly alpha, beta, and gamma in that order." + } + ], + "expected_calls": [ + { + "name": "tag_document", + "arguments": { + "document_id": "doc-17", + "tags": [ + "alpha", + "beta", + "gamma" + ] + } + } + ] + } + ] + }, + { + "id": "single-boolean-flag", + "category": "single_call", + "description": "Call one tool with a boolean argument.", + "rationale": "This asserts encoding a disabled state as the boolean false. The opaque feature name audit-log is supplied verbatim, and the imperative Disable pins the boolean without relying on a paraphrasable text answer.", + "stream": false, + "arguments_match": "exact", + "tools": [ + { + "name": "set_feature", + "description": "Enable or disable a feature.", + "parameters": { + "properties": { + "enabled": { + "type": "boolean" + }, + "feature": { + "type": "string" + } + }, + "required": [ + "feature", + "enabled" + ], + "type": "object" + } + } + ], + "tool_choice": { + "mode": "auto" + }, + "turns": [ + { + "messages": [ + { + "role": "user", + "content": "Disable the feature named audit-log." + } + ], + "expected_calls": [ + { + "name": "set_feature", + "arguments": { + "enabled": false, + "feature": "audit-log" + } + } + ] + } + ] + }, + { + "id": "single-date-range", + "category": "single_call", + "description": "Call one tool with multiple required arguments.", + "rationale": "This asserts populating all fields of an inclusive date-range call. Denver and both ISO dates are supplied verbatim, while from and through map unambiguously to the named start_date and end_date parameters.", + "stream": false, + "arguments_match": "exact", + "tools": [ + { + "name": "find_events", + "description": "Find events in an inclusive date range.", + "parameters": { + "properties": { + "city": { + "type": "string" + }, + "end_date": { + "type": "string" + }, + "start_date": { + "type": "string" + } + }, + "required": [ + "city", + "start_date", + "end_date" + ], + "type": "object" + } + } + ], + "tool_choice": { + "mode": "auto" + }, + "turns": [ + { + "messages": [ + { + "role": "user", + "content": "Find events in Denver from 2026-08-03 through 2026-08-05." + } + ], + "expected_calls": [ + { + "name": "find_events", + "arguments": { + "city": "Denver", + "end_date": "2026-08-05", + "start_date": "2026-08-03" + } + } + ] + } + ] + }, + { + "id": "single-decimal-price", + "category": "single_call", + "description": "Call one tool with a decimal number.", + "rationale": "This asserts preserving a decimal numeric argument in a price call. The opaque SKU AX-9 and exact value 19.95 are supplied verbatim in the prompt, so neither field admits a legitimate rewording.", + "stream": false, + "arguments_match": "exact", + "tools": [ + { + "name": "set_price", + "description": "Set the price of a product.", + "parameters": { + "properties": { + "price": { + "type": "number" + }, + "sku": { + "type": "string" + } + }, + "required": [ + "sku", + "price" + ], + "type": "object" + } + } + ], + "tool_choice": { + "mode": "auto" + }, + "turns": [ + { + "messages": [ + { + "role": "user", + "content": "Set the price of SKU AX-9 to exactly 19.95." + } + ], + "expected_calls": [ + { + "name": "set_price", + "arguments": { + "price": 19.95, + "sku": "AX-9" + } + } + ] + } + ] + }, + { + "id": "single-empty-arguments", + "category": "single_call", + "description": "Call a function that accepts no arguments.", + "rationale": "This asserts calling an empty-argument function. The API request enforces required tool use, only get_service_status is offered, and its object schema defines no properties, so the empty argument object is forced.", + "stream": false, + "arguments_match": "exact", + "tools": [ + { + "name": "get_service_status", + "description": "Get the current service status.", + "parameters": { + "type": "object" + } + } + ], + "tool_choice": { + "mode": "required" + }, + "turns": [ + { + "messages": [ + { + "role": "user", + "content": "Get the current service status now." + } + ], + "expected_calls": [ + { + "name": "get_service_status", + "arguments": {} + } + ] + } + ] + }, + { + "id": "single-enum-format", + "category": "single_call", + "description": "Call one tool with an enum argument.", + "rationale": "This asserts selecting a schema enum value alongside an opaque report id. The prompt supplies quarterly-7 and csv verbatim, and csv is the exact supported enum spelling, leaving no valid alternate serialization.", + "stream": false, + "arguments_match": "exact", + "tools": [ + { + "name": "export_report", + "description": "Export a report in a supported format.", + "parameters": { + "properties": { + "format": { + "enum": [ + "csv", + "json", + "pdf" + ], + "type": "string" + }, + "report_id": { + "type": "string" + } + }, + "required": [ + "report_id", + "format" + ], + "type": "object" + } + } + ], + "tool_choice": { + "mode": "auto" + }, + "turns": [ + { + "messages": [ + { + "role": "user", + "content": "Export report quarterly-7 as csv." + } + ], + "expected_calls": [ + { + "name": "export_report", + "arguments": { + "format": "csv", + "report_id": "quarterly-7" + } + } + ] + } + ] + }, + { + "id": "single-integer-limit", + "category": "single_call", + "description": "Call one tool with an integer argument.", + "rationale": "This asserts encoding an exact integer limit in a mailbox call. The prompt names the support mailbox and says exactly 12, so both expected values are literal and no open-ended text is compared.", + "stream": false, + "arguments_match": "exact", + "tools": [ + { + "name": "list_messages", + "description": "List a fixed number of messages.", + "parameters": { + "properties": { + "limit": { + "type": "integer" + }, + "mailbox": { + "type": "string" + } + }, + "required": [ + "mailbox", + "limit" + ], + "type": "object" + } + } + ], + "tool_choice": { + "mode": "auto" + }, + "turns": [ + { + "messages": [ + { + "role": "user", + "content": "List exactly 12 messages from the support mailbox." + } + ], + "expected_calls": [ + { + "name": "list_messages", + "arguments": { + "limit": 12, + "mailbox": "support" + } + } + ] + } + ] + }, + { + "id": "single-nested-options", + "category": "single_call", + "description": "Call one tool with nested option arguments.", + "rationale": "This asserts constructing the required nested options object with numeric and boolean fields. The quoted prompt, dimensions, and enabled transparency state supply every expected value explicitly, so the nested structure follows directly from the schema.", + "stream": false, + "arguments_match": "exact", + "tools": [ + { + "name": "render_image", + "description": "Render an image with explicit options.", + "parameters": { + "properties": { + "options": { + "properties": { + "height": { + "type": "integer" + }, + "transparent": { + "type": "boolean" + }, + "width": { + "type": "integer" + } + }, + "required": [ + "width", + "height", + "transparent" + ], + "type": "object" + }, + "prompt": { + "type": "string" + } + }, + "required": [ + "prompt", + "options" + ], + "type": "object" + } + } + ], + "tool_choice": { + "mode": "auto" + }, + "turns": [ + { + "messages": [ + { + "role": "user", + "content": "Render 'red circle' at width 640 and height 480 with transparency enabled." + } + ], + "expected_calls": [ + { + "name": "render_image", + "arguments": { + "options": { + "height": 480, + "transparent": true, + "width": 640 + }, + "prompt": "red circle" + } + } + ] + } + ] + }, + { + "id": "single-optional-omitted", + "category": "single_call", + "description": "Omit an optional argument that the prompt does not provide.", + "rationale": "This asserts omitting an optional schema field rather than inventing it. The prompt gives the exact search query and explicitly says not to set a result limit, so the exact expected object is complete.", + "stream": false, + "arguments_match": "exact", + "tools": [ + { + "name": "search_catalog", + "description": "Search a catalog with an optional result limit.", + "parameters": { + "properties": { + "limit": { + "type": "integer" + }, + "query": { + "type": "string" + } + }, + "required": [ + "query" + ], + "type": "object" + } + } + ], + "tool_choice": { + "mode": "auto" + }, + "turns": [ + { + "messages": [ + { + "role": "user", + "content": "Search the catalog for brass compass. Do not set a result limit." + } + ], + "expected_calls": [ + { + "name": "search_catalog", + "arguments": { + "query": "brass compass" + } + } + ] + } + ] + }, + { + "id": "single-optional-present", + "category": "single_call", + "description": "Include an optional argument when explicitly provided.", + "rationale": "This asserts including an optional limit when the user supplies it. The prompt gives brass compass verbatim as the search query and says exactly 4 results, which pins both expected arguments.", + "stream": false, + "arguments_match": "exact", + "tools": [ + { + "name": "search_catalog", + "description": "Search a catalog with an optional result limit.", + "parameters": { + "properties": { + "limit": { + "type": "integer" + }, + "query": { + "type": "string" + } + }, + "required": [ + "query" + ], + "type": "object" + } + } + ], + "tool_choice": { + "mode": "auto" + }, + "turns": [ + { + "messages": [ + { + "role": "user", + "content": "Search the catalog for brass compass and return exactly 4 results." + } + ], + "expected_calls": [ + { + "name": "search_catalog", + "arguments": { + "limit": 4, + "query": "brass compass" + } + } + ] + } + ] + }, + { + "id": "single-string-list", + "category": "single_call", + "description": "Call one tool with an ordered array argument.", + "rationale": "This asserts preserving an ordered array of opaque track ids. Array comparison is positional, and the prompt supplies the playlist name and every id verbatim while saying in that order.", + "stream": false, + "arguments_match": "exact", + "tools": [ + { + "name": "create_playlist", + "description": "Create a playlist from ordered track ids.", + "parameters": { + "properties": { + "name": { + "type": "string" + }, + "track_ids": { + "items": { + "type": "string" + }, + "type": "array" + } + }, + "required": [ + "name", + "track_ids" + ], + "type": "object" + } + } + ], + "tool_choice": { + "mode": "auto" + }, + "turns": [ + { + "messages": [ + { + "role": "user", + "content": "Create playlist road-test with tracks t-3, t-1, and t-8 in that order." + } + ], + "expected_calls": [ + { + "name": "create_playlist", + "arguments": { + "name": "road-test", + "track_ids": [ + "t-3", + "t-1", + "t-8" + ] + } + } + ] + } + ] + }, + { + "id": "single-weather", + "category": "single_call", + "description": "Call one weather tool with a city argument.", + "rationale": "This asserts selecting the weather tool with one city argument. Boston appears verbatim in the prompt, the schema requires only city, and no generated natural-language answer is compared.", + "stream": false, + "arguments_match": "exact", + "tools": [ + { + "name": "get_weather", + "description": "Get the current weather for a city.", + "parameters": { + "properties": { + "city": { + "type": "string" + } + }, + "required": [ + "city" + ], + "type": "object" + } + } + ], + "tool_choice": { + "mode": "auto" + }, + "turns": [ + { + "messages": [ + { + "role": "user", + "content": "What is the weather in Boston?" + } + ], + "expected_calls": [ + { + "name": "get_weather", + "arguments": { + "city": "Boston" + } + } + ] + } + ] + }, + { + "id": "single-zero-and-negative", + "category": "single_call", + "description": "Call one tool with zero and a negative number.", + "rationale": "This asserts preserving zero and a negative decimal in distinct numeric fields. The prompt says from -12.5 to 0, and the named minimum and maximum parameters force which literal occupies each position.", + "stream": false, + "arguments_match": "exact", + "tools": [ + { + "name": "set_axis_range", + "description": "Set an axis range.", + "parameters": { + "properties": { + "maximum": { + "type": "number" + }, + "minimum": { + "type": "number" + } + }, + "required": [ + "minimum", + "maximum" + ], + "type": "object" + } + } + ], + "tool_choice": { + "mode": "auto" + }, + "turns": [ + { + "messages": [ + { + "role": "user", + "content": "Set the axis range from -12.5 to 0." + } + ], + "expected_calls": [ + { + "name": "set_axis_range", + "arguments": { + "maximum": 0, + "minimum": -12.5 + } + } + ] + } + ] + }, + { + "id": "streaming-array-arguments", + "category": "streaming", + "description": "Stream a tool call with an array argument.", + "rationale": "This asserts reconstructing an ordered array from streamed arguments. Array comparison is positional, the prompt supplies all opaque record ids verbatim and says in that order, and the API request enforces the named tool.", + "stream": true, + "arguments_match": "exact", + "tools": [ + { + "name": "fetch_records", + "description": "Fetch records by exact ids.", + "parameters": { + "properties": { + "record_ids": { + "items": { + "type": "string" + }, + "type": "array" + } + }, + "required": [ + "record_ids" + ], + "type": "object" + } + } + ], + "tool_choice": { + "mode": "named", + "name": "fetch_records" + }, + "turns": [ + { + "messages": [ + { + "role": "user", + "content": "Fetch record ids r-2, r-5, and r-9 in that order." + } + ], + "expected_calls": [ + { + "name": "fetch_records", + "arguments": { + "record_ids": [ + "r-2", + "r-5", + "r-9" + ] + } + } + ] + } + ] + }, + { + "id": "streaming-boolean-argument", + "category": "streaming", + "description": "Stream a tool call with a boolean argument.", + "rationale": "This asserts reconstructing a boolean argument from a streamed tool call. The prompt supplies billing verbatim and the imperative Enable pins enabled to true, so no paraphrasable response text is scored.", + "stream": true, + "arguments_match": "exact", + "tools": [ + { + "name": "set_maintenance", + "description": "Enable or disable maintenance mode.", + "parameters": { + "properties": { + "enabled": { + "type": "boolean" + }, + "service": { + "type": "string" + } + }, + "required": [ + "service", + "enabled" + ], + "type": "object" + } + } + ], + "tool_choice": { + "mode": "auto" + }, + "turns": [ + { + "messages": [ + { + "role": "user", + "content": "Enable maintenance mode for the billing service." + } + ], + "expected_calls": [ + { + "name": "set_maintenance", + "arguments": { + "enabled": true, + "service": "billing" + } + } + ] + } + ] + }, + { + "id": "streaming-empty-arguments", + "category": "streaming", + "description": "Stream a call to a function with no arguments.", + "rationale": "This asserts reconstructing an empty argument object from a streamed call. The API request requires tool use, get_uptime is the only offered tool, and its schema has no properties, so the expected call is forced.", + "stream": true, + "arguments_match": "exact", + "tools": [ + { + "name": "get_uptime", + "description": "Get service uptime.", + "parameters": { + "type": "object" + } + } + ], + "tool_choice": { + "mode": "required" + }, + "turns": [ + { + "messages": [ + { + "role": "user", + "content": "Get the service uptime." + } + ], + "expected_calls": [ + { + "name": "get_uptime", + "arguments": {} + } + ] + } + ] + }, + { + "id": "streaming-enum-arguments", + "category": "streaming", + "description": "Stream a tool call with an enum argument.", + "rationale": "This asserts reconstructing an enum argument from streamed fragments. The prompt supplies payments and warn verbatim, and warn exactly matches a supported schema enum value, leaving no valid alternative.", + "stream": true, + "arguments_match": "exact", + "tools": [ + { + "name": "set_log_level", + "description": "Set a supported log level.", + "parameters": { + "properties": { + "level": { + "enum": [ + "debug", + "info", + "warn", + "error" + ], + "type": "string" + }, + "service": { + "type": "string" + } + }, + "required": [ + "service", + "level" + ], + "type": "object" + } + } + ], + "tool_choice": { + "mode": "auto" + }, + "turns": [ + { + "messages": [ + { + "role": "user", + "content": "Set the log level for payments to warn." + } + ], + "expected_calls": [ + { + "name": "set_log_level", + "arguments": { + "level": "warn", + "service": "payments" + } + } + ] + } + ] + }, + { + "id": "streaming-nested-arguments", + "category": "streaming", + "description": "Stream a tool call with nested arguments.", + "rationale": "This asserts reconstructing a nested object from streamed argument fragments. The opaque marker name and both signed coordinates are literal prompt values, and the schema uniquely assigns x and y within position.", + "stream": true, + "arguments_match": "exact", + "tools": [ + { + "name": "create_marker", + "description": "Create a named map marker.", + "parameters": { + "properties": { + "name": { + "type": "string" + }, + "position": { + "properties": { + "x": { + "type": "number" + }, + "y": { + "type": "number" + } + }, + "required": [ + "x", + "y" + ], + "type": "object" + } + }, + "required": [ + "name", + "position" + ], + "type": "object" + } + } + ], + "tool_choice": { + "mode": "named", + "name": "create_marker" + }, + "turns": [ + { + "messages": [ + { + "role": "user", + "content": "Create marker origin-shift at x 4.5 and y -3.25." + } + ], + "expected_calls": [ + { + "name": "create_marker", + "arguments": { + "name": "origin-shift", + "position": { + "x": 4.5, + "y": -3.25 + } + } + } + ] + } + ] + }, + { + "id": "streaming-parallel-three", + "category": "streaming", + "description": "Stream three calls to the same tool.", + "rationale": "This asserts reconstructing three parallel calls from one streamed response, so sequential-only calling is a true red. Accra, Kyoto, and Quito are supplied verbatim, and the prompt explicitly requests three calls.", + "stream": true, + "arguments_match": "exact", + "tools": [ + { + "name": "get_time", + "description": "Get local time for a city.", + "parameters": { + "properties": { + "city": { + "type": "string" + } + }, + "required": [ + "city" + ], + "type": "object" + } + } + ], + "tool_choice": { + "mode": "required" + }, + "turns": [ + { + "messages": [ + { + "role": "user", + "content": "Get local time for Accra, Kyoto, and Quito with three calls." + } + ], + "expected_calls": [ + { + "name": "get_time", + "arguments": { + "city": "Accra" + } + }, + { + "name": "get_time", + "arguments": { + "city": "Kyoto" + } + }, + { + "name": "get_time", + "arguments": { + "city": "Quito" + } + } + ] + } + ] + }, + { + "id": "streaming-parallel-two", + "category": "streaming", + "description": "Stream two different tool calls.", + "rationale": "This asserts reconstructing two parallel calls from one streamed response, so sequential-only calling is a true red. The prompt explicitly maps literal Prague to weather and Manila to time, pinning both tool-argument pairs.", + "stream": true, + "arguments_match": "exact", + "tools": [ + { + "name": "get_weather", + "description": "Get weather for a city.", + "parameters": { + "properties": { + "city": { + "type": "string" + } + }, + "required": [ + "city" + ], + "type": "object" + } + }, + { + "name": "get_time", + "description": "Get local time for a city.", + "parameters": { + "properties": { + "city": { + "type": "string" + } + }, + "required": [ + "city" + ], + "type": "object" + } + } + ], + "tool_choice": { + "mode": "required" + }, + "turns": [ + { + "messages": [ + { + "role": "user", + "content": "Get the weather in Prague and the local time in Manila." + } + ], + "expected_calls": [ + { + "name": "get_weather", + "arguments": { + "city": "Prague" + } + }, + { + "name": "get_time", + "arguments": { + "city": "Manila" + } + } + ] + } + ] + }, + { + "id": "streaming-weather", + "category": "streaming", + "description": "Stream one weather tool call.", + "rationale": "This asserts reconstructing a basic tool call from streamed output. The API request enforces the named get_weather tool, and the prompt supplies Seattle verbatim as its only required argument.", + "stream": true, + "arguments_match": "exact", + "tools": [ + { + "name": "get_weather", + "description": "Get the current weather for a city.", + "parameters": { + "properties": { + "city": { + "type": "string" + } + }, + "required": [ + "city" + ], + "type": "object" + } + } + ], + "tool_choice": { + "mode": "named", + "name": "get_weather" + }, + "turns": [ + { + "messages": [ + { + "role": "user", + "content": "Use get_weather for Seattle." + } + ], + "expected_calls": [ + { + "name": "get_weather", + "arguments": { + "city": "Seattle" + } + } + ] + } + ] + }, + { + "id": "tool-choice-auto-call", + "category": "tool_choice_modes", + "description": "Use auto tool choice when a tool is clearly needed.", + "rationale": "This asserts choosing a needed lookup under auto tool choice, where the API leaves call-or-text discretion to the model. The requested temperature requires the only offered tool, and Helsinki is supplied verbatim as its city argument.", + "stream": false, + "arguments_match": "exact", + "tools": [ + { + "name": "get_temperature", + "description": "Get the temperature for a city.", + "parameters": { + "properties": { + "city": { + "type": "string" + } + }, + "required": [ + "city" + ], + "type": "object" + } + } + ], + "tool_choice": { + "mode": "auto" + }, + "turns": [ + { + "messages": [ + { + "role": "user", + "content": "Get the temperature in Helsinki." + } + ], + "expected_calls": [ + { + "name": "get_temperature", + "arguments": { + "city": "Helsinki" + } + } + ] + } + ] + }, + { + "id": "tool-choice-auto-no-call", + "category": "tool_choice_modes", + "description": "Use auto tool choice without calling an irrelevant tool.", + "rationale": "This asserts declining an irrelevant tool under auto choice, where the API leaves call-or-text discretion to the model. The prompt requests literal text and the only tool fetches temperatures, while response wording is not scored.", + "stream": false, + "arguments_match": "exact", + "tools": [ + { + "name": "get_temperature", + "description": "Get the temperature for a city.", + "parameters": { + "properties": { + "city": { + "type": "string" + } + }, + "required": [ + "city" + ], + "type": "object" + } + } + ], + "tool_choice": { + "mode": "auto" + }, + "turns": [ + { + "messages": [ + { + "role": "user", + "content": "Reply with the exact text: ready" + } + ], + "expected_calls": [] + } + ] + }, + { + "id": "tool-choice-named", + "category": "tool_choice_modes", + "description": "Force one named function from two available tools.", + "rationale": "This asserts honoring a named tool choice among two tools. The API request enforces search_contacts rather than leaving tool selection to the model, and the opaque name Ada Stone is supplied verbatim for the required argument.", + "stream": false, + "arguments_match": "exact", + "tools": [ + { + "name": "search_files", + "description": "Search files for an exact phrase.", + "parameters": { + "properties": { + "phrase": { + "type": "string" + } + }, + "required": [ + "phrase" + ], + "type": "object" + } + }, + { + "name": "search_contacts", + "description": "Search contacts for an exact name.", + "parameters": { + "properties": { + "name": { + "type": "string" + } + }, + "required": [ + "name" + ], + "type": "object" + } + } + ], + "tool_choice": { + "mode": "named", + "name": "search_contacts" + }, + "turns": [ + { + "messages": [ + { + "role": "user", + "content": "Search contacts for Ada Stone." + } + ], + "expected_calls": [ + { + "name": "search_contacts", + "arguments": { + "name": "Ada Stone" + } + } + ] + } + ] + }, + { + "id": "tool-choice-named-empty", + "category": "tool_choice_modes", + "description": "Force a named empty-argument function.", + "rationale": "This asserts honoring a named tool choice when another tool is offered. The API request enforces current_region, and its schema has no properties, so the model has no discretion over the tool or arguments.", + "stream": false, + "arguments_match": "exact", + "tools": [ + { + "name": "current_region", + "description": "Get the current deployment region.", + "parameters": { + "type": "object" + } + }, + { + "name": "current_zone", + "description": "Get the current deployment zone.", + "parameters": { + "type": "object" + } + } + ], + "tool_choice": { + "mode": "named", + "name": "current_region" + }, + "turns": [ + { + "messages": [ + { + "role": "user", + "content": "Get the current deployment region." + } + ], + "expected_calls": [ + { + "name": "current_region", + "arguments": {} + } + ] + } + ] + }, + { + "id": "tool-choice-none", + "category": "tool_choice_modes", + "description": "Forbid tool calls for a direct text response.", + "rationale": "This asserts honoring tool_choice none. The API request forbids every offered tool call, so the model has discretion only over its direct text response, whose contents are not compared by the scorer.", + "stream": false, + "arguments_match": "exact", + "tools": [ + { + "name": "get_weather", + "description": "Get weather for a city.", + "parameters": { + "properties": { + "city": { + "type": "string" + } + }, + "required": [ + "city" + ], + "type": "object" + } + } + ], + "tool_choice": { + "mode": "none" + }, + "turns": [ + { + "messages": [ + { + "role": "user", + "content": "Reply with the exact text: no tools" + } + ], + "expected_calls": [] + } + ] + }, + { + "id": "tool-choice-required", + "category": "tool_choice_modes", + "description": "Require one tool call with explicit arguments.", + "rationale": "This asserts honoring required tool choice with an explicit argument. The API request enforces a call, lookup_order is the only offered tool, and opaque order id ORD-204 appears verbatim in the prompt.", + "stream": false, + "arguments_match": "exact", + "tools": [ + { + "name": "lookup_order", + "description": "Look up an order by exact id.", + "parameters": { + "properties": { + "order_id": { + "type": "string" + } + }, + "required": [ + "order_id" + ], + "type": "object" + } + } + ], + "tool_choice": { + "mode": "required" + }, + "turns": [ + { + "messages": [ + { + "role": "user", + "content": "Look up order ORD-204." + } + ], + "expected_calls": [ + { + "name": "lookup_order", + "arguments": { + "order_id": "ORD-204" + } + } + ] + } + ] + }, + { + "id": "tool-choice-required-empty", + "category": "tool_choice_modes", + "description": "Require a call to an empty-argument function.", + "rationale": "This asserts honoring required tool choice with an empty schema. The API request enforces at least one call, current_sequence is the only offered tool, and its schema permits no arguments, removing model discretion.", + "stream": false, + "arguments_match": "exact", + "tools": [ + { + "name": "current_sequence", + "description": "Get the current sequence number.", + "parameters": { + "type": "object" + } + } + ], + "tool_choice": { + "mode": "required" + }, + "turns": [ + { + "messages": [ + { + "role": "user", + "content": "Get the current sequence number." + } + ], + "expected_calls": [ + { + "name": "current_sequence", + "arguments": {} + } + ] + } + ] + } + ] +} diff --git a/crates/wic-core/scenarios/multi-turn-calendar-followup.toml b/crates/wic-core/scenarios/multi-turn-calendar-followup.toml index d48c1b0..1f46a9a 100644 --- a/crates/wic-core/scenarios/multi-turn-calendar-followup.toml +++ b/crates/wic-core/scenarios/multi-turn-calendar-followup.toml @@ -29,6 +29,7 @@ type = "string" mode = "auto" [[turns]] +response_requirement = "tool_calls" [[turns.messages]] role = "user" content = "Check my calendar on 2026-08-11, then create event Review at the available time." @@ -39,6 +40,7 @@ name = "get_calendar" date = "2026-08-11" [[turns]] +response_requirement = "tool_calls" [[turns.messages]] role = "tool" content = '{"available_time":"15:30"}' diff --git a/crates/wic-core/scenarios/multi-turn-flight-booking.toml b/crates/wic-core/scenarios/multi-turn-flight-booking.toml index 300f615..f232ff4 100644 --- a/crates/wic-core/scenarios/multi-turn-flight-booking.toml +++ b/crates/wic-core/scenarios/multi-turn-flight-booking.toml @@ -29,6 +29,7 @@ type = "string" mode = "auto" [[turns]] +response_requirement = "tool_calls" [[turns.messages]] role = "user" content = "Find a flight from BOS to DEN on 2026-10-04, then book the returned flight." @@ -41,6 +42,7 @@ destination = "DEN" date = "2026-10-04" [[turns]] +response_requirement = "tool_calls" [[turns.messages]] role = "tool" content = '{"flight_id":"FL-808"}' diff --git a/crates/wic-core/scenarios/multi-turn-inventory-order.toml b/crates/wic-core/scenarios/multi-turn-inventory-order.toml index 3aca163..6736c7b 100644 --- a/crates/wic-core/scenarios/multi-turn-inventory-order.toml +++ b/crates/wic-core/scenarios/multi-turn-inventory-order.toml @@ -27,6 +27,7 @@ type = "integer" mode = "auto" [[turns]] +response_requirement = "tool_calls" [[turns.messages]] role = "user" content = "Check inventory for SKU QZ-4 and order exactly 3 if available." @@ -37,6 +38,7 @@ name = "check_inventory" sku = "QZ-4" [[turns]] +response_requirement = "tool_calls" [[turns.messages]] role = "tool" content = '{"sku":"QZ-4","available":12}' diff --git a/crates/wic-core/scenarios/multi-turn-parallel-followup.toml b/crates/wic-core/scenarios/multi-turn-parallel-followup.toml index cf3fd2f..58570d0 100644 --- a/crates/wic-core/scenarios/multi-turn-parallel-followup.toml +++ b/crates/wic-core/scenarios/multi-turn-parallel-followup.toml @@ -33,6 +33,7 @@ type = "number" mode = "auto" [[turns]] +response_requirement = "tool_calls" [[turns.messages]] role = "user" content = "Get temperatures for Boston and Tokyo, then compare the returned values." @@ -48,6 +49,7 @@ name = "get_temperature" city = "Tokyo" [[turns]] +response_requirement = "tool_calls" [[turns.messages]] role = "tool" content = '{"city":"Boston","temperature":21}' diff --git a/crates/wic-core/scenarios/multi-turn-route.toml b/crates/wic-core/scenarios/multi-turn-route.toml index 9a34022..7cd63ed 100644 --- a/crates/wic-core/scenarios/multi-turn-route.toml +++ b/crates/wic-core/scenarios/multi-turn-route.toml @@ -32,6 +32,7 @@ type = "number" mode = "auto" [[turns]] +response_requirement = "tool_calls" [[turns.messages]] role = "user" @@ -45,6 +46,7 @@ arguments_match = "ignore" place = "Fenway Park" [[turns]] +response_requirement = "tool_calls" [[turns.messages]] role = "tool" diff --git a/crates/wic-core/scenarios/multi-turn-ticket-update.toml b/crates/wic-core/scenarios/multi-turn-ticket-update.toml index a8b0252..6e07a80 100644 --- a/crates/wic-core/scenarios/multi-turn-ticket-update.toml +++ b/crates/wic-core/scenarios/multi-turn-ticket-update.toml @@ -28,6 +28,7 @@ enum = ["low", "normal", "high"] mode = "auto" [[turns]] +response_requirement = "tool_calls" [[turns.messages]] role = "user" content = "Find ticket EXT-55 and set its priority to high." @@ -38,6 +39,7 @@ name = "find_ticket" external_ref = "EXT-55" [[turns]] +response_requirement = "tool_calls" [[turns.messages]] role = "tool" content = '{"ticket_id":"ticket-902"}' diff --git a/crates/wic-core/scenarios/multi-turn-two-hop-chain.toml b/crates/wic-core/scenarios/multi-turn-two-hop-chain.toml index cae884e..233aaf8 100644 --- a/crates/wic-core/scenarios/multi-turn-two-hop-chain.toml +++ b/crates/wic-core/scenarios/multi-turn-two-hop-chain.toml @@ -34,6 +34,7 @@ type = "string" mode = "auto" [[turns]] +response_requirement = "tool_calls" [[turns.messages]] role = "user" content = "Archive the account owned by sam@example.test. Resolve each required id first." @@ -44,6 +45,7 @@ name = "find_user" email = "sam@example.test" [[turns]] +response_requirement = "tool_calls" [[turns.messages]] role = "tool" content = '{"user_id":"user-41"}' @@ -55,6 +57,7 @@ name = "find_account" user_id = "user-41" [[turns]] +response_requirement = "tool_calls" [[turns.messages]] role = "tool" content = '{"account_id":"account-73"}' diff --git a/crates/wic-core/scenarios/negative-auto-arithmetic.toml b/crates/wic-core/scenarios/negative-auto-arithmetic.toml index 5a9fcbf..e150318 100644 --- a/crates/wic-core/scenarios/negative-auto-arithmetic.toml +++ b/crates/wic-core/scenarios/negative-auto-arithmetic.toml @@ -1,5 +1,6 @@ id = "negative-auto-arithmetic" category = "negative_trap" +facets = ["abstention"] description = "Do not call an irrelevant tool for simple arithmetic." rationale = "This asserts abstaining from tool use for arithmetic. The prompt explicitly says to answer without a tool, and the only offered tool is topically irrelevant weather lookup, so every correct response has no tool call." @@ -18,6 +19,7 @@ type = "string" mode = "auto" [[turns]] +response_requirement = "text_without_tool_calls" [[turns.messages]] role = "user" content = "What is 19 plus 23? Answer without using a tool." diff --git a/crates/wic-core/scenarios/negative-auto-knowledge.toml b/crates/wic-core/scenarios/negative-auto-knowledge.toml index 48965ba..1ae1a65 100644 --- a/crates/wic-core/scenarios/negative-auto-knowledge.toml +++ b/crates/wic-core/scenarios/negative-auto-knowledge.toml @@ -1,5 +1,6 @@ id = "negative-auto-knowledge" category = "negative_trap" +facets = ["abstention"] description = "Do not call an irrelevant tool for a general knowledge reply." rationale = "This asserts abstaining from tool use for general knowledge. The prompt explicitly says not to use a tool, and the only offered order lookup cannot answer the ocean question, so every correct response has no call." @@ -18,6 +19,7 @@ type = "string" mode = "auto" [[turns]] +response_requirement = "text_without_tool_calls" [[turns.messages]] role = "user" content = "Name the largest ocean on Earth. Do not use a tool." diff --git a/crates/wic-core/scenarios/negative-greeting.toml b/crates/wic-core/scenarios/negative-greeting.toml index 99e2848..23402ec 100644 --- a/crates/wic-core/scenarios/negative-greeting.toml +++ b/crates/wic-core/scenarios/negative-greeting.toml @@ -1,5 +1,6 @@ id = "negative-greeting" category = "negative_trap" +facets = ["abstention"] description = "Do not call a tool for a simple greeting." rationale = "This asserts honoring a no-tools request for a greeting. The API request enforces tool_choice none, so tool calls are forbidden and the unscored wording of the short text reply cannot cause a false red." @@ -18,6 +19,7 @@ type = "string" mode = "none" [[turns]] +response_requirement = "text_without_tool_calls" [[turns.messages]] role = "user" diff --git a/crates/wic-core/scenarios/negative-invalid-schema.toml b/crates/wic-core/scenarios/negative-invalid-schema.toml index c1c321f..5a39fa3 100644 --- a/crates/wic-core/scenarios/negative-invalid-schema.toml +++ b/crates/wic-core/scenarios/negative-invalid-schema.toml @@ -1,5 +1,6 @@ id = "negative-invalid-schema" category = "negative_trap" +facets = ["abstention"] description = "Refuse a tool call when the requested value is outside its schema." rationale = "This asserts refusing to fabricate a schema-valid call for an unsupported value. Kelvin is absent from the celsius-or-fahrenheit enum, and the prompt explicitly requires plain text without a call when the schema cannot express it." @@ -19,6 +20,7 @@ enum = ["celsius", "fahrenheit"] mode = "auto" [[turns]] +response_requirement = "text_without_tool_calls" [[turns.messages]] role = "user" content = "Set the temperature unit to kelvin. If the schema cannot express it, explain that in plain text without calling a tool." diff --git a/crates/wic-core/scenarios/negative-long-argument.toml b/crates/wic-core/scenarios/negative-long-argument.toml index db13ad7..d1172d4 100644 --- a/crates/wic-core/scenarios/negative-long-argument.toml +++ b/crates/wic-core/scenarios/negative-long-argument.toml @@ -1,5 +1,6 @@ id = "negative-long-argument" category = "negative_trap" +facets = ["argument_fidelity", "long_context"] description = "Preserve one long ASCII argument exactly." rationale = "This asserts exact transport of the 256-character token supplied verbatim in the prompt. The named tool choice fixes the tool, and the run-path max_tokens default is large enough that a correct model is not truncated." @@ -19,6 +20,7 @@ mode = "named" name = "store_token" [[turns]] +response_requirement = "tool_calls" [[turns.messages]] role = "user" content = "Store this exact token: abcdefghijklmnopqrstuvwxyz0123456789-ABCDEFGHIJKLMNOPQRSTUVWXYZ-abcdefghijklmnopqrstuvwxyz0123456789-ABCDEFGHIJKLMNOPQRSTUVWXYZ-abcdefghijklmnopqrstuvwxyz0123456789-ABCDEFGHIJKLMNOPQRSTUVWXYZ-abcdefghijklmnopqrstuvwxyz0123456789-ABCDEFGHIJKLMNOPQRSTUVWXYZ" diff --git a/crates/wic-core/scenarios/negative-plain-text.toml b/crates/wic-core/scenarios/negative-plain-text.toml index 4b4d1c5..585af4d 100644 --- a/crates/wic-core/scenarios/negative-plain-text.toml +++ b/crates/wic-core/scenarios/negative-plain-text.toml @@ -1,5 +1,6 @@ id = "negative-plain-text" category = "negative_trap" +facets = ["abstention"] description = "Use a plain text reply instead of an unrelated tool." rationale = "This asserts choosing plain text for an explanatory question. The prompt explicitly says not to send email, and the only offered tool sends email rather than answering the question, so a correct response cannot call it." @@ -21,6 +22,7 @@ type = "string" mode = "auto" [[turns]] +response_requirement = "text_without_tool_calls" [[turns.messages]] role = "user" content = "In one plain text sentence, explain why leaves are green. Do not send email." diff --git a/crates/wic-core/scenarios/negative-unicode-argument.toml b/crates/wic-core/scenarios/negative-unicode-argument.toml index 858112e..016ec88 100644 --- a/crates/wic-core/scenarios/negative-unicode-argument.toml +++ b/crates/wic-core/scenarios/negative-unicode-argument.toml @@ -1,5 +1,6 @@ id = "negative-unicode-argument" category = "negative_trap" +facets = ["argument_fidelity", "unicode"] description = "Preserve a non-ASCII city argument." rationale = "This asserts copying the exact city spelling supplied verbatim in the prompt. The scorer compares strings in NFC and uses subset matching for the optional country code, so an NFD-emitting model or one adding that optional field is not failed." arguments_match = "subset" @@ -22,6 +23,7 @@ type = "string" mode = "auto" [[turns]] +response_requirement = "tool_calls" [[turns.messages]] role = "user" content = "Get the weather for the exact city name München." diff --git a/crates/wic-core/scenarios/parallel-city-time.toml b/crates/wic-core/scenarios/parallel-city-time.toml index 493ea17..bbbc654 100644 --- a/crates/wic-core/scenarios/parallel-city-time.toml +++ b/crates/wic-core/scenarios/parallel-city-time.toml @@ -18,6 +18,7 @@ type = "string" mode = "required" [[turns]] +response_requirement = "tool_calls" [[turns.messages]] role = "user" diff --git a/crates/wic-core/scenarios/parallel-different-tools.toml b/crates/wic-core/scenarios/parallel-different-tools.toml index 28ca94d..c6029c9 100644 --- a/crates/wic-core/scenarios/parallel-different-tools.toml +++ b/crates/wic-core/scenarios/parallel-different-tools.toml @@ -29,6 +29,7 @@ type = "string" mode = "required" [[turns]] +response_requirement = "tool_calls" [[turns.messages]] role = "user" content = "Get the weather in Oslo and the local time in Cairo." diff --git a/crates/wic-core/scenarios/parallel-three-cities.toml b/crates/wic-core/scenarios/parallel-three-cities.toml index ce51e5d..9654e11 100644 --- a/crates/wic-core/scenarios/parallel-three-cities.toml +++ b/crates/wic-core/scenarios/parallel-three-cities.toml @@ -18,6 +18,7 @@ type = "string" mode = "required" [[turns]] +response_requirement = "tool_calls" [[turns.messages]] role = "user" content = "Get weather for Austin, Dublin, and Seoul with three calls." diff --git a/crates/wic-core/scenarios/parallel-three-different-tools.toml b/crates/wic-core/scenarios/parallel-three-different-tools.toml index fa1a7be..cd27dc1 100644 --- a/crates/wic-core/scenarios/parallel-three-different-tools.toml +++ b/crates/wic-core/scenarios/parallel-three-different-tools.toml @@ -38,6 +38,7 @@ type = "string" mode = "required" [[turns]] +response_requirement = "tool_calls" [[turns.messages]] role = "user" content = "Get weather in Rome, time in Nairobi, and convert 25 USD to EUR." diff --git a/crates/wic-core/scenarios/parallel-two-calculations.toml b/crates/wic-core/scenarios/parallel-two-calculations.toml index 7bc1d1b..e487a6c 100644 --- a/crates/wic-core/scenarios/parallel-two-calculations.toml +++ b/crates/wic-core/scenarios/parallel-two-calculations.toml @@ -21,6 +21,7 @@ type = "number" mode = "required" [[turns]] +response_requirement = "tool_calls" [[turns.messages]] role = "user" content = "Use separate calls to divide 42 by 6 and divide 10 by 4." diff --git a/crates/wic-core/scenarios/parallel-two-multiple-args.toml b/crates/wic-core/scenarios/parallel-two-multiple-args.toml index 23bd31d..7e18b87 100644 --- a/crates/wic-core/scenarios/parallel-two-multiple-args.toml +++ b/crates/wic-core/scenarios/parallel-two-multiple-args.toml @@ -31,6 +31,7 @@ type = "string" mode = "required" [[turns]] +response_requirement = "tool_calls" [[turns.messages]] role = "user" content = "Book room Cedar on 2026-09-02 at hour 14 and reserve a car in Boise on 2026-09-03." diff --git a/crates/wic-core/scenarios/parallel-two-searches.toml b/crates/wic-core/scenarios/parallel-two-searches.toml index f9b61a9..8e3b509 100644 --- a/crates/wic-core/scenarios/parallel-two-searches.toml +++ b/crates/wic-core/scenarios/parallel-two-searches.toml @@ -18,6 +18,7 @@ type = "string" mode = "required" [[turns]] +response_requirement = "tool_calls" [[turns.messages]] role = "user" content = "Run separate document searches for 'alpha protocol' and 'beta protocol'." diff --git a/crates/wic-core/scenarios/parallel-weather-and-time.toml b/crates/wic-core/scenarios/parallel-weather-and-time.toml index 58310dc..976126d 100644 --- a/crates/wic-core/scenarios/parallel-weather-and-time.toml +++ b/crates/wic-core/scenarios/parallel-weather-and-time.toml @@ -25,6 +25,7 @@ type = "string" mode = "auto" [[turns]] +response_requirement = "tool_calls" [[turns.messages]] role = "user" content = "Get both the weather and local time for Denver." diff --git a/crates/wic-core/scenarios/single-array-tags.toml b/crates/wic-core/scenarios/single-array-tags.toml index 57c5da3..4e836dc 100644 --- a/crates/wic-core/scenarios/single-array-tags.toml +++ b/crates/wic-core/scenarios/single-array-tags.toml @@ -24,6 +24,7 @@ type = "string" mode = "auto" [[turns]] +response_requirement = "tool_calls" [[turns.messages]] role = "user" content = "Tag document doc-17 with exactly alpha, beta, and gamma in that order." diff --git a/crates/wic-core/scenarios/single-boolean-flag.toml b/crates/wic-core/scenarios/single-boolean-flag.toml index 21a3bac..24de662 100644 --- a/crates/wic-core/scenarios/single-boolean-flag.toml +++ b/crates/wic-core/scenarios/single-boolean-flag.toml @@ -21,6 +21,7 @@ type = "boolean" mode = "auto" [[turns]] +response_requirement = "tool_calls" [[turns.messages]] role = "user" content = "Disable the feature named audit-log." diff --git a/crates/wic-core/scenarios/single-date-range.toml b/crates/wic-core/scenarios/single-date-range.toml index 3a2cb46..f09be8f 100644 --- a/crates/wic-core/scenarios/single-date-range.toml +++ b/crates/wic-core/scenarios/single-date-range.toml @@ -24,6 +24,7 @@ type = "string" mode = "auto" [[turns]] +response_requirement = "tool_calls" [[turns.messages]] role = "user" content = "Find events in Denver from 2026-08-03 through 2026-08-05." diff --git a/crates/wic-core/scenarios/single-decimal-price.toml b/crates/wic-core/scenarios/single-decimal-price.toml index a9d2588..e270bed 100644 --- a/crates/wic-core/scenarios/single-decimal-price.toml +++ b/crates/wic-core/scenarios/single-decimal-price.toml @@ -21,6 +21,7 @@ type = "number" mode = "auto" [[turns]] +response_requirement = "tool_calls" [[turns.messages]] role = "user" content = "Set the price of SKU AX-9 to exactly 19.95." diff --git a/crates/wic-core/scenarios/single-empty-arguments.toml b/crates/wic-core/scenarios/single-empty-arguments.toml index fb9d800..85dcce2 100644 --- a/crates/wic-core/scenarios/single-empty-arguments.toml +++ b/crates/wic-core/scenarios/single-empty-arguments.toml @@ -14,6 +14,7 @@ type = "object" mode = "required" [[turns]] +response_requirement = "tool_calls" [[turns.messages]] role = "user" content = "Get the current service status now." diff --git a/crates/wic-core/scenarios/single-enum-format.toml b/crates/wic-core/scenarios/single-enum-format.toml index de33cae..51c7601 100644 --- a/crates/wic-core/scenarios/single-enum-format.toml +++ b/crates/wic-core/scenarios/single-enum-format.toml @@ -22,6 +22,7 @@ enum = ["csv", "json", "pdf"] mode = "auto" [[turns]] +response_requirement = "tool_calls" [[turns.messages]] role = "user" content = "Export report quarterly-7 as csv." diff --git a/crates/wic-core/scenarios/single-integer-limit.toml b/crates/wic-core/scenarios/single-integer-limit.toml index d048fed..f1711e9 100644 --- a/crates/wic-core/scenarios/single-integer-limit.toml +++ b/crates/wic-core/scenarios/single-integer-limit.toml @@ -21,6 +21,7 @@ type = "integer" mode = "auto" [[turns]] +response_requirement = "tool_calls" [[turns.messages]] role = "user" content = "List exactly 12 messages from the support mailbox." diff --git a/crates/wic-core/scenarios/single-nested-options.toml b/crates/wic-core/scenarios/single-nested-options.toml index d42fe2e..d4ec632 100644 --- a/crates/wic-core/scenarios/single-nested-options.toml +++ b/crates/wic-core/scenarios/single-nested-options.toml @@ -31,6 +31,7 @@ type = "boolean" mode = "auto" [[turns]] +response_requirement = "tool_calls" [[turns.messages]] role = "user" content = "Render 'red circle' at width 640 and height 480 with transparency enabled." diff --git a/crates/wic-core/scenarios/single-optional-omitted.toml b/crates/wic-core/scenarios/single-optional-omitted.toml index e4a8faa..890e479 100644 --- a/crates/wic-core/scenarios/single-optional-omitted.toml +++ b/crates/wic-core/scenarios/single-optional-omitted.toml @@ -21,6 +21,7 @@ type = "integer" mode = "auto" [[turns]] +response_requirement = "tool_calls" [[turns.messages]] role = "user" content = "Search the catalog for brass compass. Do not set a result limit." diff --git a/crates/wic-core/scenarios/single-optional-present.toml b/crates/wic-core/scenarios/single-optional-present.toml index 3121585..f8206db 100644 --- a/crates/wic-core/scenarios/single-optional-present.toml +++ b/crates/wic-core/scenarios/single-optional-present.toml @@ -21,6 +21,7 @@ type = "integer" mode = "auto" [[turns]] +response_requirement = "tool_calls" [[turns.messages]] role = "user" content = "Search the catalog for brass compass and return exactly 4 results." diff --git a/crates/wic-core/scenarios/single-string-list.toml b/crates/wic-core/scenarios/single-string-list.toml index 48ce6ec..bcdaa11 100644 --- a/crates/wic-core/scenarios/single-string-list.toml +++ b/crates/wic-core/scenarios/single-string-list.toml @@ -24,6 +24,7 @@ type = "string" mode = "auto" [[turns]] +response_requirement = "tool_calls" [[turns.messages]] role = "user" content = "Create playlist road-test with tracks t-3, t-1, and t-8 in that order." diff --git a/crates/wic-core/scenarios/single-weather.toml b/crates/wic-core/scenarios/single-weather.toml index 7017796..5978c97 100644 --- a/crates/wic-core/scenarios/single-weather.toml +++ b/crates/wic-core/scenarios/single-weather.toml @@ -18,6 +18,7 @@ type = "string" mode = "auto" [[turns]] +response_requirement = "tool_calls" [[turns.messages]] role = "user" diff --git a/crates/wic-core/scenarios/single-zero-and-negative.toml b/crates/wic-core/scenarios/single-zero-and-negative.toml index c5731cf..691a504 100644 --- a/crates/wic-core/scenarios/single-zero-and-negative.toml +++ b/crates/wic-core/scenarios/single-zero-and-negative.toml @@ -21,6 +21,7 @@ type = "number" mode = "auto" [[turns]] +response_requirement = "tool_calls" [[turns.messages]] role = "user" content = "Set the axis range from -12.5 to 0." diff --git a/crates/wic-core/scenarios/streaming-array-arguments.toml b/crates/wic-core/scenarios/streaming-array-arguments.toml index cf7c8ee..43dce87 100644 --- a/crates/wic-core/scenarios/streaming-array-arguments.toml +++ b/crates/wic-core/scenarios/streaming-array-arguments.toml @@ -23,6 +23,7 @@ mode = "named" name = "fetch_records" [[turns]] +response_requirement = "tool_calls" [[turns.messages]] role = "user" content = "Fetch record ids r-2, r-5, and r-9 in that order." diff --git a/crates/wic-core/scenarios/streaming-boolean-argument.toml b/crates/wic-core/scenarios/streaming-boolean-argument.toml index b49fa63..ee188bb 100644 --- a/crates/wic-core/scenarios/streaming-boolean-argument.toml +++ b/crates/wic-core/scenarios/streaming-boolean-argument.toml @@ -22,6 +22,7 @@ type = "boolean" mode = "auto" [[turns]] +response_requirement = "tool_calls" [[turns.messages]] role = "user" content = "Enable maintenance mode for the billing service." diff --git a/crates/wic-core/scenarios/streaming-empty-arguments.toml b/crates/wic-core/scenarios/streaming-empty-arguments.toml index da957f9..2676d7f 100644 --- a/crates/wic-core/scenarios/streaming-empty-arguments.toml +++ b/crates/wic-core/scenarios/streaming-empty-arguments.toml @@ -15,6 +15,7 @@ type = "object" mode = "required" [[turns]] +response_requirement = "tool_calls" [[turns.messages]] role = "user" content = "Get the service uptime." diff --git a/crates/wic-core/scenarios/streaming-enum-arguments.toml b/crates/wic-core/scenarios/streaming-enum-arguments.toml index ff3d13d..3c69513 100644 --- a/crates/wic-core/scenarios/streaming-enum-arguments.toml +++ b/crates/wic-core/scenarios/streaming-enum-arguments.toml @@ -23,6 +23,7 @@ enum = ["debug", "info", "warn", "error"] mode = "auto" [[turns]] +response_requirement = "tool_calls" [[turns.messages]] role = "user" content = "Set the log level for payments to warn." diff --git a/crates/wic-core/scenarios/streaming-nested-arguments.toml b/crates/wic-core/scenarios/streaming-nested-arguments.toml index 6369db4..f4114d8 100644 --- a/crates/wic-core/scenarios/streaming-nested-arguments.toml +++ b/crates/wic-core/scenarios/streaming-nested-arguments.toml @@ -30,6 +30,7 @@ mode = "named" name = "create_marker" [[turns]] +response_requirement = "tool_calls" [[turns.messages]] role = "user" content = "Create marker origin-shift at x 4.5 and y -3.25." diff --git a/crates/wic-core/scenarios/streaming-parallel-three.toml b/crates/wic-core/scenarios/streaming-parallel-three.toml index 84f9bdc..de56bc1 100644 --- a/crates/wic-core/scenarios/streaming-parallel-three.toml +++ b/crates/wic-core/scenarios/streaming-parallel-three.toml @@ -19,6 +19,7 @@ type = "string" mode = "required" [[turns]] +response_requirement = "tool_calls" [[turns.messages]] role = "user" content = "Get local time for Accra, Kyoto, and Quito with three calls." diff --git a/crates/wic-core/scenarios/streaming-parallel-two.toml b/crates/wic-core/scenarios/streaming-parallel-two.toml index a29b8b3..0ea8f47 100644 --- a/crates/wic-core/scenarios/streaming-parallel-two.toml +++ b/crates/wic-core/scenarios/streaming-parallel-two.toml @@ -26,6 +26,7 @@ type = "string" mode = "required" [[turns]] +response_requirement = "tool_calls" [[turns.messages]] role = "user" content = "Get the weather in Prague and the local time in Manila." diff --git a/crates/wic-core/scenarios/streaming-weather.toml b/crates/wic-core/scenarios/streaming-weather.toml index a4aad6e..a1eb4f2 100644 --- a/crates/wic-core/scenarios/streaming-weather.toml +++ b/crates/wic-core/scenarios/streaming-weather.toml @@ -20,6 +20,7 @@ mode = "named" name = "get_weather" [[turns]] +response_requirement = "tool_calls" [[turns.messages]] role = "user" diff --git a/crates/wic-core/scenarios/tool-choice-auto-call.toml b/crates/wic-core/scenarios/tool-choice-auto-call.toml index 5a34e65..0d404d2 100644 --- a/crates/wic-core/scenarios/tool-choice-auto-call.toml +++ b/crates/wic-core/scenarios/tool-choice-auto-call.toml @@ -18,6 +18,7 @@ type = "string" mode = "auto" [[turns]] +response_requirement = "tool_calls" [[turns.messages]] role = "user" content = "Get the temperature in Helsinki." diff --git a/crates/wic-core/scenarios/tool-choice-auto-no-call.toml b/crates/wic-core/scenarios/tool-choice-auto-no-call.toml index 8781459..ff66732 100644 --- a/crates/wic-core/scenarios/tool-choice-auto-no-call.toml +++ b/crates/wic-core/scenarios/tool-choice-auto-no-call.toml @@ -18,6 +18,7 @@ type = "string" mode = "auto" [[turns]] +response_requirement = "text_without_tool_calls" [[turns.messages]] role = "user" content = "Reply with the exact text: ready" diff --git a/crates/wic-core/scenarios/tool-choice-named-empty.toml b/crates/wic-core/scenarios/tool-choice-named-empty.toml index e6a4546..1940068 100644 --- a/crates/wic-core/scenarios/tool-choice-named-empty.toml +++ b/crates/wic-core/scenarios/tool-choice-named-empty.toml @@ -20,6 +20,7 @@ mode = "named" name = "current_region" [[turns]] +response_requirement = "tool_calls" [[turns.messages]] role = "user" content = "Get the current deployment region." diff --git a/crates/wic-core/scenarios/tool-choice-named.toml b/crates/wic-core/scenarios/tool-choice-named.toml index 6716279..c7709ee 100644 --- a/crates/wic-core/scenarios/tool-choice-named.toml +++ b/crates/wic-core/scenarios/tool-choice-named.toml @@ -26,6 +26,7 @@ mode = "named" name = "search_contacts" [[turns]] +response_requirement = "tool_calls" [[turns.messages]] role = "user" content = "Search contacts for Ada Stone." diff --git a/crates/wic-core/scenarios/tool-choice-none.toml b/crates/wic-core/scenarios/tool-choice-none.toml index 1e23e5f..9bd9e8a 100644 --- a/crates/wic-core/scenarios/tool-choice-none.toml +++ b/crates/wic-core/scenarios/tool-choice-none.toml @@ -18,6 +18,7 @@ type = "string" mode = "none" [[turns]] +response_requirement = "text_without_tool_calls" [[turns.messages]] role = "user" content = "Reply with the exact text: no tools" diff --git a/crates/wic-core/scenarios/tool-choice-required-empty.toml b/crates/wic-core/scenarios/tool-choice-required-empty.toml index 354fd28..cd5fcb2 100644 --- a/crates/wic-core/scenarios/tool-choice-required-empty.toml +++ b/crates/wic-core/scenarios/tool-choice-required-empty.toml @@ -14,6 +14,7 @@ type = "object" mode = "required" [[turns]] +response_requirement = "tool_calls" [[turns.messages]] role = "user" content = "Get the current sequence number." diff --git a/crates/wic-core/scenarios/tool-choice-required.toml b/crates/wic-core/scenarios/tool-choice-required.toml index 042f8b8..1f9afe9 100644 --- a/crates/wic-core/scenarios/tool-choice-required.toml +++ b/crates/wic-core/scenarios/tool-choice-required.toml @@ -18,6 +18,7 @@ type = "string" mode = "required" [[turns]] +response_requirement = "tool_calls" [[turns.messages]] role = "user" content = "Look up order ORD-204." diff --git a/crates/wic-core/src/corpus.rs b/crates/wic-core/src/corpus.rs new file mode 100644 index 0000000..e72ceb8 --- /dev/null +++ b/crates/wic-core/src/corpus.rs @@ -0,0 +1,144 @@ +use std::fmt; + +use ring::digest::{digest, SHA256}; +use serde::{Deserialize, Serialize}; +use serde_json::Value; + +use crate::Scenario; + +const CORPUS_FORMAT: &[u8] = b"willitcall-corpus-v1"; +const WIC_50_V1_CATALOG: &[u8] = include_bytes!("../catalogs/wic-50-v1.json"); + +#[derive(Debug, Deserialize, Serialize)] +#[serde(deny_unknown_fields)] +pub struct CorpusCatalog { + pub id: String, + pub revision: String, + pub scenario_count: u32, + pub sha256: String, + pub scenarios: Vec, +} + +#[derive(Debug)] +pub struct CorpusCatalogError(String); + +impl fmt::Display for CorpusCatalogError { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + formatter.write_str(&self.0) + } +} + +impl std::error::Error for CorpusCatalogError {} + +pub fn load_catalog(bytes: &[u8]) -> Result { + let catalog: CorpusCatalog = serde_json::from_slice(bytes) + .map_err(|error| CorpusCatalogError(format!("invalid corpus catalog: {error}")))?; + catalog.verify()?; + Ok(catalog) +} + +pub fn load_frozen_v1_catalog() -> Result { + load_catalog(WIC_50_V1_CATALOG) +} + +impl CorpusCatalog { + pub fn verify(&self) -> Result<(), CorpusCatalogError> { + let scenario_count = u32::try_from(self.scenarios.len()).map_err(|_| { + CorpusCatalogError("catalog scenario count does not fit in u32".to_owned()) + })?; + if self.scenario_count != scenario_count { + return Err(CorpusCatalogError(format!( + "catalog declares {} scenarios but contains {scenario_count}", + self.scenario_count + ))); + } + + let actual = corpus_identity(&self.scenarios); + if self.sha256 != actual { + return Err(CorpusCatalogError(format!( + "catalog hash mismatch: declared {}, computed {actual}", + self.sha256 + ))); + } + Ok(()) + } +} + +pub fn corpus_identity(scenarios: &[Scenario]) -> String { + let serialization = canonical_serialization(scenarios); + let hash = digest(&SHA256, &serialization); + format!("sha256:{}", hex(hash.as_ref())) +} + +fn canonical_serialization(scenarios: &[Scenario]) -> Vec { + let mut ordered = scenarios.iter().collect::>(); + ordered.sort_by(|left, right| left.id.cmp(&right.id)); + + let mut output = Vec::new(); + write_bytes(&mut output, CORPUS_FORMAT); + write_len(&mut output, ordered.len()); + for scenario in ordered { + let value = serde_json::to_value(scenario).expect("scenario should serialize to JSON"); + let mut encoded = Vec::new(); + write_value(&mut encoded, &value); + write_bytes(&mut output, &encoded); + } + output +} + +fn write_value(output: &mut Vec, value: &Value) { + match value { + Value::Null => output.push(0), + Value::Bool(false) => output.push(1), + Value::Bool(true) => output.push(2), + Value::Number(number) => { + output.push(3); + write_bytes(output, number.to_string().as_bytes()); + } + Value::String(string) => { + output.push(4); + write_bytes(output, string.as_bytes()); + } + Value::Array(values) => { + output.push(5); + write_len(output, values.len()); + for value in values { + let mut encoded = Vec::new(); + write_value(&mut encoded, value); + write_bytes(output, &encoded); + } + } + Value::Object(object) => { + output.push(6); + write_len(output, object.len()); + let mut fields = object.iter().collect::>(); + fields.sort_by(|left, right| left.0.cmp(right.0)); + for (name, value) in fields { + write_bytes(output, name.as_bytes()); + let mut encoded = Vec::new(); + write_value(&mut encoded, value); + write_bytes(output, &encoded); + } + } + } +} + +fn write_bytes(output: &mut Vec, bytes: &[u8]) { + write_len(output, bytes.len()); + output.extend_from_slice(bytes); +} + +fn write_len(output: &mut Vec, len: usize) { + let len = u64::try_from(len).expect("corpus serialization length should fit in u64"); + output.extend_from_slice(&len.to_be_bytes()); +} + +fn hex(bytes: &[u8]) -> String { + const DIGITS: &[u8; 16] = b"0123456789abcdef"; + let mut encoded = String::with_capacity(bytes.len() * 2); + for byte in bytes { + encoded.push(DIGITS[(byte >> 4) as usize] as char); + encoded.push(DIGITS[(byte & 0x0f) as usize] as char); + } + encoded +} diff --git a/crates/wic-core/src/lib.rs b/crates/wic-core/src/lib.rs index 333a9b4..dbf7914 100644 --- a/crates/wic-core/src/lib.rs +++ b/crates/wic-core/src/lib.rs @@ -7,6 +7,8 @@ use include_dir::{include_dir, Dir}; use serde::{Deserialize, Serialize}; pub mod client; +pub mod corpus; +pub mod registry; pub mod result; pub mod runner; pub mod score; @@ -18,6 +20,8 @@ static EMBEDDED_SCENARIOS: Dir<'_> = include_dir!("$CARGO_MANIFEST_DIR/scenarios pub struct Scenario { pub id: String, pub category: ScenarioCategory, + #[serde(default, skip_serializing_if = "Vec::is_empty")] + pub facets: Vec, pub description: String, pub rationale: String, #[serde(default)] @@ -29,6 +33,15 @@ pub struct Scenario { pub turns: Vec, } +#[derive(Clone, Copy, Debug, Deserialize, Eq, Hash, PartialEq, Serialize)] +#[serde(rename_all = "snake_case")] +pub enum ScenarioFacet { + Abstention, + ArgumentFidelity, + Unicode, + LongContext, +} + #[derive(Clone, Copy, Debug, Deserialize, Eq, Hash, PartialEq, Serialize)] #[serde(rename_all = "snake_case")] pub enum ScenarioCategory { @@ -77,6 +90,24 @@ pub struct Turn { pub messages: Vec, #[serde(default)] pub expected_calls: Vec, + #[serde(default, skip_serializing_if = "ResponseRequirement::is_either")] + pub response_requirement: ResponseRequirement, +} + +#[derive(Clone, Copy, Debug, Default, Deserialize, Eq, PartialEq, Serialize)] +#[serde(rename_all = "snake_case")] +pub enum ResponseRequirement { + ToolCalls, + TextWithoutToolCalls, + NoToolCalls, + #[default] + Either, +} + +impl ResponseRequirement { + fn is_either(&self) -> bool { + *self == Self::Either + } } #[derive(Clone, Debug, Deserialize, Serialize)] diff --git a/crates/wic-core/src/registry.rs b/crates/wic-core/src/registry.rs new file mode 100644 index 0000000..b283acb --- /dev/null +++ b/crates/wic-core/src/registry.rs @@ -0,0 +1,387 @@ +use std::collections::HashSet; +use std::fmt; +use std::path::Path; + +use include_dir::{include_dir, Dir}; +use serde::{Deserialize, Serialize}; + +use crate::result::{ + ArtifactFormat, ArtifactMetadata, ArtifactSourceKind, IdentityStatus, ModelMetadata, + QuantizationMetadata, +}; + +static EMBEDDED_EVIDENCE: Dir<'_> = include_dir!("$CARGO_MANIFEST_DIR/../../registry/evidence"); + +#[derive(Clone, Debug, Deserialize, Serialize)] +#[serde(deny_unknown_fields)] +pub struct ModelRegistry { + pub schema_version: u32, + pub entries: Vec, +} + +#[derive(Clone, Debug, Deserialize, Serialize)] +#[serde(deny_unknown_fields)] +pub struct ModelRegistryEntry { + pub selectors: Vec, + pub display_name: Provenanced, + pub family_id: Option>, + pub canonical_id: Option, + pub parameter_count_b: Option>, + pub identity_status: IdentityStatus, + pub artifact: RegistryArtifact, +} + +#[derive(Clone, Debug, Deserialize, Serialize)] +#[serde(deny_unknown_fields)] +pub struct RegistryArtifact { + pub source_kind: Provenanced, + pub source_id: Option>, + pub revision: Option, + pub sha256: Option>, + pub format: Provenanced, + pub quantization: Option>, +} + +#[derive(Clone, Debug, Deserialize, Serialize)] +#[serde(deny_unknown_fields)] +pub struct Provenanced { + pub value: T, + pub provenance_ref: String, +} + +#[derive(Clone, Debug, Deserialize, Serialize)] +#[serde(deny_unknown_fields)] +pub struct CanonicalIdClaim { + pub value: String, + pub provenance_ref: String, + pub corroboration: CanonicalCorroboration, +} + +#[derive(Clone, Copy, Debug, Deserialize, Eq, PartialEq, Serialize)] +#[serde(rename_all = "snake_case")] +pub enum CanonicalCorroboration { + Declared, + Corroborated, +} + +#[derive(Clone, Debug, Deserialize, Serialize)] +#[serde(deny_unknown_fields)] +pub struct RevisionClaim { + pub value: String, + pub provenance_ref: String, + pub revision_scope: RevisionScope, +} + +#[derive(Clone, Copy, Debug, Deserialize, Eq, PartialEq, Serialize)] +#[serde(rename_all = "snake_case")] +pub enum RevisionScope { + MeasuredArtifact, + RepositoryHeadAtCapture, +} + +#[derive(Debug)] +pub struct RegistryError(String); + +impl fmt::Display for RegistryError { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + formatter.write_str(&self.0) + } +} + +impl std::error::Error for RegistryError {} + +impl ModelRegistry { + pub fn from_json(bytes: &[u8]) -> Result { + let registry: Self = serde_json::from_slice(bytes) + .map_err(|error| RegistryError(format!("invalid model registry: {error}")))?; + registry.validate()?; + Ok(registry) + } + + pub fn resolve(&self, raw_endpoint_selector: &str) -> ModelMetadata { + let endpoint_id = sanitize_endpoint_id(raw_endpoint_selector); + let entry = self.entries.iter().find(|entry| { + entry.selectors.iter().any(|selector| { + selector == raw_endpoint_selector || selector == endpoint_id.as_str() + }) + }); + + let Some(entry) = entry else { + return unresolved_model(endpoint_id); + }; + + ModelMetadata { + display_name: entry.display_name.value.clone(), + family_id: entry.family_id.as_ref().map(|field| field.value.clone()), + canonical_id: entry.canonical_id.as_ref().map(|field| field.value.clone()), + parameter_count_b: entry.parameter_count_b.as_ref().map(|field| field.value), + endpoint_id, + identity_status: identity_status(entry), + artifact: ArtifactMetadata { + source_kind: entry.artifact.source_kind.value, + source_id: entry + .artifact + .source_id + .as_ref() + .map(|field| field.value.clone()), + revision: entry + .artifact + .revision + .as_ref() + .map(|field| field.value.clone()), + sha256: entry + .artifact + .sha256 + .as_ref() + .map(|field| field.value.clone()), + format: entry.artifact.format.value, + quantization: entry + .artifact + .quantization + .as_ref() + .map(|field| field.value.clone()), + }, + } + } + + fn validate(&self) -> Result<(), RegistryError> { + if self.schema_version != 1 { + return Err(RegistryError(format!( + "unsupported model registry schema_version {}; expected 1", + self.schema_version + ))); + } + + let mut selectors = HashSet::new(); + for (index, entry) in self.entries.iter().enumerate() { + if entry.selectors.is_empty() { + return Err(RegistryError(format!( + "model registry entry {index} has no selectors" + ))); + } + for selector in &entry.selectors { + if !is_safe_registry_selector(selector) { + return Err(RegistryError(format!( + "model registry entry {index} has unsafe selector {selector:?}" + ))); + } + if !selectors.insert(selector) { + return Err(RegistryError(format!( + "duplicate model registry selector {selector:?}" + ))); + } + } + + validate_nonempty(index, "display_name", &entry.display_name.value)?; + validate_provenance(index, "display_name", &entry.display_name.provenance_ref)?; + validate_optional_provenanced(index, "family_id", entry.family_id.as_ref())?; + if let Some(canonical_id) = &entry.canonical_id { + validate_nonempty(index, "canonical_id", &canonical_id.value)?; + validate_provenance(index, "canonical_id", &canonical_id.provenance_ref)?; + } + if let Some(parameter_count_b) = &entry.parameter_count_b { + if !parameter_count_b.value.is_finite() || parameter_count_b.value < 0.0 { + return Err(RegistryError(format!( + "model registry entry {index} parameter_count_b must be finite and non-negative" + ))); + } + validate_provenance( + index, + "parameter_count_b", + ¶meter_count_b.provenance_ref, + )?; + } + + validate_provenance( + index, + "artifact.source_kind", + &entry.artifact.source_kind.provenance_ref, + )?; + validate_optional_provenanced( + index, + "artifact.source_id", + entry.artifact.source_id.as_ref(), + )?; + if let Some(revision) = &entry.artifact.revision { + validate_nonempty(index, "artifact.revision", &revision.value)?; + validate_provenance(index, "artifact.revision", &revision.provenance_ref)?; + } + if let Some(sha256) = &entry.artifact.sha256 { + if !is_sha256(&sha256.value) { + return Err(RegistryError(format!( + "model registry entry {index} artifact.sha256 must be a sha256: digest" + ))); + } + validate_provenance(index, "artifact.sha256", &sha256.provenance_ref)?; + } + validate_provenance( + index, + "artifact.format", + &entry.artifact.format.provenance_ref, + )?; + if let Some(quantization) = &entry.artifact.quantization { + validate_nonempty( + index, + "artifact.quantization.label", + &quantization.value.label, + )?; + validate_provenance(index, "artifact.quantization", &quantization.provenance_ref)?; + } + + let decided_status = identity_status(entry); + if entry.identity_status != decided_status { + return Err(RegistryError(format!( + "model registry entry {index} declares identity_status {:?}, but its evidence requires {:?}", + entry.identity_status, decided_status + ))); + } + } + Ok(()) + } +} + +pub fn sanitize_endpoint_id(raw_endpoint_selector: &str) -> String { + if raw_endpoint_selector.is_empty() || raw_endpoint_selector.chars().any(char::is_control) { + return "unresolved-endpoint".to_owned(); + } + + if is_local_absolute_path(raw_endpoint_selector) { + let normalized = raw_endpoint_selector.replace('\\', "/"); + if normalized.ends_with('/') { + return "unresolved-endpoint".to_owned(); + } + return normalized + .rsplit('/') + .next() + .filter(|name| !name.is_empty() && !matches!(*name, "." | "..")) + .unwrap_or("unresolved-endpoint") + .to_owned(); + } + + raw_endpoint_selector.to_owned() +} + +fn identity_status(entry: &ModelRegistryEntry) -> IdentityStatus { + let Some(canonical_id) = &entry.canonical_id else { + return IdentityStatus::Unresolved; + }; + if entry.artifact.source_kind.value == ArtifactSourceKind::LocalFile + && entry.artifact.sha256.is_none() + { + return IdentityStatus::Unresolved; + } + + let has_immutable_artifact = entry.artifact.sha256.is_some() + || entry.artifact.revision.as_ref().is_some_and(|revision| { + revision.revision_scope == RevisionScope::MeasuredArtifact + && entry.artifact.source_kind.value != ArtifactSourceKind::LocalFile + }); + if canonical_id.corroboration == CanonicalCorroboration::Corroborated && has_immutable_artifact + { + IdentityStatus::Verified + } else { + IdentityStatus::Declared + } +} + +fn unresolved_model(endpoint_id: String) -> ModelMetadata { + ModelMetadata { + display_name: "Unresolved model".to_owned(), + family_id: None, + canonical_id: None, + parameter_count_b: None, + endpoint_id, + identity_status: IdentityStatus::Unresolved, + artifact: ArtifactMetadata { + source_kind: ArtifactSourceKind::Other, + source_id: None, + revision: None, + sha256: None, + format: ArtifactFormat::Unknown, + quantization: None, + }, + } +} + +fn validate_optional_provenanced( + index: usize, + field_name: &str, + field: Option<&Provenanced>, +) -> Result<(), RegistryError> { + if let Some(field) = field { + validate_nonempty(index, field_name, &field.value)?; + validate_provenance(index, field_name, &field.provenance_ref)?; + } + Ok(()) +} + +fn validate_nonempty(index: usize, field_name: &str, value: &str) -> Result<(), RegistryError> { + if value.trim().is_empty() { + return Err(RegistryError(format!( + "model registry entry {index} {field_name} must not be empty" + ))); + } + Ok(()) +} + +fn validate_provenance( + index: usize, + field_name: &str, + provenance_ref: &str, +) -> Result<(), RegistryError> { + let file_ref = provenance_ref + .split_once('#') + .map_or(provenance_ref, |part| part.0); + let relative = file_ref + .strip_prefix("registry/evidence/") + .filter(|relative| !relative.is_empty()) + .ok_or_else(|| { + RegistryError(format!( + "model registry entry {index} {field_name} provenance_ref must point into registry/evidence" + )) + })?; + if Path::new(relative).components().count() != 1 + || !relative.ends_with(".json") + || EMBEDDED_EVIDENCE.get_file(relative).is_none() + { + return Err(RegistryError(format!( + "model registry entry {index} {field_name} provenance_ref does not name checked-in evidence" + ))); + } + if let Some((_, fragment)) = provenance_ref.split_once('#') { + if !fragment.starts_with('/') || fragment.len() == 1 { + return Err(RegistryError(format!( + "model registry entry {index} {field_name} provenance_ref has an invalid JSON pointer" + ))); + } + } + Ok(()) +} + +fn is_safe_registry_selector(selector: &str) -> bool { + !selector.is_empty() + && !selector.chars().any(char::is_control) + && !is_local_absolute_path(selector) +} + +fn is_local_absolute_path(selector: &str) -> bool { + let bytes = selector.as_bytes(); + selector.starts_with('/') + || selector.starts_with("~/") + || selector.starts_with("~\\") + || selector.starts_with("\\\\") + || selector.starts_with("file://") + || (bytes.len() >= 3 + && bytes[0].is_ascii_alphabetic() + && bytes[1] == b':' + && matches!(bytes[2], b'/' | b'\\')) +} + +fn is_sha256(value: &str) -> bool { + value.strip_prefix("sha256:").is_some_and(|digest| { + digest.len() == 64 + && digest + .bytes() + .all(|byte| byte.is_ascii_hexdigit() && !byte.is_ascii_uppercase()) + }) +} diff --git a/crates/wic-core/src/result.rs b/crates/wic-core/src/result.rs index 9138c2c..39fe929 100644 --- a/crates/wic-core/src/result.rs +++ b/crates/wic-core/src/result.rs @@ -3,6 +3,7 @@ use std::fs::File; use std::io::{self, Write}; use std::path::Path; +use ring::digest::{digest, SHA256}; use serde::{Deserialize, Serialize}; use serde_json::Value; @@ -12,13 +13,165 @@ pub const RESULT_SCHEMA_VERSION: u32 = 2; #[derive(Clone, Debug, Deserialize, Serialize)] #[serde(deny_unknown_fields)] -pub struct RunResult { +pub struct RunResultV1V2 { pub schema_version: u32, pub metadata: RunMetadata, pub scenarios: Vec, pub totals: Totals, } +pub type RunResult = RunResultV1V2; + +#[derive(Clone, Debug, Deserialize, Serialize)] +#[serde(deny_unknown_fields)] +pub struct RunResultV3 { + pub schema_version: u32, + pub metadata: RunMetadataV3, + pub scenarios: Vec, + pub totals: Totals, +} + +#[derive(Clone, Debug, Deserialize, Serialize)] +#[serde(deny_unknown_fields)] +pub struct RunMetadataV3 { + pub run_id: String, + pub timestamp: String, + pub willitcall_version: String, + pub endpoint: String, + pub model: ModelMetadata, + pub corpus: CorpusMetadata, + pub server: ServerMetadataV3, + pub environment: EnvironmentMetadataV3, + pub sampling: SamplingParams, + pub replication: Option, + pub arm_fingerprint: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub preflight_override: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub preflight_ignored_ports: Option>, +} + +#[derive(Clone, Debug, Deserialize, Serialize)] +#[serde(deny_unknown_fields)] +pub struct ModelMetadata { + pub display_name: String, + pub family_id: Option, + pub canonical_id: Option, + pub parameter_count_b: Option, + pub endpoint_id: String, + pub identity_status: IdentityStatus, + pub artifact: ArtifactMetadata, +} + +#[derive(Clone, Copy, Debug, Deserialize, Eq, PartialEq, Serialize)] +#[serde(rename_all = "snake_case")] +pub enum IdentityStatus { + Verified, + Declared, + Unresolved, +} + +#[derive(Clone, Debug, Deserialize, Serialize)] +#[serde(deny_unknown_fields)] +pub struct ArtifactMetadata { + pub source_kind: ArtifactSourceKind, + pub source_id: Option, + pub revision: Option, + pub sha256: Option, + pub format: ArtifactFormat, + pub quantization: Option, +} + +#[derive(Clone, Copy, Debug, Deserialize, Eq, PartialEq, Serialize)] +#[serde(rename_all = "snake_case")] +pub enum ArtifactSourceKind { + Huggingface, + Ollama, + LocalFile, + Other, +} + +#[derive(Clone, Copy, Debug, Deserialize, Eq, PartialEq, Serialize)] +#[serde(rename_all = "snake_case")] +pub enum ArtifactFormat { + Gguf, + Mlx, + Safetensors, + OllamaBlob, + Unknown, +} + +#[derive(Clone, Debug, Deserialize, Serialize)] +#[serde(deny_unknown_fields)] +pub struct QuantizationMetadata { + pub label: String, + pub scheme: Option, + pub bits: Option, +} + +#[derive(Clone, Debug, Deserialize, Serialize)] +#[serde(deny_unknown_fields)] +pub struct CorpusMetadata { + pub id: String, + pub revision: String, + pub sha256: String, + pub scenario_count: u32, + pub scoring_version: String, +} + +#[derive(Clone, Debug, Deserialize, Serialize)] +#[serde(deny_unknown_fields)] +pub struct ServerMetadataV3 { + pub preset_name: String, + pub reported_version: Option, + pub quirk_flags: Vec, + pub decode_mode: DecodeMode, + pub chat_template: Option, + pub launch_config_sha256: Option, +} + +#[derive(Clone, Copy, Debug, Deserialize, Eq, PartialEq, Serialize)] +#[serde(rename_all = "snake_case")] +pub enum DecodeMode { + GrammarConstrained, + UnconstrainedPostHoc, + Unknown, +} + +#[derive(Clone, Debug, Deserialize, Serialize)] +#[serde(deny_unknown_fields)] +pub struct ChatTemplateMetadata { + pub id: Option, + pub sha256: Option, +} + +#[derive(Clone, Debug, Deserialize, Serialize)] +#[serde(deny_unknown_fields)] +pub struct EnvironmentMetadataV3 { + pub display_label: String, + pub os_name: Option, + pub os_version: Option, + pub architecture: Option, + pub accelerator: Option, + pub memory_bytes: Option, +} + +#[derive(Clone, Debug, Deserialize, Serialize)] +#[serde(deny_unknown_fields)] +pub struct ReplicationMetadata { + pub study_id: String, + pub arm_id: String, + pub run_index: u32, + pub mode: ReplicationMode, +} + +#[derive(Clone, Copy, Debug, Deserialize, Eq, PartialEq, Serialize)] +#[serde(rename_all = "snake_case")] +pub enum ReplicationMode { + GreedyReproducibility, + SeedVariedVariance, +} + #[derive(Clone, Debug, Deserialize, Serialize)] #[serde(deny_unknown_fields)] pub struct RunMetadata { @@ -87,6 +240,294 @@ pub struct ScenarioOutcome { pub retried: bool, } +#[derive(Clone, Debug, Deserialize, Serialize)] +#[serde(deny_unknown_fields)] +pub struct ScenarioOutcomeV3 { + pub id: String, + pub category: ScenarioCategory, + pub status: Status, + pub failure_reason: Option, + pub failure: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub failure_class: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub cause: Option, + pub evidence_hash: Option, + pub evidence_path: Option, + pub retried: bool, +} + +#[derive(Clone, Debug, Deserialize, Serialize)] +#[serde(deny_unknown_fields)] +pub struct ScenarioFailure { + pub stage: String, + pub code: String, + pub http_status: Option, + pub failed_turn_index: Option, +} + +#[derive(Clone, Debug)] +pub struct Measurement { + pub schema_version: u32, + pub metadata: MeasurementMetadata, + pub scenarios: Vec, + pub totals: Totals, +} + +#[derive(Clone, Debug)] +pub struct MeasurementMetadata { + pub run_id: String, + pub timestamp: String, + pub willitcall_version: String, + pub endpoint: String, + pub model: ModelMetadata, + pub corpus: Option, + pub server: MeasurementServerMetadata, + pub environment: Option, + pub sampling: SamplingParams, + pub replication: Option, + pub arm_fingerprint: Option, + pub preflight_override: Option, + pub preflight_ignored_ports: Option>, +} + +#[derive(Clone, Debug)] +pub struct MeasurementServerMetadata { + pub preset_name: String, + pub reported_version: Option, + pub quirk_flags: Vec, + pub decode_mode: DecodeMode, + pub chat_template: Option, + pub launch_config_sha256: Option, +} + +#[derive(Clone, Debug)] +pub struct MeasurementScenarioOutcome { + pub id: String, + pub category: ScenarioCategory, + pub status: Status, + pub failure_reason: Option, + pub failure: Option, + pub failure_class: Option, + pub cause: Option, + pub evidence_hash: Option, + pub evidence_path: Option, + pub retried: bool, +} + +impl Measurement { + pub fn cross_model_key(&self) -> Option<&str> { + if self.metadata.model.identity_status == IdentityStatus::Unresolved { + None + } else { + self.metadata.model.canonical_id.as_deref() + } + } +} + +pub fn arm_fingerprint( + corpus: &CorpusMetadata, + model: &ModelMetadata, + server: &MeasurementServerMetadata, + environment: Option<&EnvironmentMetadataV3>, + sampling: &SamplingParams, +) -> Option { + let artifact = &model.artifact; + if model.identity_status == IdentityStatus::Unresolved + || artifact.source_kind == ArtifactSourceKind::Other + || artifact.source_id.is_none() + || (artifact.revision.is_none() && artifact.sha256.is_none()) + || artifact.format == ArtifactFormat::Unknown + { + return None; + } + + let reported_version = server.reported_version.as_ref()?; + let launch_config_sha256 = server.launch_config_sha256.as_ref()?; + let chat_template = server.chat_template.as_ref()?; + let chat_template_id = chat_template.id.as_ref()?; + let chat_template_sha256 = chat_template.sha256.as_ref()?; + if server.decode_mode == DecodeMode::Unknown { + return None; + } + + let environment = environment?; + environment.os_name.as_ref()?; + environment.os_version.as_ref()?; + environment.architecture.as_ref()?; + environment.accelerator.as_ref()?; + environment.memory_bytes?; + + let temperature = sampling.temperature.filter(|value| value.is_finite())?; + let top_p = sampling.top_p.filter(|value| value.is_finite())?; + let max_tokens = sampling.max_tokens?; + let input = serde_json::to_vec(&serde_json::json!({ + "version": 1, + "corpus": corpus, + "artifact": artifact, + "server": { + "preset_name": server.preset_name, + "reported_version": reported_version, + "launch_config_sha256": launch_config_sha256, + }, + "chat_template": { + "id": chat_template_id, + "sha256": chat_template_sha256, + }, + "decode_mode": server.decode_mode, + "environment": environment, + "sampling": { + "temperature": temperature, + "top_p": top_p, + "max_tokens": max_tokens, + }, + })) + .ok()?; + let hash = digest(&SHA256, &input); + Some(format!("v1:sha256:{}", hex(hash.as_ref()))) +} + +impl From for Measurement { + fn from(result: RunResultV1V2) -> Self { + let metadata = result.metadata; + let quantization = metadata.declared_quant.map(|label| QuantizationMetadata { + label, + scheme: None, + bits: None, + }); + let environment = metadata + .environment + .map(|environment| EnvironmentMetadataV3 { + display_label: format!( + "{}; {}", + environment.host_hardware_class, environment.host_os + ), + os_name: None, + os_version: None, + architecture: None, + accelerator: None, + memory_bytes: None, + }); + Self { + schema_version: result.schema_version, + metadata: MeasurementMetadata { + run_id: metadata.run_id, + timestamp: metadata.timestamp, + willitcall_version: metadata.willitcall_version, + endpoint: metadata.endpoint, + model: ModelMetadata { + display_name: metadata.model_id.clone(), + family_id: None, + canonical_id: None, + parameter_count_b: None, + endpoint_id: metadata.model_id, + identity_status: IdentityStatus::Unresolved, + artifact: ArtifactMetadata { + source_kind: ArtifactSourceKind::Other, + source_id: None, + revision: None, + sha256: None, + format: ArtifactFormat::Unknown, + quantization, + }, + }, + corpus: None, + server: MeasurementServerMetadata { + preset_name: metadata.server.preset_name, + reported_version: metadata.server.reported_version, + quirk_flags: metadata.server.quirk_flags, + decode_mode: DecodeMode::Unknown, + chat_template: None, + launch_config_sha256: None, + }, + environment, + sampling: metadata.sampling, + replication: None, + arm_fingerprint: None, + preflight_override: metadata.preflight_override, + preflight_ignored_ports: metadata.preflight_ignored_ports, + }, + scenarios: result + .scenarios + .into_iter() + .map(MeasurementScenarioOutcome::from) + .collect(), + totals: result.totals, + } + } +} + +impl From for Measurement { + fn from(result: RunResultV3) -> Self { + let metadata = result.metadata; + Self { + schema_version: result.schema_version, + metadata: MeasurementMetadata { + run_id: metadata.run_id, + timestamp: metadata.timestamp, + willitcall_version: metadata.willitcall_version, + endpoint: metadata.endpoint, + model: metadata.model, + corpus: Some(metadata.corpus), + server: MeasurementServerMetadata { + preset_name: metadata.server.preset_name, + reported_version: metadata.server.reported_version, + quirk_flags: metadata.server.quirk_flags, + decode_mode: metadata.server.decode_mode, + chat_template: metadata.server.chat_template, + launch_config_sha256: metadata.server.launch_config_sha256, + }, + environment: Some(metadata.environment), + sampling: metadata.sampling, + replication: metadata.replication, + arm_fingerprint: metadata.arm_fingerprint, + preflight_override: metadata.preflight_override, + preflight_ignored_ports: metadata.preflight_ignored_ports, + }, + scenarios: result + .scenarios + .into_iter() + .map(MeasurementScenarioOutcome::from) + .collect(), + totals: result.totals, + } + } +} + +impl From for MeasurementScenarioOutcome { + fn from(outcome: ScenarioOutcome) -> Self { + Self { + id: outcome.id, + category: outcome.category, + status: outcome.status, + failure_reason: outcome.failure_reason, + failure: None, + failure_class: outcome.failure_class, + cause: outcome.cause, + evidence_hash: outcome.evidence_hash, + evidence_path: outcome.evidence_path, + retried: outcome.retried, + } + } +} + +impl From for MeasurementScenarioOutcome { + fn from(outcome: ScenarioOutcomeV3) -> Self { + Self { + id: outcome.id, + category: outcome.category, + status: outcome.status, + failure_reason: outcome.failure_reason, + failure: outcome.failure, + failure_class: outcome.failure_class, + cause: outcome.cause, + evidence_hash: outcome.evidence_hash, + evidence_path: outcome.evidence_path, + retried: outcome.retried, + } + } +} + #[derive(Clone, Debug, Deserialize, Serialize)] #[serde(deny_unknown_fields)] pub struct Cause { @@ -192,38 +633,87 @@ pub fn exit_code_for_totals(totals: &Totals) -> u8 { } pub fn parse_and_validate_result(bytes: &[u8]) -> Result { - let result: RunResult = serde_json::from_slice(bytes) + let (document, schema_version) = inspect_result_document(bytes)?; + if !matches!(schema_version, 1 | RESULT_SCHEMA_VERSION) { + return Err(format!( + "unsupported schema_version {schema_version}; expected 1 or {RESULT_SCHEMA_VERSION}" + )); + } + let result: RunResultV1V2 = serde_json::from_value(document.clone()) .map_err(|error| format!("invalid result document: {error}"))?; validate_result(&result)?; - if result.schema_version == 2 { - let document: Value = serde_json::from_slice(bytes) - .map_err(|error| format!("invalid result document: {error}"))?; - let metadata = document - .get("metadata") - .and_then(Value::as_object) - .ok_or_else(|| "invalid result document: metadata must be an object".to_owned())?; - if !metadata.contains_key("run_id") { - return Err( - "invalid result document: metadata.run_id is required for schema_version 2" - .to_owned(), - ); + if schema_version == 2 { + validate_v2_required_properties(&document)?; + } + Ok(result) +} + +pub fn parse_and_validate_measurement(bytes: &[u8]) -> Result { + let (document, schema_version) = inspect_result_document(bytes)?; + let measurement = match schema_version { + 1 | 2 => { + let result: RunResultV1V2 = serde_json::from_value(document.clone()) + .map_err(|error| format!("invalid result document: {error}"))?; + validate_result(&result)?; + if schema_version == 2 { + validate_v2_required_properties(&document)?; + } + Measurement::from(result) } - let scenarios = document - .get("scenarios") - .and_then(Value::as_array) - .ok_or_else(|| "invalid result document: scenarios must be an array".to_owned())?; - if scenarios.iter().any(|scenario| { - !scenario - .as_object() - .is_some_and(|scenario| scenario.contains_key("evidence_path")) - }) { - return Err( - "invalid result document: scenario evidence_path is required for schema_version 2" - .to_owned(), - ); + 3 => { + let result: RunResultV3 = serde_json::from_value(document) + .map_err(|error| format!("invalid result document: {error}"))?; + let measurement = Measurement::from(result); + validate_measurement(&measurement)?; + measurement + } + _ => { + return Err(format!( + "unsupported schema_version {schema_version}; expected 1, 2, or 3" + )); } + }; + Ok(measurement) +} + +fn inspect_result_document(bytes: &[u8]) -> Result<(Value, u32), String> { + let document: Value = serde_json::from_slice(bytes) + .map_err(|error| format!("invalid result document: {error}"))?; + let schema_version = document + .get("schema_version") + .and_then(Value::as_u64) + .and_then(|version| u32::try_from(version).ok()) + .ok_or_else(|| { + "invalid result document: schema_version must be an unsigned 32-bit integer".to_owned() + })?; + Ok((document, schema_version)) +} + +fn validate_v2_required_properties(document: &Value) -> Result<(), String> { + let metadata = document + .get("metadata") + .and_then(Value::as_object) + .ok_or_else(|| "invalid result document: metadata must be an object".to_owned())?; + if !metadata.contains_key("run_id") { + return Err( + "invalid result document: metadata.run_id is required for schema_version 2".to_owned(), + ); } - Ok(result) + let scenarios = document + .get("scenarios") + .and_then(Value::as_array) + .ok_or_else(|| "invalid result document: scenarios must be an array".to_owned())?; + if scenarios.iter().any(|scenario| { + !scenario + .as_object() + .is_some_and(|scenario| scenario.contains_key("evidence_path")) + }) { + return Err( + "invalid result document: scenario evidence_path is required for schema_version 2" + .to_owned(), + ); + } + Ok(()) } pub fn validate_result(result: &RunResult) -> Result<(), String> { @@ -233,24 +723,50 @@ pub fn validate_result(result: &RunResult) -> Result<(), String> { result.schema_version, RESULT_SCHEMA_VERSION )); } - if result.totals.total != result.scenarios.len() as u32 { + validate_totals( + &result.totals, + result.scenarios.iter().map(|outcome| outcome.status), + result.scenarios.len(), + ) +} + +pub fn validate_measurement(measurement: &Measurement) -> Result<(), String> { + if !matches!(measurement.schema_version, 1..=3) { + return Err(format!( + "unsupported schema_version {}; expected 1, 2, or 3", + measurement.schema_version + )); + } + validate_totals( + &measurement.totals, + measurement.scenarios.iter().map(|outcome| outcome.status), + measurement.scenarios.len(), + ) +} + +fn validate_totals( + totals: &Totals, + statuses: impl Iterator, + scenario_count: usize, +) -> Result<(), String> { + if totals.total != scenario_count as u32 { return Err(format!( "totals.total is {} but scenarios contains {} outcome{}", - result.totals.total, - result.scenarios.len(), - if result.scenarios.len() == 1 { "" } else { "s" } + totals.total, + scenario_count, + if scenario_count == 1 { "" } else { "s" } )); } let mut actual = Totals { - total: result.scenarios.len() as u32, + total: scenario_count as u32, passed: 0, failed: 0, errors: 0, skipped: 0, }; - for outcome in &result.scenarios { - match outcome.status { + for status in statuses { + match status { Status::Pass => actual.passed += 1, Status::Fail => actual.failed += 1, Status::Error => actual.errors += 1, @@ -258,10 +774,10 @@ pub fn validate_result(result: &RunResult) -> Result<(), String> { } } for (name, declared, counted) in [ - ("passed", result.totals.passed, actual.passed), - ("failed", result.totals.failed, actual.failed), - ("errors", result.totals.errors, actual.errors), - ("skipped", result.totals.skipped, actual.skipped), + ("passed", totals.passed, actual.passed), + ("failed", totals.failed, actual.failed), + ("errors", totals.errors, actual.errors), + ("skipped", totals.skipped, actual.skipped), ] { if declared != counted { return Err(format!( @@ -411,8 +927,11 @@ mod tests { use std::io::{self, Write}; use super::{ - atomic_write_with, write_result_atomic, Cause, CauseKind, EnvironmentMetadata, RunMetadata, - RunResult, SamplingParams, ScenarioOutcome, ServerMetadata, Status, Totals, + arm_fingerprint, atomic_write_with, write_result_atomic, ArtifactFormat, ArtifactMetadata, + ArtifactSourceKind, Cause, CauseKind, ChatTemplateMetadata, CorpusMetadata, DecodeMode, + EnvironmentMetadata, EnvironmentMetadataV3, IdentityStatus, MeasurementServerMetadata, + ModelMetadata, QuantizationMetadata, RunMetadata, RunResult, SamplingParams, + ScenarioOutcome, ServerMetadata, Status, Totals, }; use crate::ScenarioCategory; @@ -604,6 +1123,356 @@ mod tests { assert!(result.metadata.environment.is_none()); } + #[test] + fn v3_runtime_contract_is_complete() { + let corpus = CorpusMetadata { + id: "wic-50".to_owned(), + revision: "v1".to_owned(), + sha256: "sha256:corpus".to_owned(), + scenario_count: 50, + scoring_version: "v1".to_owned(), + }; + let model = ModelMetadata { + display_name: "Fixture model".to_owned(), + family_id: Some("fixture".to_owned()), + canonical_id: Some("fixture/model".to_owned()), + parameter_count_b: Some(7.0), + endpoint_id: "fixture.gguf".to_owned(), + identity_status: IdentityStatus::Verified, + artifact: ArtifactMetadata { + source_kind: ArtifactSourceKind::LocalFile, + source_id: Some("fixture.gguf".to_owned()), + revision: None, + sha256: Some("sha256:artifact".to_owned()), + format: ArtifactFormat::Gguf, + quantization: Some(QuantizationMetadata { + label: "Q4_K_M".to_owned(), + scheme: Some("k-quant".to_owned()), + bits: Some(4), + }), + }, + }; + let server = MeasurementServerMetadata { + preset_name: "llamacpp".to_owned(), + reported_version: Some("b7000".to_owned()), + quirk_flags: vec!["grammar_constrained_decoding".to_owned()], + decode_mode: DecodeMode::GrammarConstrained, + chat_template: Some(ChatTemplateMetadata { + id: Some("fixture-template".to_owned()), + sha256: Some("sha256:template".to_owned()), + }), + launch_config_sha256: Some("sha256:launch".to_owned()), + }; + let environment = EnvironmentMetadataV3 { + display_label: "Fixture accelerator, 64GB; FixtureOS 1".to_owned(), + os_name: Some("FixtureOS".to_owned()), + os_version: Some("1".to_owned()), + architecture: Some("aarch64".to_owned()), + accelerator: Some("Fixture accelerator".to_owned()), + memory_bytes: Some(68_719_476_736), + }; + let sampling = SamplingParams { + temperature: Some(0.0), + top_p: Some(1.0), + seed: Some(42), + max_tokens: Some(1024), + }; + + let fingerprint = arm_fingerprint(&corpus, &model, &server, Some(&environment), &sampling) + .expect("complete contract should produce a fingerprint"); + assert!(fingerprint.starts_with("v1:sha256:")); + + let mut rescored_corpus = corpus.clone(); + rescored_corpus.scoring_version = "v2".to_owned(); + assert_ne!( + arm_fingerprint( + &rescored_corpus, + &model, + &server, + Some(&environment), + &sampling, + ) + .as_deref(), + Some(fingerprint.as_str()) + ); + + let mut varied_seed = sampling.clone(); + varied_seed.seed = Some(7); + assert_eq!( + arm_fingerprint(&corpus, &model, &server, Some(&environment), &varied_seed,).as_deref(), + Some(fingerprint.as_str()) + ); + + let mut incomplete_model = model.clone(); + incomplete_model.artifact.source_id = None; + assert!(arm_fingerprint( + &corpus, + &incomplete_model, + &server, + Some(&environment), + &sampling, + ) + .is_none()); + + let mut incomplete_server = server.clone(); + incomplete_server.launch_config_sha256 = None; + assert!(arm_fingerprint( + &corpus, + &model, + &incomplete_server, + Some(&environment), + &sampling, + ) + .is_none()); + + let mut unknown_decode_mode = server.clone(); + unknown_decode_mode.decode_mode = DecodeMode::Unknown; + assert!(arm_fingerprint( + &corpus, + &model, + &unknown_decode_mode, + Some(&environment), + &sampling, + ) + .is_none()); + + let mut incomplete_template = server.clone(); + incomplete_template + .chat_template + .as_mut() + .expect("chat template") + .sha256 = None; + assert!(arm_fingerprint( + &corpus, + &model, + &incomplete_template, + Some(&environment), + &sampling, + ) + .is_none()); + + let mut incomplete_environment = environment.clone(); + incomplete_environment.accelerator = None; + assert!(arm_fingerprint( + &corpus, + &model, + &server, + Some(&incomplete_environment), + &sampling, + ) + .is_none()); + + let mut incomplete_sampling = sampling.clone(); + incomplete_sampling.max_tokens = None; + assert!(arm_fingerprint( + &corpus, + &model, + &server, + Some(&environment), + &incomplete_sampling, + ) + .is_none()); + } + + #[test] + fn parses_v1_v2_v3_into_measurement() { + let mut v1 = serde_json::to_value(sample_result()).expect("serialize v1 fixture"); + v1["schema_version"] = serde_json::json!(1); + v1["metadata"] + .as_object_mut() + .expect("v1 metadata") + .remove("run_id"); + v1["scenarios"][0] + .as_object_mut() + .expect("v1 scenario") + .remove("evidence_path"); + let v1 = super::parse_and_validate_measurement( + &serde_json::to_vec(&v1).expect("encode v1 fixture"), + ) + .expect("parse v1 measurement"); + assert_eq!(v1.schema_version, 1); + assert_eq!(v1.metadata.model.display_name, "local-model"); + assert_eq!( + v1.metadata.model.identity_status, + super::IdentityStatus::Unresolved + ); + assert_eq!(v1.cross_model_key(), None); + + let v2_document = serde_json::to_value(sample_result()).expect("serialize v2 fixture"); + let v2 = super::parse_and_validate_measurement( + &serde_json::to_vec(&v2_document).expect("encode v2 fixture"), + ) + .expect("parse v2 measurement"); + assert_eq!(v2.schema_version, 2); + assert_eq!(v2.metadata.model.endpoint_id, "local-model"); + assert_eq!(v2.cross_model_key(), None); + + let v3_document = serde_json::json!({ + "schema_version": 3, + "metadata": { + "run_id": "019c8a4a-05b0-7c22-9f44-2df806328a22", + "timestamp": "2026-08-05T12:00:00Z", + "willitcall_version": "0.1.0", + "endpoint": "http://127.0.0.1:8080/v1", + "model": { + "display_name": "Qwen 2.5 7B Instruct", + "family_id": "qwen2.5", + "canonical_id": "Qwen/Qwen2.5-7B-Instruct", + "parameter_count_b": 7.62, + "endpoint_id": "qwen2.5-7b-instruct-q4_k_m.gguf", + "identity_status": "verified", + "artifact": { + "source_kind": "huggingface", + "source_id": "bartowski/Qwen2.5-7B-Instruct-GGUF", + "revision": "0123456789abcdef", + "sha256": "sha256:0123456789abcdef", + "format": "gguf", + "quantization": { + "label": "Q4_K_M", + "scheme": "k-quant", + "bits": 4 + } + } + }, + "corpus": { + "id": "willitcall-core", + "revision": "v1", + "sha256": "sha256:corpus", + "scenario_count": 1, + "scoring_version": "v1" + }, + "server": { + "preset_name": "llamacpp", + "reported_version": "b6000", + "quirk_flags": ["grammar_constrained_decoding"], + "decode_mode": "grammar_constrained", + "chat_template": { + "id": "qwen2.5", + "sha256": "sha256:template" + }, + "launch_config_sha256": "sha256:launch" + }, + "environment": { + "display_label": "Apple M4 Max, 64GB; macOS 15.5", + "os_name": "macOS", + "os_version": "15.5", + "architecture": "aarch64", + "accelerator": "Apple M4 Max", + "memory_bytes": 68719476736_u64 + }, + "sampling": { + "temperature": 0.0, + "top_p": 1.0, + "seed": 42, + "max_tokens": 1024 + }, + "replication": { + "study_id": "m7-baseline", + "arm_id": "qwen2.5-7b-llamacpp", + "run_index": 0, + "mode": "greedy_reproducibility" + }, + "arm_fingerprint": "v1:arm" + }, + "scenarios": [{ + "id": "single-weather", + "category": "single_call", + "status": "error", + "failure_reason": "turn 1: server returned HTTP 400", + "failure": { + "stage": "request", + "code": "http_error", + "http_status": 400, + "failed_turn_index": 1 + }, + "evidence_hash": "sha256:abc123", + "evidence_path": "evidence/run/single-weather.json", + "retried": false + }], + "totals": { + "total": 1, + "passed": 0, + "failed": 0, + "errors": 1, + "skipped": 0 + } + }); + let schema: serde_json::Value = + serde_json::from_str(include_str!("../../../schemas/result-v3.schema.json")) + .expect("parse v3 schema"); + let validator = jsonschema::validator_for(&schema).expect("compile v3 schema"); + validator + .validate(&v3_document) + .expect("v3 fixture should satisfy schema"); + + let v3 = super::parse_and_validate_measurement( + &serde_json::to_vec(&v3_document).expect("encode v3 fixture"), + ) + .expect("parse v3 measurement"); + assert_eq!(v3.schema_version, 3); + assert_eq!(v3.cross_model_key(), Some("Qwen/Qwen2.5-7B-Instruct")); + assert_eq!( + v3.scenarios[0] + .failure + .as_ref() + .expect("structured failure") + .http_status, + Some(400) + ); + + let mut historical = v3_document.clone(); + for pointer in [ + "/metadata/model/family_id", + "/metadata/model/parameter_count_b", + "/metadata/model/artifact/quantization/scheme", + "/metadata/model/artifact/quantization/bits", + "/metadata/server/chat_template", + "/metadata/server/launch_config_sha256", + "/metadata/environment/os_name", + "/metadata/environment/os_version", + "/metadata/environment/architecture", + "/metadata/environment/accelerator", + "/metadata/environment/memory_bytes", + "/metadata/replication", + "/metadata/arm_fingerprint", + ] { + *historical.pointer_mut(pointer).expect("historical field") = serde_json::Value::Null; + } + validator + .validate(&historical) + .expect("historical v3 nulls should satisfy schema"); + let historical = super::parse_and_validate_measurement( + &serde_json::to_vec(&historical).expect("encode historical fixture"), + ) + .expect("parse historical measurement"); + assert!(historical.metadata.replication.is_none()); + assert!(historical.metadata.arm_fingerprint.is_none()); + + let mut unresolved = v3_document.clone(); + unresolved["metadata"]["model"]["identity_status"] = serde_json::json!("unresolved"); + let unresolved = super::parse_and_validate_measurement( + &serde_json::to_vec(&unresolved).expect("encode unresolved fixture"), + ) + .expect("parse unresolved measurement"); + assert_eq!(unresolved.cross_model_key(), None); + + let mut missing_canonical_id = v3_document.clone(); + missing_canonical_id["metadata"]["model"]["canonical_id"] = serde_json::Value::Null; + let missing_canonical_id = super::parse_and_validate_measurement( + &serde_json::to_vec(&missing_canonical_id).expect("encode declared fixture"), + ) + .expect("parse declared measurement"); + assert_eq!(missing_canonical_id.cross_model_key(), None); + + let mut v3_field_in_v2 = v2_document; + v3_field_in_v2["metadata"]["model"] = v3_document["metadata"]["model"].clone(); + let error = super::parse_and_validate_measurement( + &serde_json::to_vec(&v3_field_in_v2).expect("encode invalid v2 fixture"), + ) + .expect_err("v3-only field in v2 must fail"); + assert!(error.contains("unknown field `model`"), "{error}"); + } + #[test] fn failed_mid_write_never_creates_the_destination() { let directory = tempfile::tempdir().expect("temp directory"); diff --git a/crates/wic-core/src/runner.rs b/crates/wic-core/src/runner.rs index 9ad4f98..3978072 100644 --- a/crates/wic-core/src/runner.rs +++ b/crates/wic-core/src/runner.rs @@ -7,13 +7,18 @@ use std::time::{Duration, SystemTime, UNIX_EPOCH}; use reqwest::header::HeaderMap; use ring::digest::{digest, SHA256}; +use ring::rand::{SecureRandom, SystemRandom}; use serde_json::{json, Map, Value}; use crate::client::{AssistantResponse, CompletionResult, EndpointClient, ToolCall}; +use crate::corpus::{corpus_identity, load_frozen_v1_catalog}; +use crate::registry::ModelRegistry; use crate::result::{ - redact_transcript_turn, write_transcript_atomic, CapturedTurn, EnvironmentMetadata, - RunMetadata, RunResult, SamplingParams, ScenarioOutcome, ServerMetadata, Status, Totals, - Transcript, RESULT_SCHEMA_VERSION, + arm_fingerprint, redact_transcript_turn, write_transcript_atomic, CapturedTurn, + ChatTemplateMetadata, CorpusMetadata, DecodeMode, EnvironmentMetadata, EnvironmentMetadataV3, + Measurement, MeasurementMetadata, MeasurementScenarioOutcome, MeasurementServerMetadata, + ReplicationMetadata, RunMetadata, RunResult, SamplingParams, ScenarioFailure, ScenarioOutcome, + ServerMetadata, Status, Totals, Transcript, RESULT_SCHEMA_VERSION, }; use crate::score::score_response; use crate::{Message, MessageRole, Scenario}; @@ -26,7 +31,13 @@ pub struct RunConfig { pub sampling: SamplingParams, pub server: ServerConfig, pub environment: EnvironmentMetadata, + pub measurement_environment: EnvironmentMetadataV3, pub declared_quant: Option, + pub model_registry: ModelRegistry, + pub decode_mode: DecodeMode, + pub chat_template: Option, + pub launch_config_sha256: Option, + pub replication: Option, pub request_headers: HeaderMap, } @@ -51,6 +62,8 @@ impl RunConfig { seed: u64, temperature: f64, ) -> Self { + let environment = detect_environment(); + let measurement_environment = detect_measurement_environment(&environment); Self { endpoint, model, @@ -66,20 +79,32 @@ impl RunConfig { quirk_flags: Vec::new(), version_probe: None, }, - environment: detect_environment(), + environment, + measurement_environment, declared_quant: None, + model_registry: ModelRegistry { + schema_version: 1, + entries: Vec::new(), + }, + decode_mode: DecodeMode::Unknown, + chat_template: None, + launch_config_sha256: None, + replication: None, request_headers: HeaderMap::new(), } } pub fn with_server(mut self, server: ServerConfig) -> Self { + self.decode_mode = decode_mode(&server.quirk_flags); self.server = server; self } pub fn with_host_hardware_class(mut self, host_hardware_class: Option) -> Self { if let Some(host_hardware_class) = host_hardware_class { - self.environment.host_hardware_class = host_hardware_class; + self.environment.host_hardware_class = host_hardware_class.clone(); + self.measurement_environment.display_label = + format!("{host_hardware_class}; {}", self.environment.host_os); } self } @@ -88,6 +113,31 @@ impl RunConfig { self.declared_quant = declared_quant; self } + + pub fn with_model_registry(mut self, model_registry: ModelRegistry) -> Self { + self.model_registry = model_registry; + self + } + + pub fn with_decode_mode(mut self, decode_mode: DecodeMode) -> Self { + self.decode_mode = decode_mode; + self + } + + pub fn with_chat_template(mut self, chat_template: Option) -> Self { + self.chat_template = chat_template; + self + } + + pub fn with_launch_config_sha256(mut self, launch_config_sha256: Option) -> Self { + self.launch_config_sha256 = launch_config_sha256; + self + } + + pub fn with_replication(mut self, replication: Option) -> Self { + self.replication = replication; + self + } } fn detect_environment() -> EnvironmentMetadata { @@ -97,6 +147,79 @@ fn detect_environment() -> EnvironmentMetadata { } } +fn detect_measurement_environment(environment: &EnvironmentMetadata) -> EnvironmentMetadataV3 { + let (os_name, os_version) = detect_os_name_and_version(); + EnvironmentMetadataV3 { + display_label: format!( + "{}; {}", + environment.host_hardware_class, environment.host_os + ), + os_name, + os_version, + architecture: Some(std::env::consts::ARCH.to_owned()), + accelerator: detect_accelerator(), + memory_bytes: detect_memory_bytes(), + } +} + +fn detect_os_name_and_version() -> (Option, Option) { + #[cfg(target_os = "macos")] + { + ( + command_value("sw_vers", &["-productName"]), + command_value("sw_vers", &["-productVersion"]), + ) + } + + #[cfg(not(target_os = "macos"))] + { + ( + command_value("uname", &["-s"]), + command_value("uname", &["-r"]), + ) + } +} + +fn detect_accelerator() -> Option { + #[cfg(target_os = "macos")] + { + command_value("sysctl", &["-n", "machdep.cpu.brand_string"]) + } + + #[cfg(not(target_os = "macos"))] + { + None + } +} + +fn detect_memory_bytes() -> Option { + #[cfg(target_os = "macos")] + { + command_value("sysctl", &["-n", "hw.memsize"]).and_then(|value| value.parse().ok()) + } + + #[cfg(not(target_os = "macos"))] + { + None + } +} + +fn decode_mode(quirk_flags: &[String]) -> DecodeMode { + if quirk_flags + .iter() + .any(|flag| flag == "grammar_constrained_decoding") + { + DecodeMode::GrammarConstrained + } else if quirk_flags + .iter() + .any(|flag| flag == "unconstrained_post_hoc_parse") + { + DecodeMode::UnconstrainedPostHoc + } else { + DecodeMode::Unknown + } +} + fn detect_host_hardware_class() -> String { #[cfg(target_os = "macos")] { @@ -264,6 +387,50 @@ pub async fn run_scenarios( scenarios: &[Scenario], result_path: &Path, ) -> io::Result { + let measurement = run_measurement(config, scenarios, result_path).await?; + Ok(RunResult { + schema_version: RESULT_SCHEMA_VERSION, + metadata: RunMetadata { + run_id: measurement.metadata.run_id, + timestamp: measurement.metadata.timestamp, + willitcall_version: measurement.metadata.willitcall_version, + endpoint: measurement.metadata.endpoint, + model_id: config.model.clone(), + declared_quant: config.declared_quant.clone(), + server: ServerMetadata { + preset_name: measurement.metadata.server.preset_name, + reported_version: measurement.metadata.server.reported_version, + quirk_flags: measurement.metadata.server.quirk_flags, + }, + environment: Some(config.environment.clone()), + sampling: measurement.metadata.sampling, + preflight_override: measurement.metadata.preflight_override, + preflight_ignored_ports: measurement.metadata.preflight_ignored_ports, + }, + scenarios: measurement + .scenarios + .into_iter() + .map(|outcome| ScenarioOutcome { + id: outcome.id, + category: outcome.category, + status: outcome.status, + failure_reason: outcome.failure_reason, + failure_class: outcome.failure_class, + cause: outcome.cause, + evidence_hash: outcome.evidence_hash, + evidence_path: outcome.evidence_path, + retried: outcome.retried, + }) + .collect(), + totals: measurement.totals, + }) +} + +pub async fn run_measurement( + config: &RunConfig, + scenarios: &[Scenario], + result_path: &Path, +) -> io::Result { let endpoint_client = client(config); let reported_version = match config.server.version_probe { Some(probe) => { @@ -276,7 +443,7 @@ pub async fn run_scenarios( let mut ordered = scenarios.iter().collect::>(); ordered.sort_by(|left, right| left.id.cmp(&right.id)); let timestamp = utc_timestamp(); - let run_id = run_id(×tamp, &config.endpoint, &config.model); + let run_id = run_id(×tamp, &config.endpoint, &config.model)?; let result_parent = match result_path.parent() { Some(parent) if !parent.as_os_str().is_empty() => parent, _ => Path::new("."), @@ -287,22 +454,47 @@ pub async fn run_scenarios( } let totals = totals(&outcomes); - Ok(RunResult { - schema_version: RESULT_SCHEMA_VERSION, - metadata: RunMetadata { + let catalog = load_frozen_v1_catalog().map_err(|error| io::Error::other(error.to_string()))?; + let scenario_count = u32::try_from(scenarios.len()) + .map_err(|_| io::Error::other("scenario count does not fit in u32"))?; + let corpus = CorpusMetadata { + id: catalog.id, + revision: catalog.revision, + sha256: corpus_identity(scenarios), + scenario_count, + scoring_version: "v2".to_owned(), + }; + let model = config.model_registry.resolve(&config.model); + let server = MeasurementServerMetadata { + preset_name: config.server.preset_name.clone(), + reported_version, + quirk_flags: config.server.quirk_flags.clone(), + decode_mode: config.decode_mode, + chat_template: config.chat_template.clone(), + launch_config_sha256: config.launch_config_sha256.clone(), + }; + let environment = Some(config.measurement_environment.clone()); + let fingerprint = arm_fingerprint( + &corpus, + &model, + &server, + environment.as_ref(), + &config.sampling, + ); + Ok(Measurement { + schema_version: 3, + metadata: MeasurementMetadata { run_id, timestamp, willitcall_version: env!("CARGO_PKG_VERSION").to_owned(), endpoint: config.endpoint.clone(), - model_id: config.model.clone(), - declared_quant: config.declared_quant.clone(), - server: ServerMetadata { - preset_name: config.server.preset_name.clone(), - reported_version, - quirk_flags: config.server.quirk_flags.clone(), - }, - environment: Some(config.environment.clone()), + model, + corpus: Some(corpus), + server, + environment, sampling: config.sampling.clone(), + replication: config.replication.clone(), + arm_fingerprint: fingerprint, preflight_override: None, preflight_ignored_ports: None, }, @@ -326,7 +518,7 @@ async fn run_scenario( scenario: &Scenario, run_id: &str, result_parent: &Path, -) -> io::Result { +) -> io::Result { let mut messages = Vec::new(); let mut previous_calls = Vec::new(); let mut evidence = Vec::new(); @@ -337,10 +529,12 @@ async fn run_scenario( match request_message(message, &previous_calls) { Ok(message) => messages.push(message), Err(reason) => { + let failure = + scenario_failure("request", "invalid_turn_message", None, turn_index); return outcome( scenario, Status::Fail, - Some(turn_reason(scenario, turn_index, reason)), + Some((turn_reason(scenario, turn_index, reason), failure)), &evidence, retried, run_id, @@ -368,12 +562,19 @@ async fn run_scenario( turns, retried: completion_retried, } => { + let http_status = last_http_status(&turns); evidence.extend(turns); retried |= completion_retried; + let failure = scenario_failure( + "response_parse", + "invalid_response", + http_status, + turn_index, + ); return outcome( scenario, Status::Fail, - Some(turn_reason(scenario, turn_index, reason)), + Some((turn_reason(scenario, turn_index, reason), failure)), &evidence, retried, run_id, @@ -386,12 +587,23 @@ async fn run_scenario( turns, retried: completion_retried, } => { + let http_status = failed_http_status(&turns); evidence.extend(turns); retried |= completion_retried; + let failure = scenario_failure( + "request", + if http_status.is_some() { + "http_error" + } else { + "request_error" + }, + http_status, + turn_index, + ); return outcome( scenario, Status::Error, - Some(turn_reason(scenario, turn_index, reason)), + Some((turn_reason(scenario, turn_index, reason), failure)), &evidence, retried, run_id, @@ -404,19 +616,31 @@ async fn run_scenario( &scenario.tools, &turn.expected_calls, scenario.arguments_match, + turn.response_requirement, response.content.as_deref(), &response.tool_calls, ) { + let reason = failure.reason; + let failure_class = failure.failure_class; + let structured_failure = scenario_failure( + "scoring", + failure_class.as_deref().unwrap_or("score_mismatch"), + None, + turn_index, + ); let mut result = outcome( scenario, Status::Fail, - Some(turn_reason(scenario, turn_index, failure.reason)), + Some(( + turn_reason(scenario, turn_index, reason), + structured_failure, + )), &evidence, retried, run_id, result_parent, )?; - result.failure_class = failure.failure_class; + result.failure_class = failure_class; return Ok(result); } @@ -489,6 +713,31 @@ fn assistant_message(response: &AssistantResponse) -> Value { }) } +fn last_http_status(turns: &[CapturedTurn]) -> Option { + turns + .last() + .and_then(|turn| turn.response.as_ref()) + .map(|response| response.status) +} + +fn failed_http_status(turns: &[CapturedTurn]) -> Option { + last_http_status(turns).filter(|status| !(200..300).contains(status)) +} + +fn scenario_failure( + stage: &str, + code: &str, + http_status: Option, + turn_index: usize, +) -> ScenarioFailure { + ScenarioFailure { + stage: stage.to_owned(), + code: code.to_owned(), + http_status, + failed_turn_index: u32::try_from(turn_index + 1).ok(), + } +} + fn turn_reason(scenario: &Scenario, turn_index: usize, reason: String) -> String { if scenario.turns.len() > 1 { format!("turn {}: {reason}", turn_index + 1) @@ -500,12 +749,16 @@ fn turn_reason(scenario: &Scenario, turn_index: usize, reason: String) -> String fn outcome( scenario: &Scenario, status: Status, - failure_reason: Option, + failure: Option<(String, ScenarioFailure)>, captured_turns: &[CapturedTurn], retried: bool, run_id: &str, result_parent: &Path, -) -> io::Result { +) -> io::Result { + let (failure_reason, failure) = match failure { + Some((reason, failure)) => (Some(reason), Some(failure)), + None => (None, None), + }; let (evidence_hash, evidence_path) = if captured_turns.is_empty() { (None, None) } else { @@ -526,11 +779,12 @@ fn outcome( let bytes = write_transcript_atomic(&path, &transcript)?; (Some(evidence_hash(&bytes)), Some(relative_path)) }; - Ok(ScenarioOutcome { + Ok(MeasurementScenarioOutcome { id: scenario.id.clone(), category: scenario.category, status, failure_reason, + failure, failure_class: None, cause: None, evidence_hash, @@ -539,7 +793,7 @@ fn outcome( }) } -fn totals(outcomes: &[ScenarioOutcome]) -> Totals { +fn totals(outcomes: &[MeasurementScenarioOutcome]) -> Totals { let mut totals = Totals { total: outcomes.len() as u32, passed: 0, @@ -563,11 +817,15 @@ fn evidence_hash(transcript_bytes: &[u8]) -> String { format!("sha256:{}", hex(hash.as_ref())) } -fn run_id(timestamp: &str, endpoint: &str, model_id: &str) -> String { +fn run_id(timestamp: &str, endpoint: &str, model_id: &str) -> io::Result { let compact_timestamp = timestamp.replace(['-', ':'], ""); - let source = format!("{timestamp}\n{endpoint}\n{model_id}"); + let mut nonce = [0_u8; 16]; + SystemRandom::new() + .fill(&mut nonce) + .map_err(|_| io::Error::other("failed to generate random run id nonce"))?; + let source = format!("{timestamp}\n{endpoint}\n{model_id}\n{}", hex(&nonce)); let hash = digest(&SHA256, source.as_bytes()); - format!("{compact_timestamp}-{}", &hex(hash.as_ref())[..8]) + Ok(format!("{compact_timestamp}-{}", &hex(hash.as_ref())[..32])) } fn hex(bytes: &[u8]) -> String { @@ -659,6 +917,11 @@ mod tests { "Fixture workstation, 32GB" ); assert!(!config.environment.host_os.is_empty()); + assert!(config + .measurement_environment + .display_label + .starts_with("Fixture workstation, 32GB; ")); + assert!(config.measurement_environment.architecture.is_some()); } #[test] @@ -859,15 +1122,31 @@ mod tests { task.abort(); } - #[test] - fn run_id_uses_compact_timestamp_and_metadata_hash_prefix() { - assert_eq!( - super::run_id( - "2026-07-19T20:45:00Z", - "http://localhost:11434/v1", - "qwen2.5:7b-instruct", - ), - "20260719T204500Z-beda7dcb" + #[tokio::test] + async fn same_second_runs_have_distinct_ids() { + let directory = tempfile::tempdir().expect("temp directory"); + let config = super::RunConfig::new( + "http://127.0.0.1:65535/v1".to_owned(), + "fixture-model".to_owned(), + std::time::Duration::from_secs(1), + 42, + 0.0, ); + + for _ in 0..5 { + let first = super::run_measurement(&config, &[], &directory.path().join("first.json")) + .await + .expect("construct first run"); + let second = + super::run_measurement(&config, &[], &directory.path().join("second.json")) + .await + .expect("construct second run"); + if first.metadata.timestamp == second.metadata.timestamp { + assert_ne!(first.metadata.run_id, second.metadata.run_id); + return; + } + } + + panic!("could not construct two runs within the same second"); } } diff --git a/crates/wic-core/src/score.rs b/crates/wic-core/src/score.rs index 899ef81..8545d18 100644 --- a/crates/wic-core/src/score.rs +++ b/crates/wic-core/src/score.rs @@ -3,7 +3,7 @@ use unicode_normalization::UnicodeNormalization; use crate::client::ToolCall; use crate::result::Status; -use crate::{ArgumentsMatch, ExpectedCall, ToolDefinition}; +use crate::{ArgumentsMatch, ExpectedCall, ResponseRequirement, ToolDefinition}; struct UnparsedToolCall { name: String, @@ -24,17 +24,49 @@ pub fn score_response( tools: &[ToolDefinition], expected: &[ExpectedCall], default_policy: ArgumentsMatch, + response_requirement: ResponseRequirement, content: Option<&str>, actual: &[ToolCall], ) -> Result<(), ScoreFailure> { - score_calls(tools, expected, default_policy, actual).map_err(|reason| ScoreFailure { - failure_class: classify_failure(Status::Fail, tools, content, actual).map(str::to_owned), - reason: if is_empty_response(content, actual) { - "empty response: no content and no tool call".to_owned() - } else { - reason - }, - }) + score_calls(tools, expected, default_policy, actual) + .and_then(|()| score_response_requirement(response_requirement, content, actual)) + .map_err(|reason| ScoreFailure { + failure_class: classify_failure(Status::Fail, tools, content, actual) + .map(str::to_owned), + reason: if is_empty_response(content, actual) { + "empty response: no content and no tool call".to_owned() + } else { + reason + }, + }) +} + +fn score_response_requirement( + requirement: ResponseRequirement, + content: Option<&str>, + actual: &[ToolCall], +) -> Result<(), String> { + match requirement { + ResponseRequirement::ToolCalls if actual.is_empty() => { + Err("response requirement not met: expected one or more tool calls".to_owned()) + } + ResponseRequirement::TextWithoutToolCalls if !actual.is_empty() => { + Err("response requirement not met: expected text without tool calls".to_owned()) + } + ResponseRequirement::TextWithoutToolCalls if matches!(content, None | Some("")) => { + Err("empty response: no content and no tool call".to_owned()) + } + ResponseRequirement::NoToolCalls if !actual.is_empty() => { + Err("response requirement not met: expected no tool calls".to_owned()) + } + ResponseRequirement::Either if is_empty_response(content, actual) => { + Err("empty response: no content and no tool call".to_owned()) + } + ResponseRequirement::ToolCalls + | ResponseRequirement::TextWithoutToolCalls + | ResponseRequirement::NoToolCalls + | ResponseRequirement::Either => Ok(()), + } } pub fn classify_failure( diff --git a/crates/wic-core/tests/corpus.rs b/crates/wic-core/tests/corpus.rs index 9e1803a..9e7d90d 100644 --- a/crates/wic-core/tests/corpus.rs +++ b/crates/wic-core/tests/corpus.rs @@ -2,8 +2,11 @@ use std::collections::HashSet; use std::fs; use wic_core::client::ToolCall; +use wic_core::corpus::{corpus_identity, load_frozen_v1_catalog}; use wic_core::score::score_calls; -use wic_core::{load_embedded_scenarios, Scenario, ScenarioCategory}; +use wic_core::{ + load_embedded_scenarios, ResponseRequirement, Scenario, ScenarioCategory, ScenarioFacet, +}; #[test] fn embedded_corpus_is_integral_and_covers_every_category() { @@ -49,6 +52,8 @@ fn embedded_corpus_is_integral_and_covers_every_category() { for scenario in &scenarios { assert_filename_matches_id(scenario); assert_expected_calls_are_valid(scenario); + assert_response_requirements_are_explicit(scenario); + assert_facets_are_expected(scenario); } } @@ -65,6 +70,61 @@ fn every_embedded_scenario_has_a_substantive_rationale() { } } +#[test] +fn corpus_identity_is_order_stable_and_content_sensitive() { + let scenarios = load_embedded_scenarios().expect("embedded scenarios should load"); + let identity = corpus_identity(&scenarios); + + let mut reordered = scenarios.clone(); + reordered.reverse(); + assert_eq!(identity, corpus_identity(&reordered)); + + let mut prompt_changed = scenarios.clone(); + prompt_changed[0].turns[0].messages[0] + .content + .push_str(" Identity probe."); + assert_ne!(identity, corpus_identity(&prompt_changed)); + + let mut schema_changed = scenarios.clone(); + schema_changed[0].tools[0] + .parameters + .as_object_mut() + .expect("tool parameters should be an object") + .insert("identity_probe".to_owned(), serde_json::Value::Bool(true)); + assert_ne!(identity, corpus_identity(&schema_changed)); + + let mut expected_argument_changed = scenarios.clone(); + expected_argument_changed[0].turns[0].expected_calls[0] + .arguments + .as_object_mut() + .expect("expected arguments should be an object") + .insert("identity_probe".to_owned(), serde_json::Value::Bool(true)); + assert_ne!(identity, corpus_identity(&expected_argument_changed)); +} + +#[test] +fn frozen_v1_catalog_remains_verified_and_distinct_from_the_live_corpus() { + let live = load_embedded_scenarios().expect("embedded scenarios should load"); + let catalog = load_frozen_v1_catalog().expect("frozen v1 catalog should load and verify"); + let live_hash = corpus_identity(&live); + + assert_eq!(catalog.id, "wic-50"); + assert_eq!(catalog.revision, "v1"); + assert_eq!(catalog.scenario_count, 50); + assert_eq!(catalog.scenarios.len(), 50); + assert_ne!(catalog.sha256, live_hash); + assert_eq!(corpus_identity(&catalog.scenarios), catalog.sha256); + assert!(catalog + .scenarios + .iter() + .all(|scenario| scenario.facets.is_empty())); + assert!(catalog + .scenarios + .iter() + .flat_map(|scenario| &scenario.turns) + .all(|turn| turn.response_requirement == ResponseRequirement::Either)); +} + fn assert_filename_matches_id(scenario: &Scenario) { let path = format!( "{}/scenarios/{}.toml", @@ -117,3 +177,37 @@ fn assert_expected_calls_are_valid(scenario: &Scenario) { .unwrap_or_else(|error| panic!("{} has invalid expected arguments: {error}", scenario.id)); } } + +fn assert_response_requirements_are_explicit(scenario: &Scenario) { + for turn in &scenario.turns { + let expected = if turn.expected_calls.is_empty() { + ResponseRequirement::TextWithoutToolCalls + } else { + ResponseRequirement::ToolCalls + }; + assert_eq!( + turn.response_requirement, expected, + "{} has the wrong response requirement", + scenario.id + ); + } +} + +fn assert_facets_are_expected(scenario: &Scenario) { + let expected: &[ScenarioFacet] = match scenario.id.as_str() { + "negative-auto-arithmetic" + | "negative-auto-knowledge" + | "negative-greeting" + | "negative-invalid-schema" + | "negative-plain-text" => &[ScenarioFacet::Abstention], + "negative-long-argument" => &[ScenarioFacet::ArgumentFidelity, ScenarioFacet::LongContext], + "negative-unicode-argument" => &[ScenarioFacet::ArgumentFidelity, ScenarioFacet::Unicode], + _ => &[], + }; + assert_eq!( + scenario.facets.as_slice(), + expected, + "{} has the wrong facets", + scenario.id + ); +} diff --git a/crates/wic-core/tests/empty_response.rs b/crates/wic-core/tests/empty_response.rs index 3621657..4811301 100644 --- a/crates/wic-core/tests/empty_response.rs +++ b/crates/wic-core/tests/empty_response.rs @@ -5,7 +5,7 @@ use std::time::Duration; use serde_json::{json, Value}; use support::{MockServer, ScriptedResponse}; -use wic_core::runner::{run_scenarios, RunConfig}; +use wic_core::runner::{run_measurement, run_scenarios, RunConfig}; use wic_core::{load_embedded_scenarios, Scenario}; fn completion(content: Value) -> String { @@ -58,7 +58,7 @@ async fn runner_classifies_only_empty_responses() { 0.0, ); - let result = run_scenarios( + let result = run_measurement( &config, &[single_weather()], &directory.path().join("result.json"), @@ -70,11 +70,16 @@ async fn runner_classifies_only_empty_responses() { assert_eq!(outcome.status, wic_core::result::Status::Fail); assert_eq!(outcome.failure_class.as_deref(), expected_class); assert!(outcome.cause.is_none()); + let failure = outcome.failure.as_ref().expect("structured failure"); + assert_eq!(failure.stage, "scoring"); + assert_eq!(failure.code, expected_class.unwrap_or("score_mismatch")); + assert_eq!(failure.http_status, None); + assert_eq!(failure.failed_turn_index, Some(1)); } } #[tokio::test] -async fn empty_response_preserves_negative_trap_pass() { +async fn empty_response_fails_a_negative_trap_text_requirement() { let server = MockServer::start_scripted( "fixture-model", vec![ScriptedResponse::Json(completion(Value::Null))], @@ -97,6 +102,37 @@ async fn empty_response_preserves_negative_trap_pass() { .await .expect("run scenario"); + let outcome = &result.scenarios[0]; + assert_eq!(outcome.status, wic_core::result::Status::Fail); + assert_eq!(outcome.failure_class.as_deref(), Some("empty_response")); +} + +#[tokio::test] +async fn textual_refusal_passes_a_negative_trap_text_requirement() { + let server = MockServer::start_scripted( + "fixture-model", + vec![ScriptedResponse::Json(completion(json!( + "Hello. I will not use a tool." + )))], + ) + .await; + let directory = tempfile::tempdir().expect("temp directory"); + let config = RunConfig::new( + server.endpoint(), + "fixture-model".to_owned(), + Duration::from_secs(5), + 42, + 0.0, + ); + + let result = run_scenarios( + &config, + &[negative_greeting()], + &directory.path().join("result.json"), + ) + .await + .expect("run scenario"); + assert_eq!(result.scenarios[0].status, wic_core::result::Status::Pass); } diff --git a/crates/wic-core/tests/evidence.rs b/crates/wic-core/tests/evidence.rs index b18c4c4..624d5f4 100644 --- a/crates/wic-core/tests/evidence.rs +++ b/crates/wic-core/tests/evidence.rs @@ -8,8 +8,11 @@ use reqwest::header::{HeaderValue, AUTHORIZATION}; use ring::digest::{digest, SHA256}; use serde_json::{json, Value}; use support::{MockServer, ScriptedResponse}; -use wic_core::result::write_result_atomic; -use wic_core::runner::{run_scenarios, RunConfig}; +use wic_core::corpus::corpus_identity; +use wic_core::result::{ + write_result_atomic, DecodeMode, IdentityStatus, ReplicationMetadata, ReplicationMode, +}; +use wic_core::runner::{run_measurement, run_scenarios, RunConfig}; use wic_core::{load_embedded_scenarios, Scenario}; fn completion(calls: Value, content: Value) -> String { @@ -246,6 +249,76 @@ async fn written_transcript_matches_checked_in_schema() { .expect("written transcript should satisfy checked-in schema"); } +#[tokio::test] +async fn structured_http_failure_is_captured_at_the_request_boundary() { + let raw_model = "/Users/alice/private-models/fixture-model.gguf"; + let server = MockServer::start_scripted( + raw_model, + vec![ScriptedResponse::Status(400, "bad request".to_owned())], + ) + .await; + let directory = tempfile::tempdir().expect("temp directory"); + let config = RunConfig::new( + server.endpoint(), + raw_model.to_owned(), + Duration::from_secs(5), + 42, + 0.0, + ) + .with_replication(Some(ReplicationMetadata { + study_id: "fixture-study".to_owned(), + arm_id: "fixture-arm".to_owned(), + run_index: 2, + mode: ReplicationMode::GreedyReproducibility, + })); + let scenarios = [embedded_scenario("single-weather")]; + let expected_corpus_sha256 = corpus_identity(&scenarios); + + let measurement = run_measurement(&config, &scenarios, &directory.path().join("result.json")) + .await + .expect("run scenario"); + let outcome = &measurement.scenarios[0]; + let failure = outcome.failure.as_ref().expect("structured failure"); + let corpus = measurement + .metadata + .corpus + .as_ref() + .expect("runtime corpus metadata"); + + assert_eq!(corpus.id, "wic-50"); + assert_eq!(corpus.revision, "v1"); + assert_eq!(corpus.sha256, expected_corpus_sha256); + assert_eq!(corpus.scenario_count, 1); + assert_eq!(corpus.scoring_version, "v2"); + assert_eq!(measurement.metadata.model.endpoint_id, "fixture-model.gguf"); + assert_eq!( + measurement.metadata.model.identity_status, + IdentityStatus::Unresolved + ); + assert_eq!(measurement.metadata.server.decode_mode, DecodeMode::Unknown); + assert!(measurement.metadata.server.chat_template.is_none()); + assert!(measurement.metadata.server.launch_config_sha256.is_none()); + assert!(measurement.metadata.environment.is_some()); + assert_eq!( + measurement + .metadata + .replication + .as_ref() + .expect("replication metadata") + .run_index, + 2 + ); + assert!(measurement.metadata.arm_fingerprint.is_none()); + assert_eq!( + outcome.failure_reason.as_deref(), + Some("server returned HTTP 400 Bad Request") + ); + assert_eq!(failure.stage, "request"); + assert_eq!(failure.code, "http_error"); + assert_eq!(failure.http_status, Some(400)); + assert_eq!(failure.failed_turn_index, Some(1)); +} + fn hex(bytes: &[u8]) -> String { const DIGITS: &[u8; 16] = b"0123456789abcdef"; let mut encoded = String::with_capacity(bytes.len() * 2); diff --git a/crates/wic-core/tests/fixtures/result-v2-model-id-preimage.json b/crates/wic-core/tests/fixtures/result-v2-model-id-preimage.json new file mode 100644 index 0000000..cde648d --- /dev/null +++ b/crates/wic-core/tests/fixtures/result-v2-model-id-preimage.json @@ -0,0 +1,34 @@ +{ + "results/llamacpp-granite3.1-dense-8b.json": "sha256-44d19d212d76a6f3fc442e8411fdb44ea6b67ceccfb00be4b4345c9a4cf813e8", + "results/llamacpp-meta-llama-3.1-8b-instruct-q3_k_m.json": "bartowski/Meta-Llama-3.1-8B-Instruct-GGUF:Q3_K_M", + "results/llamacpp-meta-llama-3.1-8b-instruct-q4_k_m.json": "bartowski/Meta-Llama-3.1-8B-Instruct-GGUF:Q4_K_M", + "results/llamacpp-meta-llama-3.1-8b-instruct-q8_0.json": "bartowski/Meta-Llama-3.1-8B-Instruct-GGUF:Q8_0", + "results/llamacpp-phi4-mini.json": "sha256-3c168af1dea0a414299c7d9077e100ac763370e5a98b3c53801a958a47f0a5db", + "results/llamacpp-qwen2.5-1.5b-instruct-q3_k_m.json": "Qwen/Qwen2.5-1.5B-Instruct-GGUF:Q3_K_M", + "results/llamacpp-qwen2.5-1.5b-instruct-q4_k_m.json": "Qwen/Qwen2.5-1.5B-Instruct-GGUF:Q4_K_M", + "results/llamacpp-qwen2.5-1.5b-instruct-q8_0.json": "Qwen/Qwen2.5-1.5B-Instruct-GGUF:Q8_0", + "results/llamacpp-qwen2.5-7b-instruct-q3_k_m.json": "Qwen/Qwen2.5-7B-Instruct-GGUF:Q3_K_M", + "results/llamacpp-qwen2.5-7b-instruct-q4_k_m.json": "Qwen/Qwen2.5-7B-Instruct-GGUF:Q4_K_M", + "results/llamacpp-qwen2.5-7b-instruct-q8_0.json": "Qwen/Qwen2.5-7B-Instruct-GGUF:Q8_0", + "results/llamacpp-watt-tool-8b-q4_k_m.json": "watt-tool-8B.Q4_K_M.gguf", + "results/mlx_lm-meta-llama-3.1-8b-instruct-4bit.json": "mlx-community/Meta-Llama-3.1-8B-Instruct-4bit", + "results/mlx_lm-mistral-7b-instruct-v0.3-4bit.json": "mlx-community/Mistral-7B-Instruct-v0.3-4bit", + "results/mlx_lm-phi4-mini-4bit.json": "mlx-community/Phi-4-mini-instruct-4bit", + "results/mlx_lm-qwen2.5-1.5b-instruct-4bit.json": "mlx-community/Qwen2.5-1.5B-Instruct-4bit", + "results/mlx_lm-qwen2.5-7b-instruct-4bit.json": "mlx-community/Qwen2.5-7B-Instruct-4bit", + "results/mlx_lm-qwen2.5-7b-instruct-8bit.json": "mlx-community/Qwen2.5-7B-Instruct-8bit", + "results/ollama-gemma3-12b.json": "gemma3:12b", + "results/ollama-gemma3-4b.json": "gemma3:4b", + "results/ollama-granite3.1-dense-8b.json": "granite3.1-dense:8b", + "results/ollama-hermes3-8b.json": "hermes3:8b", + "results/ollama-llama3-groq-tool-use-8b.json": "llama3-groq-tool-use:8b", + "results/ollama-llama3.1-8b.json": "llama3.1:8b", + "results/ollama-mistral-7b.json": "mistral:7b", + "results/ollama-phi4-mini.json": "phi4-mini:latest", + "results/ollama-qwen2.5-7b-instruct.json": "qwen2.5:7b-instruct", + "results/ollama-qwen3-0.6b.json": "qwen3:0.6b", + "results/ollama-qwen3-1.7b.json": "qwen3:1.7b", + "results/ollama-qwen3-14b.json": "qwen3:14b", + "results/ollama-qwen3-4b.json": "qwen3:4b", + "results/ollama-qwen3-8b.json": "qwen3:8b" +} diff --git a/crates/wic-core/tests/registry.rs b/crates/wic-core/tests/registry.rs new file mode 100644 index 0000000..a02a22a --- /dev/null +++ b/crates/wic-core/tests/registry.rs @@ -0,0 +1,765 @@ +mod support; + +use std::collections::{BTreeMap, BTreeSet}; +use std::fs; +use std::path::Path; +use std::time::Duration; + +use serde_json::{json, Value}; +use support::MockServer; +use wic_core::client::EndpointClient; +use wic_core::load_embedded_scenarios; +use wic_core::registry::ModelRegistry; +use wic_core::result::{ + DecodeMode, IdentityStatus, Measurement, MeasurementMetadata, MeasurementServerMetadata, + SamplingParams, Totals, +}; + +const LOCAL_EVIDENCE: &str = "registry/evidence/local-file-recovery.json"; +const HF_EVIDENCE: &str = "registry/evidence/huggingface-recovery.json"; +const SHA256: &str = "sha256:33706b165cd6777e29fdcd777ba1e09bd3b2006b428014e1bad17df3902ec1e7"; +const PUBLISHED_REGISTRY: &str = include_str!("../../../registry/models-v1.json"); +const MIGRATION_MANIFEST: &str = include_str!("../../../migrations/result-v2-to-v3-v1.json"); +const V2_MODEL_ID_PREIMAGE: &str = include_str!("fixtures/result-v2-model-id-preimage.json"); + +#[tokio::test] +async fn absolute_selector_is_sent_raw_but_serialized_safely() { + let raw_selector = "/Users/alice/private-models/watt-tool-8B.Q4_K_M.gguf"; + let server = MockServer::start_scripted(raw_selector, Vec::new()).await; + let scenario = load_embedded_scenarios() + .expect("embedded scenarios") + .into_iter() + .find(|scenario| !scenario.stream) + .expect("non-streaming scenario"); + let client = EndpointClient::new( + server.endpoint(), + raw_selector.to_owned(), + Duration::from_secs(2), + sampling(), + ); + + let _ = client + .complete( + &scenario, + &[json!({"role": "user", "content": "Use the tool."})], + ) + .await; + + let requests = server.requests(); + assert_eq!(requests.len(), 1); + assert_eq!(requests[0]["model"], raw_selector); + + let registry = registry(local_entry(Some(SHA256), "verified")); + let model = registry.resolve(raw_selector); + let serialized_result = serde_json::to_value(json!({ + "schema_version": 3, + "metadata": {"model": model} + })) + .expect("serialize result fixture"); + assert_eq!( + serialized_result["metadata"]["model"]["endpoint_id"], + "watt-tool-8B.Q4_K_M.gguf" + ); + let serialized_bytes = + serde_json::to_string(&serialized_result).expect("encode result fixture"); + assert!(!serialized_bytes.contains(raw_selector)); + assert!(!serialized_bytes.contains("/Users/")); + assert!(!serialized_bytes.contains("alice")); +} + +#[test] +fn identity_status_requires_corroborated_identity_and_immutable_artifact() { + let verified = + registry(local_entry(Some(SHA256), "verified")).resolve("watt-tool-8B.Q4_K_M.gguf"); + assert_eq!(verified.identity_status, IdentityStatus::Verified); + + let mut declared_entry = huggingface_entry("measured_artifact", "verified"); + declared_entry["canonical_id"]["corroboration"] = json!("declared"); + declared_entry["identity_status"] = json!("declared"); + let declared = registry(declared_entry).resolve("Qwen/Qwen2.5-7B-Instruct-GGUF:Q4_K_M"); + assert_eq!(declared.identity_status, IdentityStatus::Declared); +} + +#[test] +fn repository_head_at_capture_is_not_an_immutable_measured_revision() { + let registry = registry(huggingface_entry("repository_head_at_capture", "declared")); + + let model = registry.resolve("Qwen/Qwen2.5-7B-Instruct-GGUF:Q4_K_M"); + + assert_eq!(model.identity_status, IdentityStatus::Declared); + assert_eq!( + model.artifact.revision.as_deref(), + Some("91cad51170dc346986eccefdc2dd33a9da36ead9") + ); +} + +#[test] +fn local_artifact_without_sha256_is_unresolved_and_cannot_join() { + let entry = local_entry(None, "unresolved"); + validate_schema(&json!({"schema_version": 1, "entries": [entry.clone()]})) + .expect("unresolved local entry should satisfy registry schema"); + let model = registry(entry).resolve("watt-tool-8B.Q4_K_M.gguf"); + assert_eq!(model.identity_status, IdentityStatus::Unresolved); + assert_eq!(measurement_with_model(model).cross_model_key(), None); + + let declared = json!({ + "schema_version": 1, + "entries": [local_entry(None, "declared")] + }); + assert!(validate_schema(&declared).is_err()); + assert!( + ModelRegistry::from_json(&serde_json::to_vec(&declared).expect("encode registry")).is_err() + ); +} + +#[test] +fn unknown_selector_is_unresolved_without_name_or_filename_inference() { + let registry = registry(local_entry(Some(SHA256), "verified")); + + let model = registry.resolve("Qwen/Looks-Like-A-Known-8B-GGUF:Q4_K_M"); + + assert_eq!(model.identity_status, IdentityStatus::Unresolved); + assert_eq!(model.display_name, "Unresolved model"); + assert_eq!(model.canonical_id, None); + assert_eq!(model.artifact.sha256, None); +} + +#[test] +fn registry_schema_requires_provenance_and_safe_explicit_selectors() { + let valid = json!({ + "schema_version": 1, + "entries": [local_entry(Some(SHA256), "verified")] + }); + validate_schema(&valid).expect("registry fixture should satisfy schema"); + + let mut missing_provenance = valid.clone(); + missing_provenance["entries"][0]["family_id"] = json!({"value": "watt"}); + assert!(validate_schema(&missing_provenance).is_err()); + + let mut absolute_key = valid; + absolute_key["entries"][0]["selectors"] = json!(["/Users/alice/model.gguf"]); + assert!(validate_schema(&absolute_key).is_err()); +} + +#[test] +fn published_registry_has_32_explicit_provenance_checked_mappings() { + let repo_root = Path::new(env!("CARGO_MANIFEST_DIR")).join("../.."); + let registry_document: Value = + serde_json::from_str(PUBLISHED_REGISTRY).expect("parse published model registry"); + let manifest: Value = + serde_json::from_str(MIGRATION_MANIFEST).expect("parse result migration manifest"); + + validate_published_registry(®istry_document, &manifest, &repo_root) + .expect("published registry and migration manifest must be provenance checked"); + + let registry = ModelRegistry::from_json(PUBLISHED_REGISTRY.as_bytes()) + .expect("load published model registry"); + let verified = registry + .entries + .iter() + .filter(|entry| entry.identity_status == IdentityStatus::Verified) + .count(); + let declared = registry + .entries + .iter() + .filter(|entry| entry.identity_status == IdentityStatus::Declared) + .count(); + let unresolved = registry + .entries + .iter() + .filter(|entry| entry.identity_status == IdentityStatus::Unresolved) + .count(); + assert_eq!((verified, declared, unresolved), (1, 27, 4)); + + let mut missing_provenance = registry_document.clone(); + missing_provenance["entries"][0]["display_name"] + .as_object_mut() + .expect("display_name object") + .remove("provenance_ref"); + assert!(validate_published_registry(&missing_provenance, &manifest, &repo_root).is_err()); + + let mut verified_local_without_sha = registry_document.clone(); + let verified_local = verified_local_without_sha["entries"] + .as_array_mut() + .expect("registry entries") + .iter_mut() + .find(|entry| { + entry["identity_status"] == "verified" + && entry["artifact"]["source_kind"]["value"] == "local_file" + }) + .expect("verified local-file entry"); + verified_local["artifact"]["sha256"] = Value::Null; + assert!( + validate_published_registry(&verified_local_without_sha, &manifest, &repo_root).is_err() + ); + + let mut filename_derived = registry_document.clone(); + let first_mapping = &manifest["entries"][0]; + let selector = first_mapping["registry_selector"] + .as_str() + .expect("registry selector"); + let result_path = first_mapping["result_path"].as_str().expect("result path"); + let filename_stem = Path::new(result_path) + .file_stem() + .and_then(|stem| stem.to_str()) + .expect("result filename stem"); + let filename_derived_entry = filename_derived["entries"] + .as_array_mut() + .expect("registry entries") + .iter_mut() + .find(|entry| entry["selectors"][0] == selector) + .expect("mapped registry entry"); + filename_derived_entry["display_name"]["value"] = json!(filename_stem); + assert!(validate_published_registry(&filename_derived, &manifest, &repo_root).is_err()); +} + +fn validate_published_registry( + registry_document: &Value, + manifest: &Value, + repo_root: &Path, +) -> Result<(), String> { + validate_schema(registry_document)?; + ModelRegistry::from_json( + &serde_json::to_vec(registry_document).map_err(|error| error.to_string())?, + ) + .map_err(|error| error.to_string())?; + + let manifest_object = manifest + .as_object() + .ok_or_else(|| "migration manifest must be an object".to_owned())?; + let expected_manifest_fields = [ + "schema_version", + "source_schema_version", + "target_schema_version", + "entries", + ]; + if manifest_object.len() != expected_manifest_fields.len() + || expected_manifest_fields + .iter() + .any(|field| !manifest_object.contains_key(*field)) + { + return Err("migration manifest has unexpected fields".to_owned()); + } + if manifest["schema_version"] != 1 + || manifest["source_schema_version"] != 2 + || manifest["target_schema_version"] != 3 + { + return Err("migration manifest versions must be 1, 2, and 3".to_owned()); + } + + let registry_entries = registry_document["entries"] + .as_array() + .ok_or_else(|| "registry entries must be an array".to_owned())?; + let mappings = manifest["entries"] + .as_array() + .ok_or_else(|| "migration entries must be an array".to_owned())?; + if registry_entries.len() != 32 || mappings.len() != 32 { + return Err("registry and migration manifest must each have 32 entries".to_owned()); + } + + let mut registry_by_selector = BTreeMap::new(); + for entry in registry_entries { + let selectors = entry["selectors"] + .as_array() + .ok_or_else(|| "registry selectors must be an array".to_owned())?; + if selectors.len() != 1 { + return Err("each published registry entry must have one explicit selector".to_owned()); + } + let selector = selectors[0] + .as_str() + .ok_or_else(|| "registry selector must be a string".to_owned())?; + if registry_by_selector.insert(selector, entry).is_some() { + return Err(format!("duplicate registry selector {selector:?}")); + } + } + + let published_paths = published_result_paths(repo_root)?; + let v2_model_ids: BTreeMap = serde_json::from_str(V2_MODEL_ID_PREIMAGE) + .map_err(|error| format!("parse v2 model-id preimage: {error}"))?; + if v2_model_ids.len() != 32 + || v2_model_ids.keys().cloned().collect::>() != published_paths + { + return Err( + "v2 model-id preimage must cover every published result exactly once".to_owned(), + ); + } + let mut mapped_paths = BTreeSet::new(); + let mut mapped_selectors = BTreeSet::new(); + for mapping in mappings { + let mapping_object = mapping + .as_object() + .ok_or_else(|| "migration entry must be an object".to_owned())?; + let expected_mapping_fields = ["result_path", "registry_selector", "provenance_ref"]; + if mapping_object.len() != expected_mapping_fields.len() + || expected_mapping_fields + .iter() + .any(|field| !mapping_object.contains_key(*field)) + { + return Err("migration entry has unexpected fields".to_owned()); + } + + let result_path = mapping["result_path"] + .as_str() + .ok_or_else(|| "result_path must be a string".to_owned())?; + let filename = explicit_published_filename(result_path)?; + if !mapped_paths.insert(result_path.to_owned()) { + return Err(format!("duplicate migration path {result_path:?}")); + } + + let selector = mapping["registry_selector"] + .as_str() + .ok_or_else(|| "registry_selector must be a string".to_owned())?; + if !mapped_selectors.insert(selector.to_owned()) { + return Err(format!("duplicate mapped selector {selector:?}")); + } + let registry_entry = registry_by_selector + .get(selector) + .ok_or_else(|| format!("no registry entry for selector {selector:?}"))?; + + let result_bytes = fs::read(repo_root.join(result_path)) + .map_err(|error| format!("read {result_path}: {error}"))?; + let result: Value = serde_json::from_slice(&result_bytes) + .map_err(|error| format!("parse {result_path}: {error}"))?; + if result["schema_version"] != 3 { + return Err(format!("published result {result_path} must be schema v3")); + } + if v2_model_ids.get(result_path).map(String::as_str) != Some(selector) { + return Err(format!( + "migration selector for {result_path} must equal its v2 metadata.model_id" + )); + } + + let mapping_provenance = mapping["provenance_ref"] + .as_str() + .ok_or_else(|| "mapping provenance_ref must be a string".to_owned())?; + let recovery_record = evidence_record(repo_root, mapping_provenance)?; + if recovery_record["subject"] != filename || recovery_record["model_id"] != selector { + return Err(format!( + "mapping provenance for {result_path} must record its subject and selector" + )); + } + + validate_registry_entry(registry_entry, filename, repo_root)?; + } + + if mapped_paths != published_paths { + return Err( + "migration manifest must map every published result path exactly once".to_owned(), + ); + } + if mapped_selectors.len() != registry_by_selector.len() { + return Err("every published registry entry must be used by one migration".to_owned()); + } + Ok(()) +} + +fn published_result_paths(repo_root: &Path) -> Result, String> { + let mut paths = BTreeSet::new(); + for entry in fs::read_dir(repo_root.join("results")).map_err(|error| error.to_string())? { + let entry = entry.map_err(|error| error.to_string())?; + let path = entry.path(); + if !path.is_file() + || path.extension().and_then(|extension| extension.to_str()) != Some("json") + { + continue; + } + let filename = path + .file_name() + .and_then(|name| name.to_str()) + .ok_or_else(|| "published result filename must be UTF-8".to_owned())?; + paths.insert(format!("results/{filename}")); + } + Ok(paths) +} + +fn explicit_published_filename(result_path: &str) -> Result<&str, String> { + let filename = result_path + .strip_prefix("results/") + .filter(|filename| { + !filename.is_empty() + && filename.ends_with(".json") + && !filename.contains(['/', '\\']) + && !filename + .chars() + .any(|character| matches!(character, '*' | '?' | '[' | ']' | '{' | '}')) + }) + .ok_or_else(|| format!("migration path {result_path:?} is not an explicit result path"))?; + Ok(filename) +} + +fn validate_registry_entry(entry: &Value, filename: &str, repo_root: &Path) -> Result<(), String> { + let fact_pointers = [ + "/display_name", + "/family_id", + "/canonical_id", + "/parameter_count_b", + "/artifact/source_kind", + "/artifact/source_id", + "/artifact/revision", + "/artifact/sha256", + "/artifact/format", + "/artifact/quantization", + ]; + for pointer in fact_pointers { + let fact = entry + .pointer(pointer) + .ok_or_else(|| format!("registry fact {pointer} is missing"))?; + if fact.is_null() { + continue; + } + let provenance_ref = fact["provenance_ref"] + .as_str() + .filter(|reference| !reference.is_empty()) + .ok_or_else(|| format!("non-null registry fact {pointer} lacks provenance_ref"))?; + evidence_record(repo_root, provenance_ref)?; + } + + let filename_stem = filename.strip_suffix(".json").unwrap_or(filename); + for pointer in ["/display_name", "/family_id", "/canonical_id"] { + let fact = entry + .pointer(pointer) + .ok_or_else(|| format!("registry identity fact {pointer} is missing"))?; + if fact.is_null() { + continue; + } + let value = fact["value"] + .as_str() + .ok_or_else(|| format!("registry identity fact {pointer} must be a string"))?; + if value == filename || value == filename_stem { + return Err(format!( + "registry identity fact {pointer} was derived from filename" + )); + } + let provenance_ref = fact["provenance_ref"] + .as_str() + .ok_or_else(|| format!("registry identity fact {pointer} lacks provenance_ref"))?; + let record = evidence_record(repo_root, provenance_ref)?; + let supported = [ + "model_id", + "upstream_canonical_checkpoint", + "corroborated_ollama_model", + ] + .iter() + .filter_map(|field| record[*field].as_str()) + .any(|candidate| candidate == value); + if !supported { + return Err(format!( + "registry identity fact {pointer} is not stated by its recovery record" + )); + } + } + + validate_artifact_source_id(entry, repo_root)?; + validate_artifact_revision(entry, repo_root)?; + validate_artifact_sha256(entry, repo_root)?; + validate_artifact_classifications(entry, repo_root)?; + validate_artifact_quantization(entry, repo_root)?; + Ok(()) +} + +fn validate_artifact_source_id(entry: &Value, repo_root: &Path) -> Result<(), String> { + let fact = &entry["artifact"]["source_id"]; + if fact.is_null() { + return Ok(()); + } + let value = fact["value"] + .as_str() + .ok_or_else(|| "artifact.source_id must be a string".to_owned())?; + let record = evidence_record( + repo_root, + fact["provenance_ref"] + .as_str() + .ok_or_else(|| "artifact.source_id lacks provenance_ref".to_owned())?, + )?; + let supported = [ + "model_id", + "artifact_repository", + "upstream_artifact_repository", + ] + .iter() + .filter_map(|field| record[*field].as_str()) + .any(|candidate| candidate == value); + if !supported { + return Err("artifact.source_id is not stated by its recovery record".to_owned()); + } + Ok(()) +} + +fn validate_artifact_revision(entry: &Value, repo_root: &Path) -> Result<(), String> { + let fact = &entry["artifact"]["revision"]; + if fact.is_null() { + return Ok(()); + } + let value = fact["value"] + .as_str() + .ok_or_else(|| "artifact.revision must be a string".to_owned())?; + let record = evidence_record( + repo_root, + fact["provenance_ref"] + .as_str() + .ok_or_else(|| "artifact.revision lacks provenance_ref".to_owned())?, + )?; + let supported = ["revision", "manifest_digest", "upstream_artifact_revision"] + .iter() + .filter_map(|field| record[*field].as_str()) + .any(|candidate| candidate == value); + if !supported { + return Err("artifact.revision is not stated by its recovery record".to_owned()); + } + Ok(()) +} + +fn validate_artifact_sha256(entry: &Value, repo_root: &Path) -> Result<(), String> { + let fact = &entry["artifact"]["sha256"]; + if fact.is_null() { + return Ok(()); + } + let value = fact["value"] + .as_str() + .ok_or_else(|| "artifact.sha256 must be a string".to_owned())?; + let record = evidence_record( + repo_root, + fact["provenance_ref"] + .as_str() + .ok_or_else(|| "artifact.sha256 lacks provenance_ref".to_owned())?, + )?; + let supported = ["sha256", "model_layer_blob_digest"] + .iter() + .filter_map(|field| record[*field].as_str()) + .map(|candidate| { + if candidate.starts_with("sha256:") { + candidate.to_owned() + } else { + format!("sha256:{candidate}") + } + }) + .any(|candidate| candidate == value); + if !supported { + return Err("artifact.sha256 is not stated by its recovery record".to_owned()); + } + Ok(()) +} + +fn validate_artifact_classifications(entry: &Value, repo_root: &Path) -> Result<(), String> { + let source_kind = &entry["artifact"]["source_kind"]; + let source_reference = source_kind["provenance_ref"] + .as_str() + .ok_or_else(|| "artifact.source_kind lacks provenance_ref".to_owned())?; + evidence_record(repo_root, source_reference)?; + let expected_source_kind = if source_reference.contains("huggingface-recovery.json") { + "huggingface" + } else if source_reference.contains("ollama-recovery.json") { + "ollama" + } else { + "local_file" + }; + if source_kind["value"] != expected_source_kind { + return Err("artifact.source_kind disagrees with its recovery record".to_owned()); + } + + let format = &entry["artifact"]["format"]; + let format_reference = format["provenance_ref"] + .as_str() + .ok_or_else(|| "artifact.format lacks provenance_ref".to_owned())?; + let record = evidence_record(repo_root, format_reference)?; + let expected_format = if record["model_layer_blob_digest"].is_string() { + "ollama_blob" + } else if record["artifact_repository"] + .as_str() + .is_some_and(|repository| repository.starts_with("mlx-community/")) + { + "mlx" + } else if record["artifact_repository"] + .as_str() + .or_else(|| record["upstream_artifact_repository"].as_str()) + .is_some_and(|repository| repository.ends_with("-GGUF")) + { + "gguf" + } else { + return Err("artifact.format is not supported by its recovery record".to_owned()); + }; + if format["value"] != expected_format { + return Err("artifact.format disagrees with its recovery record".to_owned()); + } + Ok(()) +} + +fn validate_artifact_quantization(entry: &Value, repo_root: &Path) -> Result<(), String> { + let fact = &entry["artifact"]["quantization"]; + if fact.is_null() { + return Ok(()); + } + let label = fact["value"]["label"] + .as_str() + .ok_or_else(|| "artifact.quantization.label must be a string".to_owned())?; + let record = evidence_record( + repo_root, + fact["provenance_ref"] + .as_str() + .ok_or_else(|| "artifact.quantization lacks provenance_ref".to_owned())?, + )?; + let supported = record["declared_quantization"] + .as_str() + .is_some_and(|quantization| quantization == label) + || record["model_id"] + .as_str() + .is_some_and(|model_id| model_id.contains(label)); + if !supported { + return Err("artifact.quantization is not stated by its recovery record".to_owned()); + } + Ok(()) +} + +fn evidence_record(repo_root: &Path, provenance_ref: &str) -> Result { + let (file_ref, pointer) = provenance_ref + .split_once('#') + .ok_or_else(|| format!("provenance_ref {provenance_ref:?} must point at a record"))?; + let record_index = pointer + .strip_prefix("/records/") + .filter(|index| !index.is_empty() && !index.contains('/')) + .and_then(|index| index.parse::().ok()) + .ok_or_else(|| format!("provenance_ref {provenance_ref:?} must point at records/N"))?; + let evidence_bytes = fs::read(repo_root.join(file_ref)) + .map_err(|error| format!("read evidence {file_ref}: {error}"))?; + let evidence: Value = serde_json::from_slice(&evidence_bytes) + .map_err(|error| format!("parse evidence {file_ref}: {error}"))?; + evidence["records"] + .get(record_index) + .filter(|record| record.is_object()) + .cloned() + .ok_or_else(|| format!("provenance_ref {provenance_ref:?} does not name a record")) +} + +fn registry(entry: Value) -> ModelRegistry { + let document = json!({"schema_version": 1, "entries": [entry]}); + validate_schema(&document).expect("registry fixture should satisfy schema"); + ModelRegistry::from_json(&serde_json::to_vec(&document).expect("encode registry")) + .expect("load registry") +} + +fn local_entry(sha256: Option<&str>, identity_status: &str) -> Value { + json!({ + "selectors": ["watt-tool-8B.Q4_K_M.gguf"], + "display_name": provenanced("Watt Tool 8B", LOCAL_EVIDENCE), + "family_id": provenanced("watt-tool", LOCAL_EVIDENCE), + "canonical_id": { + "value": "watt-ai/watt-tool-8B", + "provenance_ref": LOCAL_EVIDENCE, + "corroboration": "corroborated" + }, + "parameter_count_b": { + "value": 8.0, + "provenance_ref": LOCAL_EVIDENCE + }, + "identity_status": identity_status, + "artifact": { + "source_kind": provenanced("local_file", LOCAL_EVIDENCE), + "source_id": provenanced("watt-tool-8B.Q4_K_M.gguf", LOCAL_EVIDENCE), + "revision": null, + "sha256": sha256.map(|value| provenanced(value, LOCAL_EVIDENCE)), + "format": provenanced("gguf", LOCAL_EVIDENCE), + "quantization": { + "value": { + "label": "Q4_K_M", + "scheme": "k-quant", + "bits": 4 + }, + "provenance_ref": LOCAL_EVIDENCE + } + } + }) +} + +fn huggingface_entry(revision_scope: &str, identity_status: &str) -> Value { + json!({ + "selectors": ["Qwen/Qwen2.5-7B-Instruct-GGUF:Q4_K_M"], + "display_name": provenanced("Qwen 2.5 7B Instruct", HF_EVIDENCE), + "family_id": provenanced("qwen2.5", HF_EVIDENCE), + "canonical_id": { + "value": "Qwen/Qwen2.5-7B-Instruct", + "provenance_ref": HF_EVIDENCE, + "corroboration": "corroborated" + }, + "parameter_count_b": null, + "identity_status": identity_status, + "artifact": { + "source_kind": provenanced("huggingface", HF_EVIDENCE), + "source_id": provenanced("Qwen/Qwen2.5-7B-Instruct-GGUF", HF_EVIDENCE), + "revision": { + "value": "91cad51170dc346986eccefdc2dd33a9da36ead9", + "provenance_ref": HF_EVIDENCE, + "revision_scope": revision_scope + }, + "sha256": null, + "format": provenanced("gguf", HF_EVIDENCE), + "quantization": { + "value": { + "label": "Q4_K_M", + "scheme": "k-quant", + "bits": 4 + }, + "provenance_ref": HF_EVIDENCE + } + } + }) +} + +fn provenanced(value: &str, provenance_ref: &str) -> Value { + json!({"value": value, "provenance_ref": provenance_ref}) +} + +fn validate_schema(document: &Value) -> Result<(), String> { + let schema: Value = serde_json::from_str(include_str!( + "../../../schemas/model-registry-v1.schema.json" + )) + .expect("parse model registry schema"); + jsonschema::validator_for(&schema) + .expect("compile model registry schema") + .validate(document) + .map_err(|error| error.to_string()) +} + +fn sampling() -> SamplingParams { + SamplingParams { + temperature: Some(0.0), + top_p: Some(1.0), + seed: Some(42), + max_tokens: Some(64), + } +} + +fn measurement_with_model(model: wic_core::result::ModelMetadata) -> Measurement { + Measurement { + schema_version: 3, + metadata: MeasurementMetadata { + run_id: "fixture-run".to_owned(), + timestamp: "2026-08-05T12:00:00Z".to_owned(), + willitcall_version: "0.1.0".to_owned(), + endpoint: "http://127.0.0.1:8080/v1".to_owned(), + model, + corpus: None, + server: MeasurementServerMetadata { + preset_name: "fixture".to_owned(), + reported_version: None, + quirk_flags: Vec::new(), + decode_mode: DecodeMode::Unknown, + chat_template: None, + launch_config_sha256: None, + }, + environment: None, + sampling: sampling(), + replication: None, + arm_fingerprint: None, + preflight_override: None, + preflight_ignored_ports: None, + }, + scenarios: Vec::new(), + totals: Totals { + total: 0, + passed: 0, + failed: 0, + errors: 0, + skipped: 0, + }, + } +} diff --git a/crates/wic-core/tests/scoring.rs b/crates/wic-core/tests/scoring.rs index 5f6829a..2b20031 100644 --- a/crates/wic-core/tests/scoring.rs +++ b/crates/wic-core/tests/scoring.rs @@ -2,7 +2,7 @@ use serde_json::json; use wic_core::client::ToolCall; use wic_core::result::Status; use wic_core::score::{classify_failure, score_calls, score_response}; -use wic_core::{ArgumentsMatch, ExpectedCall, ToolDefinition}; +use wic_core::{ArgumentsMatch, ExpectedCall, ResponseRequirement, ToolDefinition}; fn weather_tool() -> ToolDefinition { ToolDefinition { @@ -74,8 +74,15 @@ fn empty_response_is_distinct_from_text_without_a_tool_call() { let expected = [expected("get_weather", json!({"city": "Boston"}))]; for content in [None, Some("")] { - let failure = score_response(&tools, &expected, ArgumentsMatch::Exact, content, &[]) - .expect_err("empty response must fail"); + let failure = score_response( + &tools, + &expected, + ArgumentsMatch::Exact, + ResponseRequirement::ToolCalls, + content, + &[], + ) + .expect_err("empty response must fail"); assert_eq!( failure.reason, "empty response: no content and no tool call" @@ -87,6 +94,7 @@ fn empty_response_is_distinct_from_text_without_a_tool_call() { &tools, &expected, ArgumentsMatch::Exact, + ResponseRequirement::ToolCalls, Some("I cannot call that tool."), &[], ) @@ -96,8 +104,51 @@ fn empty_response_is_distinct_from_text_without_a_tool_call() { } #[test] -fn empty_response_preserves_a_negative_trap_pass() { - assert!(score_response(&[weather_tool()], &[], ArgumentsMatch::Exact, None, &[]).is_ok()); +fn text_requirement_rejects_an_empty_response_and_accepts_a_real_refusal() { + let failure = score_response( + &[weather_tool()], + &[], + ArgumentsMatch::Exact, + ResponseRequirement::TextWithoutToolCalls, + None, + &[], + ) + .expect_err("empty response must fail a text requirement"); + assert_eq!(failure.failure_class.as_deref(), Some("empty_response")); + + assert!(score_response( + &[weather_tool()], + &[], + ArgumentsMatch::Exact, + ResponseRequirement::TextWithoutToolCalls, + Some("I cannot call that tool."), + &[], + ) + .is_ok()); +} + +#[test] +fn only_an_explicit_no_tool_calls_requirement_accepts_silence() { + assert!(score_response( + &[weather_tool()], + &[], + ArgumentsMatch::Exact, + ResponseRequirement::NoToolCalls, + None, + &[], + ) + .is_ok()); + + let failure = score_response( + &[weather_tool()], + &[], + ArgumentsMatch::Exact, + ResponseRequirement::Either, + None, + &[], + ) + .expect_err("either still requires a non-empty response"); + assert_eq!(failure.failure_class.as_deref(), Some("empty_response")); } #[test] @@ -111,8 +162,15 @@ fn unparsed_tool_call_shapes_classify_valid_offered_calls() { r#"[`get_weather` {"city": "Boston"}]"#, r#"`get_weather` {"city": "Boston"}"#, ] { - let failure = score_response(&tools, &expected, ArgumentsMatch::Exact, Some(content), &[]) - .expect_err("unparsed call must remain a failing verdict"); + let failure = score_response( + &tools, + &expected, + ArgumentsMatch::Exact, + ResponseRequirement::ToolCalls, + Some(content), + &[], + ) + .expect_err("unparsed call must remain a failing verdict"); assert_eq!( failure.failure_class.as_deref(), Some("unparsed_tool_call"), @@ -134,8 +192,15 @@ fn unparsed_tool_call_near_misses_remain_plain_failures() { "I would use `get_weather` for that", "```text\n[`get_weather` {\"city\": \"Boston\"}]\n```", ] { - let failure = score_response(&tools, &expected, ArgumentsMatch::Exact, Some(content), &[]) - .expect_err("missing parsed call must fail"); + let failure = score_response( + &tools, + &expected, + ArgumentsMatch::Exact, + ResponseRequirement::ToolCalls, + Some(content), + &[], + ) + .expect_err("missing parsed call must fail"); assert_ne!( failure.failure_class.as_deref(), Some("unparsed_tool_call"), @@ -150,6 +215,7 @@ fn unparsed_tool_call_preserves_a_negative_trap_pass() { &[weather_tool()], &[], ArgumentsMatch::Exact, + ResponseRequirement::TextWithoutToolCalls, Some(r#"[`get_weather` {"city": "Boston"}]"#), &[] ) diff --git a/crates/willitcall/src/main.rs b/crates/willitcall/src/main.rs index 02562bb..3d1149b 100644 --- a/crates/willitcall/src/main.rs +++ b/crates/willitcall/src/main.rs @@ -1,3 +1,4 @@ +mod migrate; mod report; mod site; @@ -5,21 +6,24 @@ mod site; #[path = "../../wic-core/tests/support/mod.rs"] mod support; -use std::io::IsTerminal; +use std::io::{IsTerminal, Write}; use std::path::{Path, PathBuf}; use std::process::ExitCode; +use std::sync::atomic::{AtomicU64, Ordering}; use std::time::Duration; use clap::{ArgGroup, Args, Parser, Subcommand, ValueEnum}; use wic_core::client::{ parse_non_streaming, parse_sse_data, reassemble_sse_payloads, AssistantResponse, }; +use wic_core::registry::ModelRegistry; use wic_core::result::{ - exit_code_for_totals, parse_and_validate_result, validate_result, write_result_atomic, Cause, - CauseKind, PreflightOverride, RunResult, Status, + exit_code_for_totals, parse_and_validate_measurement, validate_measurement, validate_result, + write_result_atomic, Cause, CauseKind, Measurement, PreflightOverride, QuantizationMetadata, + RunMetadataV3, RunResult, RunResultV3, ScenarioOutcomeV3, ServerMetadataV3, Status, }; use wic_core::runner::{ - contention_preflight_ignoring_ports, preflight, run_scenarios, RunConfig, ServerConfig, + contention_preflight_ignoring_ports, preflight, run_measurement, RunConfig, ServerConfig, ServerVersionProbe, }; use wic_core::score::classify_failure; @@ -36,6 +40,8 @@ const KNOWN_INFERENCE_SERVERS: &[(u16, &str)] = &[ (1234, "LM Studio"), (8000, "vLLM"), ]; +const MODEL_REGISTRY: &[u8] = include_bytes!("../../../registry/models-v1.json"); +static NEXT_TEMP_FILE: AtomicU64 = AtomicU64::new(0); #[derive(Debug, Parser)] #[command(name = "willitcall", after_long_help = EXIT_CODE_HELP)] @@ -49,6 +55,7 @@ enum Command { Run(RunArgs), Scenarios(ScenariosArgs), Validate(ValidateArgs), + MigrateV3(MigrateArgs), Annotate(AnnotateArgs), Rescore(RescoreArgs), Site(SiteArgs), @@ -179,6 +186,18 @@ struct ValidateArgs { result_file: PathBuf, } +#[derive(Debug, Args)] +struct MigrateArgs { + #[arg(long)] + manifest: PathBuf, + #[arg(long)] + registry: PathBuf, + #[arg(long)] + batch: Option, + #[arg(long)] + check: bool, +} + #[derive(Debug, Args)] #[command(group( ArgGroup::new("target") @@ -230,6 +249,8 @@ struct SiteArgs { results: PathBuf, #[arg(long)] out: PathBuf, + #[arg(long)] + catalog: Option, #[arg(long, default_value = "https://github.com/devYRPauli/willitcall")] repo_base: String, } @@ -241,16 +262,94 @@ enum ExecuteError { Harness(String), } -fn read_result(path: &Path) -> Result { +enum ResultWire { + V1V2(Box), + V3(Box), +} + +struct EditableResult { + measurement: Measurement, + wire: ResultWire, +} + +impl EditableResult { + fn set_cause(&mut self, index: usize, cause: Cause) { + self.measurement.scenarios[index].cause = Some(cause.clone()); + match &mut self.wire { + ResultWire::V1V2(result) => result.scenarios[index].cause = Some(cause), + ResultWire::V3(result) => result.scenarios[index].cause = Some(cause), + } + } + + fn set_failure_class(&mut self, index: usize, failure_class: String) { + self.measurement.scenarios[index].failure_class = Some(failure_class.clone()); + match &mut self.wire { + ResultWire::V1V2(result) => result.scenarios[index].failure_class = Some(failure_class), + ResultWire::V3(result) => result.scenarios[index].failure_class = Some(failure_class), + } + } +} + +fn read_result(path: &Path) -> Result { let bytes = std::fs::read(path).map_err(|error| { ExecuteError::Usage(format!("failed to read result {}: {error}", path.display())) })?; - parse_and_validate_result(&bytes).map_err(ExecuteError::Usage) + let measurement = parse_and_validate_measurement(&bytes).map_err(ExecuteError::Usage)?; + let wire = match measurement.schema_version { + 1 | 2 => ResultWire::V1V2(Box::new(serde_json::from_slice(&bytes).map_err( + |error| ExecuteError::Usage(format!("invalid result document: {error}")), + )?)), + 3 => ResultWire::V3(Box::new(serde_json::from_slice(&bytes).map_err( + |error| ExecuteError::Usage(format!("invalid result document: {error}")), + )?)), + _ => unreachable!("measurement parser accepts only schema versions 1 through 3"), + }; + Ok(EditableResult { measurement, wire }) } -fn write_updated_result(path: &Path, result: &RunResult) -> Result<(), ExecuteError> { - validate_result(result).map_err(ExecuteError::Usage)?; - write_result_atomic(path, result).map_err(|error| { +fn write_v3_result_atomic(path: &Path, result: &RunResultV3) -> std::io::Result> { + let mut bytes = serde_json::to_vec_pretty(result).map_err(std::io::Error::other)?; + bytes.push(b'\n'); + let parent = match path.parent() { + Some(parent) if !parent.as_os_str().is_empty() => parent, + _ => Path::new("."), + }; + let file_name = path + .file_name() + .and_then(|name| name.to_str()) + .unwrap_or("result.json"); + let suffix = NEXT_TEMP_FILE.fetch_add(1, Ordering::Relaxed); + let temporary_path = parent.join(format!( + ".{file_name}.{}.{}.tmp", + std::process::id(), + suffix + )); + let write_result = (|| { + let mut temporary = std::fs::OpenOptions::new() + .write(true) + .create_new(true) + .open(&temporary_path)?; + temporary.write_all(&bytes)?; + temporary.flush()?; + temporary.sync_all()?; + std::fs::rename(&temporary_path, path) + })(); + if write_result.is_err() { + let _ = std::fs::remove_file(&temporary_path); + } + write_result.map(|()| bytes) +} + +fn write_updated_result(path: &Path, result: &EditableResult) -> Result<(), ExecuteError> { + validate_measurement(&result.measurement).map_err(ExecuteError::Usage)?; + let write_result = match &result.wire { + ResultWire::V1V2(result) => { + validate_result(result).map_err(ExecuteError::Usage)?; + write_result_atomic(path, result) + } + ResultWire::V3(result) => write_v3_result_atomic(path, result).map(|_| ()), + }; + write_result.map_err(|error| { ExecuteError::Harness(format!( "failed to write result {}: {error}", path.display() @@ -267,27 +366,34 @@ fn annotate(args: AnnotateArgs) -> Result { }; let count = if let Some(id) = args.scenario { - let outcome = result + let index = result + .measurement .scenarios - .iter_mut() - .find(|outcome| outcome.id == id) + .iter() + .position(|outcome| outcome.id == id) .ok_or_else(|| ExecuteError::Usage(format!("scenario '{id}' was not found")))?; + let outcome = &result.measurement.scenarios[index]; if outcome.failure_class.as_deref() != Some("empty_response") && !args.force { return Err(ExecuteError::Usage(format!( "scenario '{id}' is not an empty-response failure; use --force to annotate it" ))); } - outcome.cause = Some(cause); + result.set_cause(index, cause); 1 } else { - let mut count = 0; - for outcome in &mut result.scenarios { - if outcome.failure_class.as_deref() == Some("empty_response") { - outcome.cause = Some(cause.clone()); - count += 1; - } + let indexes = result + .measurement + .scenarios + .iter() + .enumerate() + .filter_map(|(index, outcome)| { + (outcome.failure_class.as_deref() == Some("empty_response")).then_some(index) + }) + .collect::>(); + for index in &indexes { + result.set_cause(*index, cause.clone()); } - count + indexes.len() }; if count > 0 { @@ -368,7 +474,8 @@ fn rescore(args: RescoreArgs) -> Result<(usize, Vec), ExecuteError> { let mut changed = 0; let mut unparseable = Vec::new(); - for outcome in &mut result.scenarios { + for index in 0..result.measurement.scenarios.len() { + let outcome = &result.measurement.scenarios[index]; if outcome.status != Status::Fail || outcome.failure_class.is_some() { continue; } @@ -384,7 +491,7 @@ fn rescore(args: RescoreArgs) -> Result<(usize, Vec), ExecuteError> { response.content.as_deref(), &response.tool_calls, ) { - outcome.failure_class = Some(failure_class.to_owned()); + result.set_failure_class(index, failure_class.to_owned()); changed += 1; } } @@ -398,6 +505,58 @@ fn rescore(args: RescoreArgs) -> Result<(usize, Vec), ExecuteError> { Ok((changed, unparseable)) } +fn v3_wire_result(measurement: &Measurement) -> Result { + let metadata = &measurement.metadata; + let corpus = metadata.corpus.clone().ok_or_else(|| { + ExecuteError::Harness("new run is missing required corpus metadata".to_owned()) + })?; + let environment = metadata.environment.clone().ok_or_else(|| { + ExecuteError::Harness("new run is missing required environment metadata".to_owned()) + })?; + Ok(RunResultV3 { + schema_version: 3, + metadata: RunMetadataV3 { + run_id: metadata.run_id.clone(), + timestamp: metadata.timestamp.clone(), + willitcall_version: metadata.willitcall_version.clone(), + endpoint: metadata.endpoint.clone(), + model: metadata.model.clone(), + corpus, + server: ServerMetadataV3 { + preset_name: metadata.server.preset_name.clone(), + reported_version: metadata.server.reported_version.clone(), + quirk_flags: metadata.server.quirk_flags.clone(), + decode_mode: metadata.server.decode_mode, + chat_template: metadata.server.chat_template.clone(), + launch_config_sha256: metadata.server.launch_config_sha256.clone(), + }, + environment, + sampling: metadata.sampling.clone(), + replication: metadata.replication.clone(), + arm_fingerprint: metadata.arm_fingerprint.clone(), + preflight_override: metadata.preflight_override.clone(), + preflight_ignored_ports: metadata.preflight_ignored_ports.clone(), + }, + scenarios: measurement + .scenarios + .iter() + .map(|outcome| ScenarioOutcomeV3 { + id: outcome.id.clone(), + category: outcome.category, + status: outcome.status, + failure_reason: outcome.failure_reason.clone(), + failure: outcome.failure.clone(), + failure_class: outcome.failure_class.clone(), + cause: outcome.cause.clone(), + evidence_hash: outcome.evidence_hash.clone(), + evidence_path: outcome.evidence_path.clone(), + retried: outcome.retried, + }) + .collect(), + totals: measurement.totals.clone(), + }) +} + async fn execute(cli: Cli) -> Result { execute_with_known_servers(cli, KNOWN_INFERENCE_SERVERS).await } @@ -439,6 +598,10 @@ async fn execute_with_known_servers( "another inference server is responding on {endpoints}; {stop}, or re-run with --force" ))); } + let model_registry = ModelRegistry::from_json(MODEL_REGISTRY).map_err(|error| { + ExecuteError::Harness(format!("failed to load model registry: {error}")) + })?; + let declared_quant = args.quant.clone(); let config = RunConfig::new( endpoint, args.model, @@ -448,9 +611,10 @@ async fn execute_with_known_servers( ) .with_server(args.server.config()) .with_host_hardware_class(args.host_hardware_class) - .with_declared_quant(args.quant); + .with_declared_quant(args.quant) + .with_model_registry(model_registry); preflight(&config).await.map_err(ExecuteError::Preflight)?; - let mut result = run_scenarios(&config, &scenarios, &args.out) + let mut result = run_measurement(&config, &scenarios, &args.out) .await .map_err(|error| { ExecuteError::Harness(format!( @@ -458,6 +622,14 @@ async fn execute_with_known_servers( args.out.display() )) })?; + if result.metadata.model.artifact.quantization.is_none() { + result.metadata.model.artifact.quantization = + declared_quant.map(|label| QuantizationMetadata { + label, + scheme: None, + bits: None, + }); + } if args.force && !occupied.is_empty() { result.metadata.preflight_override = Some(PreflightOverride { forced: true, @@ -470,17 +642,18 @@ async fn execute_with_known_servers( if !args.ignore_port.is_empty() { result.metadata.preflight_ignored_ports = Some(args.ignore_port); } - write_result_atomic(&args.out, &result).map_err(|error| { + validate_measurement(&result).map_err(ExecuteError::Harness)?; + let wire_result = v3_wire_result(&result)?; + let document = write_v3_result_atomic(&args.out, &wire_result).map_err(|error| { ExecuteError::Harness(format!( "failed to write result {}: {error}", args.out.display() )) })?; if args.json { - let document = serde_json::to_string_pretty(&result).map_err(|error| { - ExecuteError::Harness(format!("failed to serialize result: {error}")) + std::io::stdout().write_all(&document).map_err(|error| { + ExecuteError::Harness(format!("failed to write JSON output: {error}")) })?; - println!("{document}"); } else { let color = std::io::stdout().is_terminal() && std::env::var_os("NO_COLOR").is_none(); @@ -538,11 +711,41 @@ async fn execute_with_known_servers( path.display() )) })?; - parse_and_validate_result(&bytes).map_err(ExecuteError::Usage)?; + parse_and_validate_measurement(&bytes).map_err(ExecuteError::Usage)?; println!("valid: {}", path.display()); } Ok(0) } + Command::MigrateV3(args) => { + let summary = migrate::run( + &args.manifest, + &args.registry, + args.batch.as_deref(), + args.check, + ) + .map_err(ExecuteError::Usage)?; + let scope = args.batch.as_deref().map_or_else( + || "published files".to_owned(), + |batch| format!("batch {batch}"), + ); + if summary.changed == 0 { + println!( + "{scope}: all {} files are canonical v3; no changes required", + summary.selected + ); + } else if args.check { + println!( + "{scope}: {} of {} files require migration; no files written", + summary.changed, summary.selected + ); + } else { + println!( + "{scope}: migrated {} of {} files to v3", + summary.changed, summary.selected + ); + } + Ok(0) + } Command::Annotate(args) => { let count = annotate(args)?; println!( @@ -565,8 +768,13 @@ async fn execute_with_known_servers( Ok(0) } Command::Site(args) => { - let count = site::generate(&args.results, &args.out, &args.repo_base) - .map_err(ExecuteError::Harness)?; + let count = site::generate( + &args.results, + &args.out, + &args.repo_base, + args.catalog.as_deref(), + ) + .map_err(ExecuteError::Harness)?; println!( "generated {} from {count} result file{}", args.out.display(), @@ -609,7 +817,8 @@ mod tests { use clap::Parser; use wic_core::result::{ - RunMetadata, RunResult, SamplingParams, ScenarioOutcome, ServerMetadata, Status, Totals, + Measurement, RunMetadata, RunResult, SamplingParams, ScenarioOutcome, ServerMetadata, + Status, Totals, }; use wic_core::ScenarioCategory; @@ -976,7 +1185,7 @@ mod tests { }, }; - let rendered = super::report::render_report(&result, false); + let rendered = super::report::render_report(&result.into(), false); assert!(rendered.contains("single_call 1 passed 1 failed 0 errors")); assert!(rendered.contains("streaming 0 passed 0 failed 1 errors")); @@ -1021,6 +1230,7 @@ mod tests { skipped: 0, }, }; + let result = Measurement::from(result); assert!(super::report::render_report(&result, true).contains('\u{1b}')); assert!(!super::report::render_report(&result, false).contains('\u{1b}')); @@ -1036,6 +1246,25 @@ mod tests { assert_eq!(args.result_file, PathBuf::from("result.json")); } + #[test] + fn site_subcommand_accepts_an_explicit_catalog_directory() { + let cli = Cli::try_parse_from([ + "willitcall", + "site", + "--results", + "results", + "--out", + "site", + "--catalog", + "scenarios", + ]) + .expect("site arguments should parse"); + let Command::Site(args) = cli.command else { + panic!("expected site command"); + }; + assert_eq!(args.catalog, Some(PathBuf::from("scenarios"))); + } + #[test] fn help_documents_all_exit_codes() { let help = Cli::try_parse_from(["willitcall", "--help"]) diff --git a/crates/willitcall/src/migrate.rs b/crates/willitcall/src/migrate.rs new file mode 100644 index 0000000..bda985a --- /dev/null +++ b/crates/willitcall/src/migrate.rs @@ -0,0 +1,738 @@ +use std::collections::{HashMap, HashSet}; +use std::io::Write; +use std::path::{Component, Path, PathBuf}; +use std::sync::atomic::{AtomicU64, Ordering}; + +use serde_json::{json, Map, Value}; +use wic_core::corpus::load_frozen_v1_catalog; +use wic_core::registry::ModelRegistry; +use wic_core::result::parse_and_validate_measurement; + +static NEXT_TEMP_FILE: AtomicU64 = AtomicU64::new(0); + +const LLAMA_A: &[&str] = &[ + "results/llamacpp-granite3.1-dense-8b.json", + "results/llamacpp-meta-llama-3.1-8b-instruct-q3_k_m.json", + "results/llamacpp-meta-llama-3.1-8b-instruct-q4_k_m.json", + "results/llamacpp-meta-llama-3.1-8b-instruct-q8_0.json", +]; +const LLAMA_B: &[&str] = &[ + "results/llamacpp-phi4-mini.json", + "results/llamacpp-qwen2.5-1.5b-instruct-q3_k_m.json", + "results/llamacpp-qwen2.5-1.5b-instruct-q4_k_m.json", + "results/llamacpp-qwen2.5-1.5b-instruct-q8_0.json", +]; +const LLAMA_C: &[&str] = &[ + "results/llamacpp-qwen2.5-7b-instruct-q3_k_m.json", + "results/llamacpp-qwen2.5-7b-instruct-q4_k_m.json", + "results/llamacpp-qwen2.5-7b-instruct-q8_0.json", + "results/llamacpp-watt-tool-8b-q4_k_m.json", +]; +const MLX: &[&str] = &[ + "results/mlx_lm-meta-llama-3.1-8b-instruct-4bit.json", + "results/mlx_lm-mistral-7b-instruct-v0.3-4bit.json", + "results/mlx_lm-phi4-mini-4bit.json", + "results/mlx_lm-qwen2.5-1.5b-instruct-4bit.json", + "results/mlx_lm-qwen2.5-7b-instruct-4bit.json", + "results/mlx_lm-qwen2.5-7b-instruct-8bit.json", +]; +const OLLAMA_A: &[&str] = &[ + "results/ollama-gemma3-12b.json", + "results/ollama-gemma3-4b.json", + "results/ollama-granite3.1-dense-8b.json", + "results/ollama-hermes3-8b.json", + "results/ollama-llama3-groq-tool-use-8b.json", +]; +const OLLAMA_B: &[&str] = &[ + "results/ollama-llama3.1-8b.json", + "results/ollama-mistral-7b.json", + "results/ollama-phi4-mini.json", + "results/ollama-qwen2.5-7b-instruct.json", + "results/ollama-qwen3-0.6b.json", +]; +const OLLAMA_C: &[&str] = &[ + "results/ollama-qwen3-1.7b.json", + "results/ollama-qwen3-14b.json", + "results/ollama-qwen3-4b.json", + "results/ollama-qwen3-8b.json", +]; + +#[derive(Debug)] +struct ManifestEntry { + result_path: String, + registry_selector: String, +} + +#[derive(Debug)] +struct Manifest { + entries: Vec, +} + +#[derive(Debug)] +struct PendingWrite { + path: PathBuf, + bytes: Vec, +} + +#[derive(Debug)] +pub(crate) struct MigrationSummary { + pub selected: usize, + pub changed: usize, +} + +pub(crate) fn run( + manifest_path: &Path, + registry_path: &Path, + batch: Option<&str>, + check: bool, +) -> Result { + let manifest_bytes = read_file(manifest_path, "migration manifest")?; + let manifest = parse_manifest(&manifest_bytes)?; + let registry_bytes = read_file(registry_path, "model registry")?; + let registry = ModelRegistry::from_json(®istry_bytes).map_err(|error| error.to_string())?; + + let resolved_paths = resolve_paths(&manifest)?; + refuse_unlisted_json_files(&resolved_paths)?; + let selected = select_entries(&manifest, batch)?; + let corpus = historical_corpus()?; + let mut pending = Vec::new(); + + for entry in &selected { + let path = resolved_paths + .get(entry.result_path.as_str()) + .expect("all manifest entries have resolved paths"); + let expected_model = resolve_manifest_model(®istry, entry)?; + let bytes = read_file(path, "result")?; + let document: Value = serde_json::from_slice(&bytes) + .map_err(|error| format!("invalid result {}: {error}", path.display()))?; + match schema_version(&document, path)? { + 2 => { + let migrated = migrate_v2(&document, &expected_model, &corpus, path)?; + validate_historical_v3(&migrated, &expected_model, &corpus, path)?; + let mut output = serde_json::to_vec_pretty(&migrated) + .map_err(|error| format!("failed to encode {}: {error}", path.display()))?; + output.push(b'\n'); + pending.push(PendingWrite { + path: path.clone(), + bytes: output, + }); + } + 3 => { + let migrated = repair_v3_decode_mode(&document, path)?; + validate_historical_v3(&migrated, &expected_model, &corpus, path)?; + if migrated != document { + let mut output = serde_json::to_vec_pretty(&migrated) + .map_err(|error| format!("failed to encode {}: {error}", path.display()))?; + output.push(b'\n'); + pending.push(PendingWrite { + path: path.clone(), + bytes: output, + }); + } + } + version => { + return Err(format!( + "result {} has schema_version {version}; manifest requires source version 2 or an already-migrated v3 file", + path.display() + )); + } + } + } + + if !check { + for write in &pending { + write_atomic(&write.path, &write.bytes).map_err(|error| { + format!("failed to write result {}: {error}", write.path.display()) + })?; + } + } + + Ok(MigrationSummary { + selected: selected.len(), + changed: pending.len(), + }) +} + +fn read_file(path: &Path, kind: &str) -> Result, String> { + std::fs::read(path) + .map_err(|error| format!("failed to read {kind} {}: {error}", path.display())) +} + +fn parse_manifest(bytes: &[u8]) -> Result { + let document: Value = serde_json::from_slice(bytes) + .map_err(|error| format!("invalid migration manifest: {error}"))?; + let object = required_object(&document, "migration manifest")?; + require_keys( + object, + &[ + "schema_version", + "source_schema_version", + "target_schema_version", + "entries", + ], + "migration manifest", + )?; + require_version(object, "schema_version", 1)?; + require_version(object, "source_schema_version", 2)?; + require_version(object, "target_schema_version", 3)?; + let raw_entries = object + .get("entries") + .and_then(Value::as_array) + .ok_or_else(|| "invalid migration manifest: entries must be an array".to_owned())?; + if raw_entries.is_empty() { + return Err("invalid migration manifest: entries must not be empty".to_owned()); + } + + let mut entries = Vec::with_capacity(raw_entries.len()); + let mut result_paths = HashSet::new(); + for (index, entry) in raw_entries.iter().enumerate() { + let context = format!("migration manifest entry {index}"); + let object = required_object(entry, &context)?; + require_keys( + object, + &["result_path", "registry_selector", "provenance_ref"], + &context, + )?; + let result_path = required_nonempty_string(object, "result_path", &context)?; + let registry_selector = required_nonempty_string(object, "registry_selector", &context)?; + required_nonempty_string(object, "provenance_ref", &context)?; + if !result_paths.insert(result_path.clone()) { + return Err(format!( + "invalid migration manifest: duplicate result_path {result_path:?}" + )); + } + entries.push(ManifestEntry { + result_path, + registry_selector, + }); + } + Ok(Manifest { entries }) +} + +fn required_object<'a>(value: &'a Value, context: &str) -> Result<&'a Map, String> { + value + .as_object() + .ok_or_else(|| format!("invalid {context}: expected an object")) +} + +fn require_keys( + object: &Map, + expected: &[&str], + context: &str, +) -> Result<(), String> { + let actual = object.keys().map(String::as_str).collect::>(); + let expected = expected.iter().copied().collect::>(); + if actual != expected { + return Err(format!( + "invalid {context}: expected exactly the fields {}", + expected.iter().copied().collect::>().join(", ") + )); + } + Ok(()) +} + +fn require_version(object: &Map, field: &str, expected: u64) -> Result<(), String> { + let actual = object.get(field).and_then(Value::as_u64); + if actual != Some(expected) { + return Err(format!( + "invalid migration manifest: {field} must be {expected}" + )); + } + Ok(()) +} + +fn required_nonempty_string( + object: &Map, + field: &str, + context: &str, +) -> Result { + object + .get(field) + .and_then(Value::as_str) + .filter(|value| !value.trim().is_empty()) + .map(str::to_owned) + .ok_or_else(|| format!("invalid {context}: {field} must be a non-empty string")) +} + +fn resolve_paths(manifest: &Manifest) -> Result, String> { + let current = std::env::current_dir() + .map_err(|error| format!("failed to resolve current directory: {error}"))?; + let mut paths = HashMap::new(); + let mut normalized = HashSet::new(); + for entry in &manifest.entries { + let raw = Path::new(&entry.result_path); + if raw.extension().is_none_or(|extension| extension != "json") { + return Err(format!( + "invalid migration manifest: result_path {:?} is not a JSON file", + entry.result_path + )); + } + let path = normalize_path(if raw.is_absolute() { + raw.to_path_buf() + } else { + current.join(raw) + }); + if !normalized.insert(path.clone()) { + return Err(format!( + "invalid migration manifest: multiple result paths resolve to {}", + path.display() + )); + } + paths.insert(entry.result_path.as_str(), path); + } + Ok(paths) +} + +fn normalize_path(path: PathBuf) -> PathBuf { + let mut normalized = PathBuf::new(); + for component in path.components() { + match component { + Component::CurDir => {} + Component::ParentDir => { + normalized.pop(); + } + other => normalized.push(other.as_os_str()), + } + } + normalized +} + +fn refuse_unlisted_json_files(paths: &HashMap<&str, PathBuf>) -> Result<(), String> { + let listed = paths.values().cloned().collect::>(); + let parents = paths + .values() + .filter_map(|path| path.parent().map(Path::to_path_buf)) + .collect::>(); + for parent in parents { + let entries = std::fs::read_dir(&parent).map_err(|error| { + format!( + "failed to inspect result directory {}: {error}", + parent.display() + ) + })?; + for entry in entries { + let path = entry + .map_err(|error| { + format!( + "failed to inspect result directory {}: {error}", + parent.display() + ) + })? + .path(); + if path.is_file() + && path + .extension() + .is_some_and(|extension| extension == "json") + && !listed.contains(&normalize_path(path.clone())) + { + return Err(format!( + "refusing unlisted result file {}; add it explicitly to the migration manifest", + path.display() + )); + } + } + } + Ok(()) +} + +fn select_entries<'a>( + manifest: &'a Manifest, + batch: Option<&str>, +) -> Result, String> { + let Some(batch) = batch else { + return Ok(manifest.entries.iter().collect()); + }; + let expected = match batch { + "llama-a" => LLAMA_A, + "llama-b" => LLAMA_B, + "llama-c" => LLAMA_C, + "mlx" => MLX, + "ollama-a" => OLLAMA_A, + "ollama-b" => OLLAMA_B, + "ollama-c" => OLLAMA_C, + _ => { + return Err(format!( + "unknown migration batch {batch:?}; expected llama-a, llama-b, llama-c, mlx, ollama-a, ollama-b, or ollama-c" + )); + } + }; + let by_path = manifest + .entries + .iter() + .map(|entry| (entry.result_path.as_str(), entry)) + .collect::>(); + expected + .iter() + .map(|path| { + by_path.get(path).copied().ok_or_else(|| { + format!("migration batch {batch:?} requires manifest entry {path:?}") + }) + }) + .collect() +} + +fn resolve_manifest_model( + registry: &ModelRegistry, + entry: &ManifestEntry, +) -> Result { + let is_listed = registry.entries.iter().any(|registry_entry| { + registry_entry + .selectors + .iter() + .any(|selector| selector == &entry.registry_selector) + }); + if !is_listed { + return Err(format!( + "manifest selector {:?} for {:?} is not explicitly listed in the registry", + entry.registry_selector, entry.result_path + )); + } + serde_json::to_value(registry.resolve(&entry.registry_selector)) + .map_err(|error| format!("failed to encode registry identity: {error}")) +} + +fn historical_corpus() -> Result { + let catalog = load_frozen_v1_catalog().map_err(|error| error.to_string())?; + Ok(json!({ + "id": catalog.id, + "revision": catalog.revision, + "sha256": catalog.sha256, + "scenario_count": catalog.scenario_count, + "scoring_version": "v1", + })) +} + +fn schema_version(document: &Value, path: &Path) -> Result { + document + .get("schema_version") + .and_then(Value::as_u64) + .ok_or_else(|| { + format!( + "invalid result {}: schema_version must be an unsigned integer", + path.display() + ) + }) +} + +fn migrate_v2( + document: &Value, + model: &Value, + corpus: &Value, + path: &Path, +) -> Result { + let source_bytes = serde_json::to_vec(document) + .map_err(|error| format!("failed to inspect {}: {error}", path.display()))?; + wic_core::result::parse_and_validate_result(&source_bytes) + .map_err(|error| format!("invalid result {}: {error}", path.display()))?; + let root = required_object(document, &format!("result {}", path.display()))?; + let metadata = required_object( + root.get("metadata") + .ok_or_else(|| format!("invalid result {}: missing metadata", path.display()))?, + &format!("result {} metadata", path.display()), + )?; + let server = required_object( + metadata + .get("server") + .ok_or_else(|| format!("invalid result {}: missing server", path.display()))?, + &format!("result {} server", path.display()), + )?; + let environment = required_object( + metadata + .get("environment") + .ok_or_else(|| format!("result {} has no historical environment", path.display()))?, + &format!("result {} environment", path.display()), + )?; + let hardware = environment + .get("host_hardware_class") + .and_then(Value::as_str) + .ok_or_else(|| { + format!( + "invalid result {}: environment.host_hardware_class must be a string", + path.display() + ) + })?; + let host_os = environment + .get("host_os") + .and_then(Value::as_str) + .ok_or_else(|| { + format!( + "invalid result {}: environment.host_os must be a string", + path.display() + ) + })?; + + let scenarios = root + .get("scenarios") + .and_then(Value::as_array) + .expect("validated v2 result has scenarios") + .iter() + .map(migrate_scenario) + .collect::, _>>()?; + let decode_mode = decode_mode_from_quirk_flags(server, path)?; + let mut migrated_metadata = Map::new(); + copy_required(metadata, &mut migrated_metadata, "run_id", path)?; + copy_required(metadata, &mut migrated_metadata, "timestamp", path)?; + copy_required(metadata, &mut migrated_metadata, "willitcall_version", path)?; + copy_required(metadata, &mut migrated_metadata, "endpoint", path)?; + migrated_metadata.insert("model".to_owned(), model.clone()); + migrated_metadata.insert("corpus".to_owned(), corpus.clone()); + migrated_metadata.insert( + "server".to_owned(), + json!({ + "preset_name": server.get("preset_name").expect("validated v2 server"), + "reported_version": server.get("reported_version").expect("validated v2 server"), + "quirk_flags": server.get("quirk_flags").expect("validated v2 server"), + "decode_mode": decode_mode, + "chat_template": null, + "launch_config_sha256": null, + }), + ); + migrated_metadata.insert( + "environment".to_owned(), + json!({ + "display_label": format!("{hardware}; {host_os}"), + "os_name": null, + "os_version": null, + "architecture": null, + "accelerator": null, + "memory_bytes": null, + }), + ); + copy_required(metadata, &mut migrated_metadata, "sampling", path)?; + migrated_metadata.insert("replication".to_owned(), Value::Null); + migrated_metadata.insert("arm_fingerprint".to_owned(), Value::Null); + copy_optional(metadata, &mut migrated_metadata, "preflight_override"); + copy_optional(metadata, &mut migrated_metadata, "preflight_ignored_ports"); + + Ok(json!({ + "schema_version": 3, + "metadata": migrated_metadata, + "scenarios": scenarios, + "totals": root.get("totals").expect("validated v2 result has totals"), + })) +} + +fn repair_v3_decode_mode(document: &Value, path: &Path) -> Result { + let mut migrated = document.clone(); + let server = migrated + .get_mut("metadata") + .and_then(Value::as_object_mut) + .and_then(|metadata| metadata.get_mut("server")) + .and_then(Value::as_object_mut) + .ok_or_else(|| { + format!( + "invalid result {}: server must be an object", + path.display() + ) + })?; + let decode_mode = decode_mode_from_quirk_flags(server, path)?; + if decode_mode != "unknown" + && server.get("decode_mode").and_then(Value::as_str) == Some("unknown") + { + server.insert( + "decode_mode".to_owned(), + Value::String(decode_mode.to_owned()), + ); + } + Ok(migrated) +} + +fn decode_mode_from_quirk_flags( + server: &Map, + path: &Path, +) -> Result<&'static str, String> { + let quirk_flags = server + .get("quirk_flags") + .and_then(Value::as_array) + .ok_or_else(|| { + format!( + "invalid result {}: server.quirk_flags must be an array", + path.display() + ) + })?; + let has_flag = |expected: &str| { + quirk_flags + .iter() + .any(|flag| flag.as_str() == Some(expected)) + }; + match ( + has_flag("grammar_constrained_decoding"), + has_flag("unconstrained_post_hoc_parse"), + ) { + (true, false) => Ok("grammar_constrained"), + (false, true) => Ok("unconstrained_post_hoc"), + (false, false) => Ok("unknown"), + (true, true) => Err(format!( + "invalid result {}: server.quirk_flags record conflicting decode modes", + path.display() + )), + } +} + +fn migrate_scenario(scenario: &Value) -> Result { + let source = required_object(scenario, "scenario outcome")?; + let mut migrated = Map::new(); + for field in ["id", "category", "status", "failure_reason"] { + copy_required(source, &mut migrated, field, Path::new("scenario outcome"))?; + } + migrated.insert("failure".to_owned(), Value::Null); + copy_optional(source, &mut migrated, "failure_class"); + copy_optional(source, &mut migrated, "cause"); + for field in ["evidence_hash", "evidence_path", "retried"] { + copy_required(source, &mut migrated, field, Path::new("scenario outcome"))?; + } + Ok(Value::Object(migrated)) +} + +fn copy_required( + source: &Map, + target: &mut Map, + field: &str, + path: &Path, +) -> Result<(), String> { + let value = source + .get(field) + .ok_or_else(|| format!("invalid {}: missing {field}", path.display()))?; + target.insert(field.to_owned(), value.clone()); + Ok(()) +} + +fn copy_optional(source: &Map, target: &mut Map, field: &str) { + if let Some(value) = source.get(field) { + target.insert(field.to_owned(), value.clone()); + } +} + +fn validate_historical_v3( + document: &Value, + expected_model: &Value, + expected_corpus: &Value, + path: &Path, +) -> Result<(), String> { + if schema_version(document, path)? != 3 { + return Err(format!("result {} is not schema v3", path.display())); + } + let metadata = document + .get("metadata") + .and_then(Value::as_object) + .ok_or_else(|| { + format!( + "invalid result {}: metadata must be an object", + path.display() + ) + })?; + if metadata.get("model") != Some(expected_model) { + return Err(format!( + "result {} model identity does not match its manifest registry selector", + path.display() + )); + } + if metadata.get("corpus") != Some(expected_corpus) { + return Err(format!( + "result {} corpus metadata is not the frozen historical corpus", + path.display() + )); + } + for field in ["replication", "arm_fingerprint"] { + if !metadata.get(field).is_some_and(Value::is_null) { + return Err(format!( + "result {} historical metadata.{field} must be null", + path.display() + )); + } + } + let server = metadata + .get("server") + .and_then(Value::as_object) + .ok_or_else(|| { + format!( + "invalid result {}: server must be an object", + path.display() + ) + })?; + let expected_decode_mode = decode_mode_from_quirk_flags(server, path)?; + if server.get("decode_mode").and_then(Value::as_str) != Some(expected_decode_mode) + || !server.get("chat_template").is_some_and(Value::is_null) + || !server + .get("launch_config_sha256") + .is_some_and(Value::is_null) + { + return Err(format!( + "result {} invents unrecoverable historical server metadata", + path.display() + )); + } + let scenarios = document + .get("scenarios") + .and_then(Value::as_array) + .ok_or_else(|| { + format!( + "invalid result {}: scenarios must be an array", + path.display() + ) + })?; + if scenarios + .iter() + .any(|scenario| !scenario.get("failure").is_some_and(Value::is_null)) + { + return Err(format!( + "result {} invents structured historical failure data", + path.display() + )); + } + + let mut wire_compatible = document.clone(); + let compatible_metadata = wire_compatible + .get_mut("metadata") + .and_then(Value::as_object_mut) + .expect("metadata was checked above"); + compatible_metadata.insert( + "replication".to_owned(), + json!({ + "study_id": "unresolved:historical", + "arm_id": "unresolved:historical", + "run_index": 0, + "mode": "greedy_reproducibility", + }), + ); + compatible_metadata.insert( + "arm_fingerprint".to_owned(), + Value::String("unresolved:historical".to_owned()), + ); + let bytes = serde_json::to_vec(&wire_compatible) + .map_err(|error| format!("failed to validate {}: {error}", path.display()))?; + parse_and_validate_measurement(&bytes) + .map_err(|error| format!("invalid result {}: {error}", path.display()))?; + Ok(()) +} + +fn write_atomic(path: &Path, bytes: &[u8]) -> std::io::Result<()> { + let parent = match path.parent() { + Some(parent) if !parent.as_os_str().is_empty() => parent, + _ => Path::new("."), + }; + let file_name = path + .file_name() + .and_then(|name| name.to_str()) + .unwrap_or("result.json"); + let suffix = NEXT_TEMP_FILE.fetch_add(1, Ordering::Relaxed); + let temporary_path = parent.join(format!( + ".{file_name}.{}.{}.tmp", + std::process::id(), + suffix + )); + let write_result = (|| { + let mut temporary = std::fs::OpenOptions::new() + .write(true) + .create_new(true) + .open(&temporary_path)?; + temporary.write_all(bytes)?; + temporary.flush()?; + temporary.sync_all()?; + std::fs::rename(&temporary_path, path) + })(); + if write_result.is_err() { + let _ = std::fs::remove_file(&temporary_path); + } + write_result +} diff --git a/crates/willitcall/src/report.rs b/crates/willitcall/src/report.rs index ef6e088..ae251be 100644 --- a/crates/willitcall/src/report.rs +++ b/crates/willitcall/src/report.rs @@ -1,4 +1,4 @@ -use wic_core::result::{RunResult, Status}; +use wic_core::result::{Measurement, Status}; use wic_core::ScenarioCategory; const CATEGORIES: [ScenarioCategory; 6] = [ @@ -10,7 +10,7 @@ const CATEGORIES: [ScenarioCategory; 6] = [ ScenarioCategory::NegativeTrap, ]; -pub fn render_report(result: &RunResult, color: bool) -> String { +pub fn render_report(result: &Measurement, color: bool) -> String { let mut rendered = String::new(); if color { rendered.push_str("\x1b[1mwillitcall report\x1b[0m\n"); diff --git a/crates/willitcall/src/site.rs b/crates/willitcall/src/site.rs index 8ffe9c5..b316c75 100644 --- a/crates/willitcall/src/site.rs +++ b/crates/willitcall/src/site.rs @@ -1,36 +1,20 @@ -use std::collections::BTreeSet; -use std::fmt::Write as _; use std::fs; use std::path::{Path, PathBuf}; -use wic_core::result::{ - parse_and_validate_result, CauseKind, EnvironmentMetadata, RunResult, Status, -}; -use wic_core::ScenarioCategory; - -const CATEGORIES: [ScenarioCategory; 6] = [ - ScenarioCategory::SingleCall, - ScenarioCategory::ParallelCalls, - ScenarioCategory::Streaming, - ScenarioCategory::ToolChoiceModes, - ScenarioCategory::MultiTurn, - ScenarioCategory::NegativeTrap, -]; - -struct ResultFile { - file_name: String, - result: RunResult, -} +mod analysis; +mod data; +mod export; +mod html; +mod svg; pub(crate) fn generate( results_directory: &Path, output_directory: &Path, repo_base: &str, + catalog_directory: Option<&Path>, ) -> Result { - let results = read_results(results_directory)?; + let dataset = data::load(results_directory, catalog_directory)?; let repo_base = repo_base.trim_end_matches('/'); - let index = render_index(&results, repo_base); - let submit = render_submit(repo_base); fs::create_dir_all(output_directory).map_err(|error| { format!( @@ -38,731 +22,36 @@ pub(crate) fn generate( output_directory.display() ) })?; - write_site_file(output_directory.join("index.html"), &index)?; - write_site_file(output_directory.join("submit.html"), &submit)?; - write_site_file(output_directory.join("style.css"), STYLE)?; - write_site_file(output_directory.join("site.js"), SCRIPT)?; - Ok(results.len()) -} - -fn read_results(directory: &Path) -> Result, String> { - let entries = fs::read_dir(directory).map_err(|error| { - format!( - "failed to read results directory {}: {error}", - directory.display() - ) - })?; - let mut paths = entries - .map(|entry| { - entry - .map(|entry| entry.path()) - .map_err(|error| format!("failed to read results directory entry: {error}")) - }) - .collect::, _>>()?; - paths.retain(|path| { - path.is_file() - && path - .extension() - .is_some_and(|extension| extension == "json") - }); - paths.sort(); - - paths - .into_iter() - .map(|path| { - let bytes = fs::read(&path) - .map_err(|error| format!("failed to read result {}: {error}", path.display()))?; - let result = parse_and_validate_result(&bytes) - .map_err(|error| format!("{}: {error}", path.display()))?; - let file_name = path - .file_name() - .ok_or_else(|| format!("result path {} has no file name", path.display()))? - .to_string_lossy() - .into_owned(); - Ok(ResultFile { file_name, result }) - }) - .collect() + write_site_file( + output_directory.join("index.html"), + &html::render_index(&dataset, repo_base), + )?; + write_site_file( + output_directory.join("outcomes.html"), + &html::render_outcomes(&dataset), + )?; + write_site_file( + output_directory.join("appendix.html"), + &html::render_appendix_page(&dataset, repo_base), + )?; + write_site_file( + output_directory.join("submit.html"), + &html::render_submit(repo_base), + )?; + write_site_file( + output_directory.join("results.json"), + &export::render_json(&dataset)?, + )?; + write_site_file( + output_directory.join("results.csv"), + &export::render_csv(&dataset), + )?; + write_site_file(output_directory.join("style.css"), html::STYLE)?; + write_site_file(output_directory.join("site.js"), html::SCRIPT)?; + Ok(dataset.rows.len()) } fn write_site_file(path: PathBuf, contents: &str) -> Result<(), String> { fs::write(&path, contents) .map_err(|error| format!("failed to write site file {}: {error}", path.display())) } - -fn render_index(results: &[ResultFile], repo_base: &str) -> String { - let scenario_count = results - .iter() - .flat_map(|result| result.result.scenarios.iter()) - .map(|scenario| scenario.id.as_str()) - .collect::>() - .len(); - let case_studies_url = format!("{repo_base}/tree/main/docs/case-studies"); - let peg_native_case_study_url = format!( - "{repo_base}/blob/main/docs/case-studies/2026-07-21-llamacpp-500s-on-llama-3.1-tool-calls.md" - ); - let uniform_environment = results - .first() - .and_then(|result| result.result.metadata.environment.as_ref()) - .filter(|environment| { - results - .iter() - .all(|result| result.result.metadata.environment.as_ref() == Some(*environment)) - }); - let environment_statement = uniform_environment - .map(render_environment_statement) - .unwrap_or_default(); - let mut html = String::new(); - write!( - html, - r#" - - - - - - willitcall support matrix - - - -
- willitcall - -
-
-
-

Measured compatibility

-

Tool-calling support matrix

-

A cell measures the whole stack: model x quant x server x server version. It is not a property of the model alone.

-

Red means the combination failed as tested, not that the weights are bad. The same weights can pass on one server and fail on another; where that is proven, the cell carries a cause annotation.

-

Every red cell links to the full request/response transcript that produced it when the result schema supplies a transcript path. Legacy schema v1 results do not record transcript paths. See the case studies under docs/case-studies/ for controlled comparisons.

-

The servers do not decode the same way. llama.cpp compiles the supplied tool definitions into a GBNF grammar and constrains decoding with it, so a call naming a function that was never supplied cannot be sampled there. Ollama and MLX LM generate unconstrained text and parse the tool call out of it afterwards. This systematically favours llama.cpp, so a llama.cpp-versus-Ollama difference is a property of the combination, not evidence of a server defect or a difference between models; the comparison that isolates the model is same-server.

-

Sample size and method: {} distinct scenarios are represented in this result set. Each published cell is one run. Findings in the case studies are replicated across at least five runs per arm before a verdict is drawn, so a cell tells you what one run measured and a case study tells you what held up under repetition. The current case studies cover 90 runs across 18 quantization arms, and 40 runs across 8 arms for the peg-native anomaly.

-

Excluded rows

-
    -
  • Meta-Llama-3.1-8B-Instruct on llama.cpp (Q8_0, Q4_K_M, Q3_K_M) is excluded from the quantization conclusion because llama.cpp returns HTTP 500 on 7-9 of 50 scenarios per run for this model ("does not match the expected peg-native format"). These are server errors, not model failures, and are not comparable across arms. See the peg-native case study.
    • -
-{} -
- -
-
-
-

Current results

-

Scenario groups

-
- -
-
- all pass - partial - none pass -
-

Showing {} result files.

-
- - - - -"#, - escape_html(&case_studies_url), - scenario_count, - escape_html(&peg_native_case_study_url), - environment_statement, - results.len() - ) - .expect("write HTML"); - for category in CATEGORIES { - writeln!( - html, - " ", - category - ) - .expect("write HTML"); - } - html.push_str( - r#" - -"#, - ); - - for (index, result_file) in results.iter().enumerate() { - render_result_rows( - &mut html, - index, - result_file, - repo_base, - uniform_environment.is_none(), - ); - } - - html.push_str( - r#"
Model / quant / server{}
-
-
-
-
-

Read ratios as passed scenarios / total scenarios in the category.

-
- - - -"#, - ); - html -} - -fn render_result_rows( - html: &mut String, - index: usize, - result_file: &ResultFile, - repo_base: &str, - disclose_environment: bool, -) { - let result = &result_file.result; - let server = &result.metadata.server.preset_name; - let server_display = display_server(server); - let model = model_label(&result_file.file_name, server); - let quant = result - .metadata - .declared_quant - .as_deref() - .unwrap_or("not declared"); - let details_id = format!("result-details-{index}"); - let environment_metadata = if disclose_environment { - let environment = result.metadata.environment.as_ref(); - format!( - "
Host hardware
{}
\n
Host OS
{}
\n", - escape_html( - environment - .map(|environment| environment.host_hardware_class.as_str()) - .unwrap_or("not recorded") - ), - escape_html( - environment - .map(|environment| environment.host_os.as_str()) - .unwrap_or("not recorded") - ) - ) - } else { - String::new() - }; - write!( - html, - " \n \n \n {}\n quant: {}\n server: {}\n \n", - escape_html(server), - escape_html(&model), - escape_html(quant), - escape_html(server_display) - ) - .expect("write HTML"); - - for category in CATEGORIES { - render_category_cell(html, result_file, category, repo_base); - } - - write!( - html, - " \n \n \n
\n View {} scenarios and row metadata\n
\n
Result file
{}
\n
Model id
{}
\n
Declared quant
{}
\n
Server
{} {}
\n
Schema
v{}
\n
Run time
{}
\n{}
\n
    \n", - result.scenarios.len(), - escape_html(&result_file.file_name), - escape_html(&result.metadata.model_id), - escape_html(quant), - escape_html(server_display), - escape_html( - result - .metadata - .server - .reported_version - .as_deref() - .unwrap_or("version not reported") - ), - result.schema_version, - escape_html(&result.metadata.timestamp), - environment_metadata - ) - .expect("write HTML"); - - for scenario in &result.scenarios { - let status = display_status(scenario.status); - write!( - html, - "
  1. {} {status}", - escape_html(&scenario.id) - ) - .expect("write HTML"); - if let Some(reason) = scenario.failure_reason.as_deref() { - write!( - html, - " {}", - escape_html(reason) - ) - .expect("write HTML"); - } - if let Some(evidence_path) = scenario.evidence_path.as_deref() { - let url = evidence_url(repo_base, evidence_path); - write!( - html, - " transcript", - escape_html(&url) - ) - .expect("write HTML"); - } - render_annotation(html, result.schema_version, scenario, repo_base); - html.push_str("
    2. \n"); - } - - html.push_str( - r#"
-
- - - -"#, - ); -} - -fn render_environment_statement(environment: &EnvironmentMetadata) -> String { - format!( - "

Measurement environment: {}; {}.

", - escape_html(&environment.host_hardware_class), - escape_html(&environment.host_os) - ) -} - -fn render_category_cell( - html: &mut String, - result_file: &ResultFile, - category: ScenarioCategory, - repo_base: &str, -) { - let scenarios = result_file - .result - .scenarios - .iter() - .filter(|scenario| scenario.category == category) - .collect::>(); - let total = scenarios.len(); - let passed = scenarios - .iter() - .filter(|scenario| scenario.status == Status::Pass) - .count(); - let class = if total == 0 { - "untested" - } else if passed == total { - "all-pass" - } else if passed == 0 { - "none-pass" - } else { - "partial" - }; - let first_evidence = scenarios - .iter() - .find(|scenario| { - scenario.status != Status::Pass && scenario.evidence_path.as_deref().is_some() - }) - .and_then(|scenario| scenario.evidence_path.as_deref()); - write!( - html, - " ", - category - ) - .expect("write HTML"); - if let Some(evidence_path) = first_evidence { - write!( - html, - "{passed}/{total}", - escape_html(&evidence_url(repo_base, evidence_path)) - ) - .expect("write HTML"); - } else { - write!(html, "{passed}/{total}").expect("write HTML"); - if total > 0 && passed < total && result_file.result.schema_version == 1 { - html.push_str("schema v1: no transcript path"); - } - } - html.push_str("\n"); -} - -fn render_annotation( - html: &mut String, - schema_version: u32, - scenario: &wic_core::result::ScenarioOutcome, - repo_base: &str, -) { - if let Some(cause) = scenario.cause.as_ref() { - let label = match cause.kind { - CauseKind::ServerDefect => "server defect", - CauseKind::Unknown => "cause unknown", - }; - let title = cause.note.as_deref().unwrap_or(label); - if let Some(reference) = cause.reference.as_deref() { - let reference = reference_url(repo_base, reference); - write!( - html, - " {label}", - escape_html(&reference), - escape_html(title) - ) - .expect("write HTML"); - } else { - write!( - html, - " {label}", - escape_html(title) - ) - .expect("write HTML"); - } - } - if scenario.failure_class.as_deref() == Some("empty_response") { - html.push_str(" empty response"); - } else if scenario.failure_class.as_deref() == Some("unparsed_tool_call") { - html.push_str(" unparsed tool call"); - } else if schema_version == 1 - && scenario.cause.is_none() - && scenario.status != Status::Pass - && scenario.evidence_hash.is_some() - { - html.push_str(" legacy evidence hash only"); - } -} - -fn render_submit(repo_base: &str) -> String { - let contributing_url = format!("{repo_base}/blob/main/CONTRIBUTING.md"); - format!( - r#" - - - - - - Submit a result - willitcall - - - -
- willitcall - -
-
-
-

Submission method

-

Produce a new result cell

-

Run one model at a time. Keep the result file and the evidence directory written beside it.

-
-
-

Ollama

- 
MODEL=qwen2.5:7b-instruct
-OUT=results/ollama-qwen2.5-7b-instruct.json
-cargo run -p willitcall -- run \
-  --model "$MODEL" \
-  --server ollama \
-  --out "$OUT"
-cargo run -p willitcall -- validate "$OUT"
-
-
-

llama.cpp

- 
MODEL_PATH=/absolute/path/to/model.Q4_K_M.gguf
-OUT=results/llamacpp-model-q4_k_m.json
-cargo run -p willitcall -- run \
-  --model "$MODEL_PATH" \
-  --server llamacpp \
-  --out "$OUT"
-cargo run -p willitcall -- validate "$OUT"
-
-
-

Pull request checklist

-
    -
  • preflight clean (no contention override), or the override is explained
    • -
  • result file schema-valid
    • -
  • evidence transcripts included
    • -
  • empty responses cross-checked on a second server per the seeding protocol
    • -
-

Read CONTRIBUTING.md for the complete contribution rules.

-
-
-

Results are reviewed as measured stack behavior, not model-only claims.

- - -"#, - escape_html(&contributing_url) - ) -} - -fn model_label(file_name: &str, server: &str) -> String { - let stem = file_name.strip_suffix(".json").unwrap_or(file_name); - stem.strip_prefix(&format!("{server}-")) - .unwrap_or(stem) - .to_owned() -} - -fn display_server(server: &str) -> &str { - if server == "llamacpp" { - "llama.cpp" - } else if server == "mlx_lm" { - "MLX LM" - } else { - server - } -} - -fn display_status(status: Status) -> &'static str { - match status { - Status::Pass => "pass", - Status::Fail => "fail", - Status::Error => "error", - Status::Skipped => "skipped", - } -} - -fn evidence_url(repo_base: &str, evidence_path: &str) -> String { - format!( - "{repo_base}/blob/main/results/{}", - evidence_path.trim_start_matches('/') - ) -} - -fn reference_url(repo_base: &str, reference: &str) -> String { - if reference.starts_with("https://") || reference.starts_with("http://") { - reference.to_owned() - } else { - format!( - "{repo_base}/blob/main/{}", - reference.trim_start_matches('/') - ) - } -} - -fn escape_html(value: &str) -> String { - let mut escaped = String::with_capacity(value.len()); - for character in value.chars() { - match character { - '&' => escaped.push_str("&"), - '<' => escaped.push_str("<"), - '>' => escaped.push_str(">"), - '"' => escaped.push_str("""), - '\'' => escaped.push_str("'"), - character if character.is_ascii() => escaped.push(character), - character => write!(escaped, "&#{};", character as u32).expect("write entity"), - } - } - escaped -} - -const SCRIPT: &str = r#"const filter = document.getElementById("server-filter"); -const groups = Array.from(document.querySelectorAll(".result-group")); -const status = document.getElementById("filter-status"); - -filter.addEventListener("change", () => { - let shown = 0; - for (const group of groups) { - const visible = filter.value === "all" || group.dataset.server === filter.value; - group.hidden = !visible; - if (visible) shown += 1; - } - status.textContent = `Showing ${shown} result ${shown === 1 ? "file" : "files"}.`; -}); -"#; - -const STYLE: &str = r#":root { - color-scheme: light; - --ink: #17212b; - --muted: #52606d; - --line: #c9d2da; - --paper: #f7f8f9; - --panel: #ffffff; - --accent: #135f69; - --pass-bg: #d8efdf; - --pass-ink: #17452a; - --partial-bg: #fff0bf; - --partial-ink: #594200; - --none-bg: #f5d8dc; - --none-ink: #681f29; - --neutral-bg: #e8edf1; - --neutral-ink: #35434f; - font-family: Inter, ui-sans-serif, system-ui, -apple-system, BlinkMacSystemFont, "Segoe UI", sans-serif; - font-size: 16px; - line-height: 1.55; -} - -* { box-sizing: border-box; } - -body { - margin: 0; - color: var(--ink); - background: var(--paper); -} - -a { color: #075f8a; text-underline-offset: 0.16em; } -a:hover { text-decoration-thickness: 2px; } -a:focus-visible, select:focus-visible, summary:focus-visible { - outline: 3px solid #e5901a; - outline-offset: 3px; -} - -.site-header { - display: flex; - align-items: center; - justify-content: space-between; - gap: 2rem; - padding: 1rem max(1.25rem, calc((100vw - 90rem) / 2)); - color: #ffffff; - background: #15313a; - border-bottom: 4px solid #4ca1a9; -} - -.wordmark { - color: #ffffff; - font-family: ui-monospace, SFMono-Regular, Menlo, Consolas, monospace; - font-size: 1.1rem; - font-weight: 800; - text-decoration: none; - letter-spacing: 0.04em; -} - -nav { display: flex; gap: 1.25rem; } -nav a { color: #dcebed; font-weight: 650; text-decoration: none; } -nav a[aria-current="page"] { color: #ffffff; text-decoration: underline; } - -main, footer { - width: min(90rem, calc(100% - 2.5rem)); - margin-inline: auto; -} - -.methods { - max-width: 75rem; - padding: 4rem 0 2.5rem; -} - -.methods p:not(.eyebrow) { max-width: 76ch; font-size: 1.06rem; } - -.eyebrow { - margin: 0 0 0.4rem; - color: var(--accent); - font-size: 0.78rem; - font-weight: 800; - letter-spacing: 0.12em; - text-transform: uppercase; -} - -h1, h2 { margin: 0 0 1rem; line-height: 1.12; } -h1 { font-size: clamp(2.2rem, 5vw, 4.4rem); letter-spacing: -0.045em; } -h2 { font-size: clamp(1.45rem, 2.5vw, 2.1rem); letter-spacing: -0.025em; } - -.matrix { - margin-bottom: 4rem; - padding: 1.5rem; - background: var(--panel); - border: 1px solid var(--line); - box-shadow: 0 10px 30px rgb(23 33 43 / 8%); -} - -.matrix-heading { - display: flex; - align-items: end; - justify-content: space-between; - gap: 2rem; -} - -label { color: var(--muted); font-size: 0.84rem; font-weight: 750; } -select { - display: block; - min-width: 11rem; - margin-top: 0.35rem; - padding: 0.65rem 2.25rem 0.65rem 0.75rem; - color: var(--ink); - background: #ffffff; - border: 1px solid #81909c; - border-radius: 0.2rem; - font: inherit; -} - -.legend { display: flex; flex-wrap: wrap; gap: 1.25rem; margin: 1.25rem 0 0; color: var(--muted); font-size: 0.82rem; } -.legend span { display: inline-flex; align-items: center; gap: 0.4rem; } -.swatch { width: 0.85rem; height: 0.85rem; border: 1px solid rgb(23 33 43 / 25%); } -.swatch.all-pass, .score.all-pass { color: var(--pass-ink); background: var(--pass-bg); } -.swatch.partial, .score.partial { color: var(--partial-ink); background: var(--partial-bg); } -.swatch.none-pass, .score.none-pass { color: var(--none-ink); background: var(--none-bg); } -.score.untested { color: var(--neutral-ink); background: var(--neutral-bg); } - -.filter-status { margin: 0.7rem 0 1rem; color: var(--muted); font-size: 0.86rem; } -.table-scroll { overflow-x: auto; border: 1px solid var(--line); } -table { width: 100%; min-width: 70rem; border-collapse: collapse; } -th, td { padding: 0.85rem; text-align: left; border: 1px solid var(--line); } -thead th { color: #ffffff; background: #284852; font-size: 0.78rem; } -thead code { color: inherit; } -.result-row > th { width: 18rem; background: #f1f4f6; } -.result-row > th strong, .result-row > th span { display: block; } -.result-row > th strong { margin-bottom: 0.35rem; font-size: 0.98rem; } -.result-row > th span { color: var(--muted); font-size: 0.78rem; font-weight: 500; } -.score { min-width: 8rem; text-align: center; } -.ratio { display: block; color: inherit; font: 800 1.05rem/1.2 ui-monospace, SFMono-Regular, Menlo, Consolas, monospace; } -.legacy-evidence { display: block; margin-top: 0.35rem; font-size: 0.68rem; line-height: 1.25; } -.detail-row > td { padding: 0; background: #fbfcfc; } -.detail-row details { padding: 0.8rem 1rem; } -.detail-row summary { width: fit-content; color: #075f8a; cursor: pointer; font-weight: 700; } - -.metadata { - display: grid; - grid-template-columns: repeat(auto-fit, minmax(13rem, 1fr)); - gap: 0.75rem; - margin: 1rem 0; -} -.metadata div { min-width: 0; padding: 0.7rem; background: #eef2f4; } -.metadata dt { color: var(--muted); font-size: 0.7rem; font-weight: 800; text-transform: uppercase; } -.metadata dd { margin: 0.2rem 0 0; overflow-wrap: anywhere; } -.scenario-list { margin: 1rem 0 0; padding-left: 1.75rem; } -.scenario { padding: 0.5rem 0 0.5rem 0.25rem; border-bottom: 1px solid #e0e5e9; } -.scenario:last-child { border-bottom: 0; } -.status-label { margin-left: 0.4rem; font-size: 0.72rem; font-weight: 850; text-transform: uppercase; } -.status-pass .status-label { color: #236d3d; } -.status-fail .status-label, .status-error .status-label { color: #9a2535; } -.failure-reason { display: inline; color: var(--muted); } -.failure-reason::before { content: "- "; } -.transcript { margin-left: 0.55rem; font-size: 0.85rem; } -.badge { display: inline-block; margin-left: 0.45rem; padding: 0.12rem 0.42rem; border-radius: 999px; font-size: 0.7rem; font-weight: 800; text-decoration: none; } -.badge.cause { color: #632014; background: #ffe0d4; border: 1px solid #e6a18d; } -.badge.neutral { color: var(--neutral-ink); background: var(--neutral-bg); border: 1px solid #bcc7cf; } -.badge.neutral.unparsed { color: #4a3410; background: #fdf0d5; border: 1px solid #d9b877; } - -.submit-page { max-width: 58rem; } -.submit-page section { margin-bottom: 2.5rem; } -pre { overflow-x: auto; padding: 1.25rem; color: #eef7f8; background: #18343d; border-left: 4px solid #4ca1a9; } -code { font-family: ui-monospace, SFMono-Regular, Menlo, Consolas, monospace; } -.checklist { padding-left: 1.3rem; } -.checklist li { margin-bottom: 0.65rem; } - -footer { padding: 1.5rem 0 3rem; color: var(--muted); border-top: 1px solid var(--line); font-size: 0.86rem; } - -@media (max-width: 44rem) { - .site-header, .matrix-heading { align-items: flex-start; flex-direction: column; gap: 1rem; } - .site-header { padding-inline: 1.25rem; } - main, footer { width: min(100% - 1.5rem, 90rem); } - .methods { padding-top: 2.5rem; } - .matrix { padding: 1rem; } - nav { gap: 1rem; } -} - -@media print { - body { background: #ffffff; } - .site-header { color: #000000; background: #ffffff; border-color: #000000; } - .wordmark, nav a { color: #000000; } - .matrix { box-shadow: none; } - label, .filter-status { display: none; } - .table-scroll { overflow: visible; } - table { min-width: 0; font-size: 9pt; } - a { color: inherit; } -} -"#; diff --git a/crates/willitcall/src/site/analysis.rs b/crates/willitcall/src/site/analysis.rs new file mode 100644 index 0000000..f15a155 --- /dev/null +++ b/crates/willitcall/src/site/analysis.rs @@ -0,0 +1,764 @@ +use std::collections::BTreeMap; +use std::fmt::Write as _; + +use wic_core::result::{DecodeMode, Status}; +use wic_core::ScenarioCategory; + +use super::data::{SiteDataset, StackRow, CATEGORIES}; +use super::html; +use super::svg::{self, Figure, FigureAccessibility}; + +const MARK_SIZE: u32 = 13; +const CELL_STEP: u32 = 15; +const ROW_STEP: u32 = 16; + +pub(super) fn render_outcomes(dataset: &SiteDataset) -> String { + let mut figures = String::from( + "
\n

Scenario evidence

\n

The complete raster shows all 50 scenarios. The focused raster shows the seven multi-turn scenarios. The signature inventory groups identical 50-outcome vectors. Figure captions link to the complete text alternative in JSON and CSV.

\n", + ); + figures.push_str(&render_scenario_raster(dataset)); + figures.push_str(&render_multi_turn_raster(dataset)); + figures.push_str(&render_signature_inventory(dataset)); + figures.push_str("
\n"); + figures +} + +pub(super) fn render_pass_counts(dataset: &SiteDataset) -> String { + let mut figures = String::from( + "
\n

One-run observations

\n

Observed pass counts

\n

Pass counts are shown in fixed stack order and withheld for rows with errors or skips. They are not scores or a ranking.

\n", + ); + figures.push_str(&render_pass_count_strip(dataset)); + figures.push_str("
\n"); + figures +} + +#[derive(Clone)] +struct ScenarioColumn { + id: String, + category: ScenarioCategory, +} + +struct PreparedMark { + x: u32, + y: u32, + label: String, + state: svg::MarkState, +} + +struct PreparedText { + x: u32, + y: u32, + text: String, + class: &'static str, + anchor: svg::TextAnchor, + rotation: Option, +} + +fn render_scenario_raster(dataset: &SiteDataset) -> String { + let columns = scenario_columns(dataset); + let row_indices = stable_row_indices(dataset); + let left = 320; + let top = 170; + let width = left + columns.len() as u32 * CELL_STEP + 20; + let height = top + row_indices.len() as u32 * ROW_STEP + 20; + let (texts, rules) = raster_axes(&columns, &row_indices, dataset, left, top, height); + let mut prepared = Vec::new(); + + for (row_position, row_index) in row_indices.iter().copied().enumerate() { + let row = &dataset.rows[row_index]; + for (column_position, column) in columns.iter().enumerate() { + let status = scenario_status(row, &column.id); + let Some(status) = status else { + continue; + }; + prepared.push(PreparedMark { + x: left + column_position as u32 * CELL_STEP, + y: top + row_position as u32 * ROW_STEP, + label: column.id.clone(), + state: mark_state(status), + }); + } + } + let marks = borrow_marks(&prepared); + let texts = borrow_texts(&texts); + let caption = "Rows show published stack runs. Columns show scenario ids grouped by capability. We omit the n=1 hatch because every raster row is a single run. The hatch would mark every cell and add no distinction."; + let mut figure = html::render_figure(Figure::StatusRaster { + accessibility: FigureAccessibility { + number: 1, + title: "Scenario-status raster", + description: "Four-state outcomes for every published stack and scenario. Cell titles contain only the scenario id and outcome; full text is linked in JSON and CSV.", + caption, + does_not_show: "This figure does not rank stacks or assign an overall score. It does not predict how a stack will behave across repeated runs.", + }, + width, + height, + marks: &marks, + texts: &texts, + rules: &rules, + }); + let mobile = render_mobile_scenario_rasters(dataset, &columns, &row_indices); + figure = figure.replacen("", &format!("{mobile}"), 1); + figure +} + +fn render_mobile_scenario_rasters( + dataset: &SiteDataset, + columns: &[ScenarioColumn], + row_indices: &[usize], +) -> String { + let mut html = String::from( + "
\n", + ); + for category in CATEGORIES { + let category_columns = columns + .iter() + .filter(|column| column.category == category) + .collect::>(); + write!( + html, + "

{}

\n Stack", + category, + category_label(category), + category_columns.len(), + ) + .expect("write mobile raster"); + for column in &category_columns { + write!( + html, + "{}", + html::escape_html(&column.id), + ) + .expect("write mobile raster column"); + } + html.push('\n'); + for row_index in row_indices { + let row = &dataset.rows[*row_index]; + let stack = stack_label(row); + write!( + html, + " {}", + html::escape_html(&stack), + ) + .expect("write mobile raster row label"); + for column in &category_columns { + if let Some(status) = scenario_status(row, &column.id) { + let label = format!("{}: {}", column.id, status_label(status)); + write!( + html, + "", + status_label(status), + html::escape_html(&label), + html::escape_html(&label), + ) + .expect("write mobile raster mark"); + } else { + html.push_str(""); + } + } + html.push('\n'); + } + html.push_str("
\n"); + } + html.push_str("
\n"); + html +} + +fn render_multi_turn_raster(dataset: &SiteDataset) -> String { + let columns = scenario_columns(dataset) + .into_iter() + .filter(|column| column.category == ScenarioCategory::MultiTurn) + .collect::>(); + let row_indices = stable_row_indices(dataset); + let left = 320; + let top = 170; + let width = left + columns.len() as u32 * CELL_STEP + 20; + let height = top + row_indices.len() as u32 * ROW_STEP + 20; + let (texts, rules) = raster_axes(&columns, &row_indices, dataset, left, top, height); + let mut prepared = Vec::new(); + let mut passed = 0; + let mut total = 0; + let mut zero_pass_rows = 0; + + for (row_position, row_index) in row_indices.iter().copied().enumerate() { + let row = &dataset.rows[row_index]; + let row_passes = row + .scenarios + .iter() + .filter(|scenario| { + scenario.category == ScenarioCategory::MultiTurn && scenario.status == Status::Pass + }) + .count(); + if row_passes == 0 { + zero_pass_rows += 1; + } + for (column_position, column) in columns.iter().enumerate() { + let status = scenario_status(row, &column.id); + if let Some(status) = status { + total += 1; + if status == Status::Pass { + passed += 1; + } + prepared.push(PreparedMark { + x: left + column_position as u32 * CELL_STEP, + y: top + row_position as u32 * ROW_STEP, + label: column.id.clone(), + state: mark_state(status), + }); + } + } + } + let percentage = if total == 0 { + 0 + } else { + (passed * 100 + total / 2) / total + }; + let caption = format!( + "In these published observations, multi_turn passes {passed}/{total} ({percentage}%). Of {} rows, {zero_pass_rows} pass none of the multi_turn scenarios.", + row_indices.len(), + ); + let marks = borrow_marks(&prepared); + let texts = borrow_texts(&texts); + html::render_figure(Figure::StatusRaster { + accessibility: FigureAccessibility { + number: 2, + title: "Multi-turn raster", + description: "Four-state outcomes for the multi_turn scenarios across every published stack run.", + caption: &caption, + does_not_show: "This figure does not explain why a turn failed. It does not isolate model and server effects or measure reliability across repeated runs.", + }, + width, + height, + marks: &marks, + texts: &texts, + rules: &rules, + }) +} + +fn render_signature_inventory(dataset: &SiteDataset) -> String { + let columns = scenario_columns(dataset); + let row_indices = stable_row_indices(dataset); + let mut grouped = BTreeMap::>::new(); + for row_index in row_indices { + let row = &dataset.rows[row_index]; + grouped + .entry(signature_key(row, &columns)) + .or_default() + .push(row_index); + } + + let left = 100; + let names_x = left + columns.len() as u32 * CELL_STEP + 18; + let top = 170; + let mut y = top; + let mut prepared = Vec::new(); + let mut owned_texts = Vec::new(); + let mut repeated_seven_signature = None; + + for (signature_index, members) in grouped.values().enumerate() { + let signature_name = format!("Signature {:02}", signature_index + 1); + let member_labels = members + .iter() + .map(|index| stack_label(&dataset.rows[*index])) + .collect::>(); + let row_height = (member_labels.len() as u32 * 14 + 4).max(20); + owned_texts.push(PreparedText { + x: left - 8, + y: y + 11, + text: signature_name.clone(), + class: "raster-row-label", + anchor: svg::TextAnchor::End, + rotation: None, + }); + for (member_index, label) in member_labels.iter().enumerate() { + owned_texts.push(PreparedText { + x: names_x, + y: y + 11 + member_index as u32 * 14, + text: label.clone(), + class: "signature-stack-label", + anchor: svg::TextAnchor::Start, + rotation: None, + }); + } + + let representative = &dataset.rows[members[0]]; + for (column_index, column) in columns.iter().enumerate() { + if let Some(status) = scenario_status(representative, &column.id) { + prepared.push(PreparedMark { + x: left + column_index as u32 * CELL_STEP, + y, + label: column.id.clone(), + state: mark_state(status), + }); + } + } + + let statuses = columns + .iter() + .filter_map(|column| { + scenario_status(representative, &column.id).map(|status| (column.category, status)) + }) + .collect::>(); + let passes = statuses + .iter() + .filter(|(_, status)| *status == Status::Pass) + .count(); + let negative_trap_passes = statuses + .iter() + .filter(|(category, status)| { + *category == ScenarioCategory::NegativeTrap && *status == Status::Pass + }) + .count(); + let tool_choice_passes = statuses + .iter() + .filter(|(category, status)| { + *category == ScenarioCategory::ToolChoiceModes && *status == Status::Pass + }) + .count(); + let includes_granite = members.iter().any(|index| { + dataset.rows[*index] + .display_name + .to_ascii_lowercase() + .contains("granite") + }); + if passes == 7 && members.len() > 1 && includes_granite { + repeated_seven_signature = + Some((members.len(), negative_trap_passes, tool_choice_passes)); + } + y += row_height; + } + + let width = names_x + 620; + let height = y + 15; + let (axis_texts, rules) = column_axes(&columns, left, top, height); + owned_texts.extend(axis_texts); + let caption = repeated_seven_signature.map_or_else( + || "Each row shows one exact status vector. The names on the right identify the stack observations that share it.".to_owned(), + |(count, negative_trap_passes, tool_choice_passes)| format!("Each row shows one exact status vector. {count} stacks share the repeated 7-pass signature. The stacks include the granite observations. Every row has the same {negative_trap_passes} negative_trap passes and {tool_choice_passes} tool_choice_modes passes."), + ); + let marks = borrow_marks(&prepared); + let texts = borrow_texts(&owned_texts); + html::render_figure(Figure::StatusRaster { + accessibility: FigureAccessibility { + number: 3, + title: "Outcome-signature inventory", + description: "Distinct exact scenario-status vectors and the published stack observations sharing each vector.", + caption: &caption, + does_not_show: "This figure does not establish shared model identity or a common cause. It does not predict future behavior.", + }, + width, + height, + marks: &marks, + texts: &texts, + rules: &rules, + }) +} + +fn render_pass_count_strip(dataset: &SiteDataset) -> String { + let row_indices = stable_row_indices(dataset); + let (fully_measurable, not_fully_measurable): (Vec<_>, Vec<_>) = + row_indices.into_iter().partition(|index| { + let counts = &dataset.rows[*index].category_counts; + counts.errors == 0 && counts.skipped == 0 + }); + let max_count = dataset + .rows + .iter() + .map(|row| row.category_counts.passed + row.category_counts.failed) + .max() + .unwrap_or(1) + .max(1); + let plot_left = 80; + let plot_width = 800; + let axis_y = 48; + let mut pass_occurrences = BTreeMap::::new(); + let mut prepared_dots = Vec::new(); + + for row_index in fully_measurable.iter().copied() { + let row = &dataset.rows[row_index]; + let passed = row.category_counts.passed; + let occurrence = pass_occurrences.entry(passed).or_default(); + let y = axis_y + 20 + *occurrence as u32 * 13; + *occurrence += 1; + prepared_dots.push(( + plot_left + passed as u32 * plot_width / max_count as u32, + y, + passed, + format!("{}: {passed} passes", stack_label(row)), + )); + } + + let max_overlap = pass_occurrences.values().copied().max().unwrap_or(1) as u32; + let not_panel_y = axis_y + 35 + max_overlap * 13; + let mut owned_texts = vec![PreparedText { + x: plot_left, + y: 20, + text: "Fully measurable observations".to_owned(), + class: "plot-panel-label", + anchor: svg::TextAnchor::Start, + rotation: None, + }]; + let mut rules = vec![svg::FigureRule { + x1: plot_left, + y1: axis_y, + x2: plot_left + plot_width, + y2: axis_y, + class: "strip-axis", + }]; + let step = (max_count / 5).max(1); + let mut tick = 0; + while tick <= max_count { + add_strip_tick( + &mut owned_texts, + &mut rules, + plot_left, + plot_width, + axis_y, + max_count, + tick, + ); + tick += step; + } + if (max_count / step) * step != max_count { + add_strip_tick( + &mut owned_texts, + &mut rules, + plot_left, + plot_width, + axis_y, + max_count, + max_count, + ); + } + owned_texts.push(PreparedText { + x: plot_left, + y: not_panel_y, + text: "Not fully measurable - pass counts withheld".to_owned(), + class: "plot-panel-label", + anchor: svg::TextAnchor::Start, + rotation: None, + }); + rules.push(svg::FigureRule { + x1: plot_left, + y1: not_panel_y + 8, + x2: plot_left + plot_width, + y2: not_panel_y + 8, + class: "panel-rule", + }); + + let mut prepared_not_measurable = Vec::new(); + for (position, row_index) in not_fully_measurable.iter().copied().enumerate() { + let row = &dataset.rows[row_index]; + let y = not_panel_y + 22 + position as u32 * 20; + let detail = format!( + "{} errors, {} skipped; pass count not plotted", + row.category_counts.errors, row.category_counts.skipped + ); + prepared_not_measurable.push((plot_left, y, stack_label(row), detail.clone())); + owned_texts.push(PreparedText { + x: plot_left + 20, + y: y + 11, + text: format!("{} - {detail}", stack_label(row)), + class: "not-measurable-label", + anchor: svg::TextAnchor::Start, + rotation: None, + }); + } + + let dots = prepared_dots + .iter() + .map(|(x, y, pass_count, label)| svg::StripDot { + x: *x, + y: *y, + pass_count: *pass_count, + label, + }) + .collect::>(); + let not_measurable_marks = prepared_not_measurable + .iter() + .map(|(x, y, label, detail)| svg::NotFullyMeasurableMark { + x: *x, + y: *y, + label, + detail, + }) + .collect::>(); + let texts = borrow_texts(&owned_texts); + let height = not_panel_y + 35 + not_fully_measurable.len() as u32 * 20; + html::render_figure(Figure::StripPlot { + accessibility: FigureAccessibility { + number: 4, + title: "Observed pass-count strip plot", + description: "One pass-count dot per fully measurable published stack observation, with error-bearing or skipped rows listed separately and not assigned a pass count.", + caption: "Each dot shows one published stack observation from a single run. We plot a pass count only when every scenario has a pass or fail verdict. A separate panel lists rows with errors or skips.", + does_not_show: "This figure does not show a model score distribution or rank models. It does not measure uncertainty or variation across repeated runs.", + }, + width: plot_left + plot_width + 40, + height, + dots: &dots, + not_fully_measurable: ¬_measurable_marks, + texts: &texts, + rules: &rules, + }) +} + +fn raster_axes( + columns: &[ScenarioColumn], + row_indices: &[usize], + dataset: &SiteDataset, + left: u32, + top: u32, + height: u32, +) -> (Vec, Vec) { + let (mut texts, rules) = column_axes(columns, left, top, height); + for (row_position, row_index) in row_indices.iter().copied().enumerate() { + texts.push(PreparedText { + x: left - 8, + y: top + row_position as u32 * ROW_STEP + 11, + text: stack_label(&dataset.rows[row_index]), + class: "raster-row-label", + anchor: svg::TextAnchor::End, + rotation: None, + }); + } + (texts, rules) +} + +fn column_axes( + columns: &[ScenarioColumn], + left: u32, + top: u32, + height: u32, +) -> (Vec, Vec) { + let mut texts = Vec::new(); + let mut rules = Vec::new(); + for (column_position, column) in columns.iter().enumerate() { + let x = left + column_position as u32 * CELL_STEP + MARK_SIZE / 2; + texts.push(PreparedText { + x, + y: top - 12, + text: column.id.clone(), + class: "raster-column-label", + anchor: svg::TextAnchor::Start, + rotation: Some(-60), + }); + } + for category in CATEGORIES { + let positions = columns + .iter() + .enumerate() + .filter(|(_, column)| column.category == category) + .map(|(index, _)| index) + .collect::>(); + let (Some(first), Some(last)) = (positions.first(), positions.last()) else { + continue; + }; + let start = left + *first as u32 * CELL_STEP; + let end = left + (*last as u32 + 1) * CELL_STEP; + texts.push(PreparedText { + x: start + (end - start) / 2, + y: 16, + text: category.to_string(), + class: "raster-group-label", + anchor: svg::TextAnchor::Middle, + rotation: None, + }); + rules.push(svg::FigureRule { + x1: start, + y1: 24, + x2: start, + y2: height - 10, + class: "category-rule", + }); + } + (texts, rules) +} + +fn add_strip_tick( + texts: &mut Vec, + rules: &mut Vec, + plot_left: u32, + plot_width: u32, + axis_y: u32, + max_count: usize, + value: usize, +) { + let x = plot_left + value as u32 * plot_width / max_count as u32; + texts.push(PreparedText { + x, + y: axis_y - 7, + text: value.to_string(), + class: "strip-tick-label", + anchor: svg::TextAnchor::Middle, + rotation: None, + }); + rules.push(svg::FigureRule { + x1: x, + y1: axis_y - 4, + x2: x, + y2: axis_y + 5, + class: "strip-tick", + }); +} + +fn scenario_columns(dataset: &SiteDataset) -> Vec { + let mut columns = Vec::new(); + for category in CATEGORIES { + let mut category_columns = BTreeMap::::new(); + for scenario in dataset + .rows + .iter() + .flat_map(|row| row.scenarios.iter()) + .filter(|scenario| scenario.category == category) + { + category_columns + .entry(scenario.id.clone()) + .or_insert_with(|| ScenarioColumn { + id: scenario.id.clone(), + category, + }); + } + columns.extend(category_columns.into_values()); + } + columns +} + +fn stable_row_indices(dataset: &SiteDataset) -> Vec { + let mut indices = (0..dataset.rows.len()).collect::>(); + indices.sort_by_key(|index| stable_stack_key(&dataset.rows[*index])); + indices +} + +fn stable_stack_key(row: &StackRow) -> String { + let model = &row.metadata.model; + let artifact = &model.artifact; + let quantization = artifact + .quantization + .as_ref() + .map(|quantization| quantization.label.as_str()) + .unwrap_or(""); + format!( + "{}\0{}\0{}\0{}\0{}\0{}\0{}\0{}\0{}\0{}", + model.canonical_id.as_deref().unwrap_or(""), + row.display_name, + decode_order(row.decode_mode), + row.metadata.server.preset_name, + artifact.source_id.as_deref().unwrap_or(""), + artifact.sha256.as_deref().unwrap_or(""), + quantization, + row.metadata + .server + .reported_version + .as_deref() + .unwrap_or(""), + row.endpoint_display, + row.file_name, + ) +} + +fn decode_order(mode: DecodeMode) -> u8 { + match mode { + DecodeMode::GrammarConstrained => 0, + DecodeMode::UnconstrainedPostHoc => 1, + DecodeMode::Unknown => 2, + } +} + +fn category_label(category: ScenarioCategory) -> &'static str { + match category { + ScenarioCategory::SingleCall => "Single call", + ScenarioCategory::ToolChoiceModes => "Tool choice", + ScenarioCategory::NegativeTrap => "Correctly declines", + ScenarioCategory::MultiTurn => "Multi-turn", + ScenarioCategory::ParallelCalls => "Parallel calls", + ScenarioCategory::Streaming => "Streaming", + } +} + +fn stack_label(row: &StackRow) -> String { + let quantization = row + .metadata + .model + .artifact + .quantization + .as_ref() + .map(|quantization| quantization.label.as_str()) + .unwrap_or("quant not declared"); + format!( + "{} | {} | {}", + row.display_name, + quantization, + display_server(&row.metadata.server.preset_name) + ) +} + +fn display_server(server: &str) -> &str { + match server { + "llamacpp" => "llama.cpp", + "mlx_lm" => "MLX LM", + other => other, + } +} + +fn scenario_status(row: &StackRow, scenario_id: &str) -> Option { + row.scenarios + .iter() + .find(|scenario| scenario.id == scenario_id) + .map(|scenario| scenario.status) +} + +fn signature_key(row: &StackRow, columns: &[ScenarioColumn]) -> String { + columns + .iter() + .map(|column| match scenario_status(row, &column.id) { + Some(Status::Pass) => 'P', + Some(Status::Fail) => 'F', + Some(Status::Error) => 'E', + Some(Status::Skipped) => 'S', + None => '-', + }) + .collect() +} + +fn borrow_marks(prepared: &[PreparedMark]) -> Vec> { + prepared + .iter() + .map(|mark| svg::StatusMark { + x: mark.x, + y: mark.y, + label: &mark.label, + state: mark.state, + }) + .collect() +} + +fn borrow_texts(prepared: &[PreparedText]) -> Vec> { + prepared + .iter() + .map(|text| svg::FigureText { + x: text.x, + y: text.y, + text: &text.text, + class: text.class, + anchor: text.anchor, + rotation: text.rotation, + }) + .collect() +} + +fn mark_state(status: Status) -> svg::MarkState { + match status { + Status::Pass => svg::MarkState::Pass, + Status::Fail => svg::MarkState::Fail, + Status::Error => svg::MarkState::Error, + Status::Skipped => svg::MarkState::Skipped, + } +} + +fn status_label(status: Status) -> &'static str { + match status { + Status::Pass => "pass", + Status::Fail => "fail", + Status::Error => "error", + Status::Skipped => "skipped", + } +} diff --git a/crates/willitcall/src/site/data.rs b/crates/willitcall/src/site/data.rs new file mode 100644 index 0000000..7f63b25 --- /dev/null +++ b/crates/willitcall/src/site/data.rs @@ -0,0 +1,924 @@ +use std::collections::{BTreeMap, BTreeSet, HashMap, HashSet}; +use std::fs; +use std::path::{Path, PathBuf}; + +use wic_core::corpus::{corpus_identity, load_frozen_v1_catalog}; +use wic_core::result::{ + parse_and_validate_measurement, Cause, DecodeMode, IdentityStatus, Measurement, + MeasurementMetadata, ReplicationMode, ScenarioFailure, Status, +}; +use wic_core::{load_scenarios_from_dir, Scenario, ScenarioCategory}; + +const DECODE_MODE_REGISTRY: &[u8] = include_bytes!("../../../../registry/decode-modes-v1.json"); + +pub(super) const CATEGORIES: [ScenarioCategory; 6] = [ + ScenarioCategory::SingleCall, + ScenarioCategory::ParallelCalls, + ScenarioCategory::Streaming, + ScenarioCategory::ToolChoiceModes, + ScenarioCategory::MultiTurn, + ScenarioCategory::NegativeTrap, +]; + +const CATALOG_UNAVAILABLE: &str = "catalog unavailable"; +const REPLICATION_GATE: usize = 5; + +#[allow(dead_code)] // Brief 8 consumes the catalog prose and replicated studies. +pub(super) struct SiteDataset { + pub(super) rows: Vec, + pub(super) scenario_count: usize, + pub(super) studies: Vec, +} + +pub(super) struct StackRow { + pub(super) file_name: String, + pub(super) schema_version: u32, + pub(super) metadata: MeasurementMetadata, + pub(super) display_name: String, + pub(super) endpoint_display: String, + pub(super) decode_mode: DecodeMode, + pub(super) decode_mode_source: DecodeModeSource, + pub(super) scenarios: Vec, + pub(super) category_counts: CategoryCounts, +} + +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub(super) enum DecodeModeSource { + Recorded, + PresetMapping, + Unknown, +} + +struct DecodeModeRegistry { + by_preset: HashMap, +} + +impl DecodeModeRegistry { + fn from_json(bytes: &[u8]) -> Result { + let document: serde_json::Value = serde_json::from_slice(bytes) + .map_err(|error| format!("invalid decode mode registry: {error}"))?; + let root = document + .as_object() + .ok_or_else(|| "invalid decode mode registry: expected an object".to_owned())?; + if root.len() != 2 || !root.contains_key("schema_version") || !root.contains_key("entries") + { + return Err( + "invalid decode mode registry: expected schema_version and entries".to_owned(), + ); + } + if root + .get("schema_version") + .and_then(serde_json::Value::as_u64) + != Some(1) + { + return Err("invalid decode mode registry: schema_version must be 1".to_owned()); + } + let entries = root + .get("entries") + .and_then(serde_json::Value::as_array) + .filter(|entries| !entries.is_empty()) + .ok_or_else(|| { + "invalid decode mode registry: entries must be a non-empty array".to_owned() + })?; + let mut by_preset = HashMap::new(); + for (index, entry) in entries.iter().enumerate() { + let entry = entry.as_object().ok_or_else(|| { + format!("invalid decode mode registry entry {index}: expected an object") + })?; + if entry.len() != 3 + || !entry.contains_key("preset_name") + || !entry.contains_key("decode_mode") + || !entry.contains_key("provenance_refs") + { + return Err(format!( + "invalid decode mode registry entry {index}: expected preset_name, decode_mode, and provenance_refs" + )); + } + let preset_name = entry + .get("preset_name") + .and_then(serde_json::Value::as_str) + .filter(|value| !value.trim().is_empty()) + .ok_or_else(|| { + format!( + "invalid decode mode registry entry {index}: preset_name must be non-empty" + ) + })?; + let decode_mode = match entry.get("decode_mode").and_then(serde_json::Value::as_str) { + Some("grammar_constrained") => DecodeMode::GrammarConstrained, + Some("unconstrained_post_hoc") => DecodeMode::UnconstrainedPostHoc, + _ => { + return Err(format!( + "invalid decode mode registry entry {index}: decode_mode must establish a known mode" + )); + } + }; + entry + .get("provenance_refs") + .and_then(serde_json::Value::as_array) + .filter(|references| { + !references.is_empty() + && references.iter().all(|reference| { + reference + .as_str() + .is_some_and(|value| !value.trim().is_empty()) + }) + }) + .ok_or_else(|| { + format!( + "invalid decode mode registry entry {index}: provenance_refs must contain non-empty references" + ) + })?; + if by_preset + .insert(preset_name.to_owned(), decode_mode) + .is_some() + { + return Err(format!( + "invalid decode mode registry: duplicate preset_name {preset_name:?}" + )); + } + } + Ok(Self { by_preset }) + } +} + +impl StackRow { + pub(super) fn cross_model_key(&self) -> Option<&str> { + if self.metadata.model.identity_status == IdentityStatus::Unresolved { + None + } else { + self.metadata.model.canonical_id.as_deref() + } + } +} + +impl SiteDataset { + pub(super) fn replication_count(&self, row_index: usize) -> usize { + let row = &self.rows[row_index]; + let (Some(replication), Some(fingerprint)) = ( + row.metadata.replication.as_ref(), + row.metadata.arm_fingerprint.as_ref(), + ) else { + return 1; + }; + self.rows + .iter() + .filter(|candidate| { + candidate.metadata.arm_fingerprint.as_ref() == Some(fingerprint) + && candidate + .metadata + .replication + .as_ref() + .is_some_and(|other| { + other.study_id == replication.study_id + && other.arm_id == replication.arm_id + && other.mode == replication.mode + }) + }) + .map(|candidate| { + candidate + .metadata + .replication + .as_ref() + .expect("filtered replicated row") + .run_index + }) + .collect::>() + .len() + .max(1) + } +} + +#[allow(dead_code)] // Aggregate fields are inputs to the analysis views in brief 8. +pub(super) struct CategoryCounts { + pub(super) passed: usize, + pub(super) failed: usize, + pub(super) errors: usize, + pub(super) skipped: usize, + pub(super) measurement_coverage: f64, + pub(super) macro_category_pass_rate: Option, + categories: [StatusCounts; CATEGORIES.len()], +} + +impl CategoryCounts { + fn from_scenarios(scenarios: &[ScenarioView]) -> Self { + let mut categories = [StatusCounts::default(); CATEGORIES.len()]; + for scenario in scenarios { + let counts = &mut categories[category_index(scenario.category)]; + match scenario.status { + Status::Pass => counts.passed += 1, + Status::Fail => counts.failed += 1, + Status::Error => counts.errors += 1, + Status::Skipped => counts.skipped += 1, + } + } + + let passed = categories.iter().map(|counts| counts.passed).sum(); + let failed = categories.iter().map(|counts| counts.failed).sum(); + let errors = categories.iter().map(|counts| counts.errors).sum(); + let skipped = categories.iter().map(|counts| counts.skipped).sum(); + let total = passed + failed + errors + skipped; + let measurement_coverage = if total == 0 { + 0.0 + } else { + (passed + failed) as f64 / total as f64 + }; + let macro_category_pass_rate = if errors > 0 || skipped > 0 { + None + } else { + let rates = categories + .iter() + .filter_map(|counts| { + let measured = counts.passed + counts.failed; + (measured > 0).then_some(counts.passed as f64 / measured as f64) + }) + .collect::>(); + (!rates.is_empty()).then(|| rates.iter().sum::() / rates.len() as f64) + }; + + Self { + passed, + failed, + errors, + skipped, + measurement_coverage, + macro_category_pass_rate, + categories, + } + } + + pub(super) fn for_category(&self, category: ScenarioCategory) -> StatusCounts { + self.categories[category_index(category)] + } +} + +#[derive(Clone, Copy, Default)] +pub(super) struct StatusCounts { + pub(super) passed: usize, + pub(super) failed: usize, + pub(super) errors: usize, + pub(super) skipped: usize, +} + +impl StatusCounts { + pub(super) fn total(self) -> usize { + self.passed + self.failed + self.errors + self.skipped + } +} + +#[allow(dead_code)] // Description, rationale, and structured failure land in brief 8. +pub(super) struct ScenarioView { + pub(super) id: String, + pub(super) category: ScenarioCategory, + pub(super) definition: Option, + pub(super) description: String, + pub(super) rationale: String, + pub(super) status: Status, + pub(super) failure_detail: Option, + pub(super) failure_reason: Option, + pub(super) failure_class: Option, + pub(super) cause: Option, + pub(super) evidence_hash: Option, + pub(super) evidence_path: Option, + pub(super) retried: bool, +} + +#[allow(dead_code)] // Rendered analysis views are introduced in brief 8. +pub(super) struct StudyView { + pub(super) study_id: String, + pub(super) arms: Vec, +} + +#[allow(dead_code)] +pub(super) struct StudyArm { + pub(super) arm_id: String, + pub(super) arm_fingerprint: String, + pub(super) mode: ReplicationMode, + pub(super) row_indices: Vec, +} + +pub(super) fn load( + results_directory: &Path, + catalog_directory: Option<&Path>, +) -> Result { + let decode_modes = DecodeModeRegistry::from_json(DECODE_MODE_REGISTRY)?; + let frozen = Catalog::from_scenarios( + load_frozen_v1_catalog() + .map_err(|error| format!("failed to load frozen wic-50-v1 catalog: {error}"))? + .scenarios, + )?; + let custom = catalog_directory + .map(|directory| { + let scenarios = load_scenarios_from_dir(directory).map_err(|error| { + format!( + "failed to load catalog directory {}: {error}", + directory.display() + ) + })?; + Catalog::from_scenarios(scenarios) + }) + .transpose()?; + + let mut rows = Vec::new(); + for path in result_paths(results_directory)? { + let bytes = fs::read(&path) + .map_err(|error| format!("failed to read result {}: {error}", path.display()))?; + let measurement = parse_and_validate_measurement(&bytes) + .map_err(|error| format!("{}: {error}", path.display()))?; + let file_name = path + .file_name() + .ok_or_else(|| format!("result path {} has no file name", path.display()))? + .to_string_lossy() + .into_owned(); + let catalog = select_catalog(&file_name, &measurement, &frozen, custom.as_ref())?; + rows.push(stack_row(file_name, measurement, catalog, &decode_modes)?); + } + + let scenario_count = rows + .iter() + .flat_map(|row| row.scenarios.iter()) + .map(|scenario| scenario.id.as_str()) + .collect::>() + .len(); + let studies = study_views(&rows); + Ok(SiteDataset { + rows, + scenario_count, + studies, + }) +} + +fn result_paths(directory: &Path) -> Result, String> { + let entries = fs::read_dir(directory).map_err(|error| { + format!( + "failed to read results directory {}: {error}", + directory.display() + ) + })?; + let mut paths = entries + .map(|entry| { + entry + .map(|entry| entry.path()) + .map_err(|error| format!("failed to read results directory entry: {error}")) + }) + .collect::, _>>()?; + paths.retain(|path| { + path.is_file() + && path + .extension() + .is_some_and(|extension| extension == "json") + }); + paths.sort(); + Ok(paths) +} + +fn select_catalog<'a>( + file_name: &str, + measurement: &Measurement, + frozen: &'a Catalog, + custom: Option<&'a Catalog>, +) -> Result, String> { + let Some(corpus) = measurement.metadata.corpus.as_ref() else { + return Ok(custom); + }; + if corpus.sha256 == frozen.sha256 { + return Ok(Some(frozen)); + } + let Some(custom) = custom else { + return Ok(None); + }; + if corpus.sha256 != custom.sha256 { + return Err(format!( + "result {file_name} corpus hash mismatch: result records {}, catalog computes {}", + corpus.sha256, custom.sha256 + )); + } + Ok(Some(custom)) +} + +fn stack_row( + file_name: String, + measurement: Measurement, + catalog: Option<&Catalog>, + decode_modes: &DecodeModeRegistry, +) -> Result { + let scenarios = join_scenarios(&file_name, &measurement.scenarios, catalog)?; + let category_counts = CategoryCounts::from_scenarios(&scenarios); + let display_name = safe_display_id(&measurement.metadata.model.display_name); + let endpoint_display = safe_display_id(&measurement.metadata.model.endpoint_id); + let (decode_mode, decode_mode_source) = effective_decode_mode( + measurement.metadata.server.decode_mode, + &measurement.metadata.server.preset_name, + decode_modes, + ); + Ok(StackRow { + file_name, + schema_version: measurement.schema_version, + metadata: measurement.metadata, + display_name, + endpoint_display, + decode_mode, + decode_mode_source, + scenarios, + category_counts, + }) +} + +fn effective_decode_mode( + recorded: DecodeMode, + server: &str, + decode_modes: &DecodeModeRegistry, +) -> (DecodeMode, DecodeModeSource) { + if recorded != DecodeMode::Unknown { + return (recorded, DecodeModeSource::Recorded); + } + decode_modes + .by_preset + .get(server) + .copied() + .map(|mode| (mode, DecodeModeSource::PresetMapping)) + .unwrap_or((DecodeMode::Unknown, DecodeModeSource::Unknown)) +} + +fn join_scenarios( + file_name: &str, + outcomes: &[wic_core::result::MeasurementScenarioOutcome], + catalog: Option<&Catalog>, +) -> Result, String> { + outcomes + .iter() + .map(|outcome| { + let definition = catalog + .map(|catalog| { + catalog.scenarios.get(&outcome.id).ok_or_else(|| { + format!( + "result {file_name} references scenario {:?} absent from catalog", + outcome.id + ) + }) + }) + .transpose()?; + if let Some(definition) = definition { + if definition.category != outcome.category { + return Err(format!( + "result {file_name} scenario {:?} category disagreement: result has {}, catalog has {}", + outcome.id, outcome.category, definition.category + )); + } + } + let description = definition + .map(|scenario| scenario.description.clone()) + .unwrap_or_else(|| CATALOG_UNAVAILABLE.to_owned()); + let rationale = definition + .map(|scenario| scenario.rationale.clone()) + .unwrap_or_else(|| CATALOG_UNAVAILABLE.to_owned()); + Ok(ScenarioView { + id: outcome.id.clone(), + category: outcome.category, + definition: definition.cloned(), + description, + rationale, + status: outcome.status, + failure_detail: outcome.failure.clone(), + failure_reason: outcome.failure_reason.clone(), + failure_class: outcome.failure_class.clone(), + cause: outcome.cause.clone(), + evidence_hash: outcome.evidence_hash.clone(), + evidence_path: outcome.evidence_path.clone(), + retried: outcome.retried, + }) + }) + .collect() +} + +struct Catalog { + sha256: String, + scenarios: HashMap, +} + +impl Catalog { + fn from_scenarios(scenarios: Vec) -> Result { + let sha256 = corpus_identity(&scenarios); + let mut by_id = HashMap::with_capacity(scenarios.len()); + for scenario in scenarios { + let id = scenario.id.clone(); + if by_id.insert(id.clone(), scenario).is_some() { + return Err(format!("duplicate scenario id {id:?} in catalog")); + } + } + Ok(Self { + sha256, + scenarios: by_id, + }) + } +} + +fn study_views(rows: &[StackRow]) -> Vec { + let mut candidates = BTreeMap::<(String, String), StudyArmCandidate>::new(); + for (row_index, row) in rows.iter().enumerate() { + let (Some(replication), Some(fingerprint)) = ( + row.metadata.replication.as_ref(), + row.metadata.arm_fingerprint.as_ref(), + ) else { + continue; + }; + let candidate = candidates + .entry((replication.study_id.clone(), replication.arm_id.clone())) + .or_insert_with(|| StudyArmCandidate { + fingerprint: fingerprint.clone(), + mode: replication.mode, + run_indices: HashSet::new(), + row_indices: Vec::new(), + valid: true, + }); + if candidate.fingerprint != *fingerprint + || candidate.mode != replication.mode + || !candidate.run_indices.insert(replication.run_index) + { + candidate.valid = false; + } + candidate.row_indices.push(row_index); + } + + let mut studies = BTreeMap::>::new(); + for ((study_id, arm_id), candidate) in candidates { + if candidate.valid && candidate.row_indices.len() >= REPLICATION_GATE { + studies.entry(study_id).or_default().push(StudyArm { + arm_id, + arm_fingerprint: candidate.fingerprint, + mode: candidate.mode, + row_indices: candidate.row_indices, + }); + } + } + studies + .into_iter() + .map(|(study_id, arms)| StudyView { study_id, arms }) + .collect() +} + +struct StudyArmCandidate { + fingerprint: String, + mode: ReplicationMode, + run_indices: HashSet, + row_indices: Vec, + valid: bool, +} + +fn safe_display_id(value: &str) -> String { + let path = Path::new(value); + if path.is_absolute() { + path.file_name() + .and_then(|component| component.to_str()) + .unwrap_or("local model") + .to_owned() + } else { + value.to_owned() + } +} + +fn category_index(category: ScenarioCategory) -> usize { + match category { + ScenarioCategory::SingleCall => 0, + ScenarioCategory::ParallelCalls => 1, + ScenarioCategory::Streaming => 2, + ScenarioCategory::ToolChoiceModes => 3, + ScenarioCategory::MultiTurn => 4, + ScenarioCategory::NegativeTrap => 5, + } +} + +#[cfg(test)] +mod tests { + use wic_core::result::{ + CorpusMetadata, DecodeMode, MeasurementScenarioOutcome, ScenarioFailure, Status, + }; + use wic_core::result::{ReplicationMetadata, ReplicationMode}; + use wic_core::{ArgumentsMatch, Scenario, ScenarioCategory, ToolChoice}; + + use super::{ + effective_decode_mode, join_scenarios, load, select_catalog, stack_row, study_views, + Catalog, CategoryCounts, DecodeModeRegistry, DecodeModeSource, CATALOG_UNAVAILABLE, + DECODE_MODE_REGISTRY, + }; + + fn scenario(id: &str, category: ScenarioCategory) -> Scenario { + Scenario { + id: id.to_owned(), + category, + facets: Vec::new(), + description: format!("description for {id}"), + rationale: format!("rationale for {id}"), + stream: false, + arguments_match: ArgumentsMatch::Exact, + tools: Vec::new(), + tool_choice: ToolChoice::Auto, + turns: Vec::new(), + } + } + + fn outcome(id: &str, category: ScenarioCategory) -> MeasurementScenarioOutcome { + outcome_with_status(id, category, Status::Pass) + } + + fn outcome_with_status( + id: &str, + category: ScenarioCategory, + status: Status, + ) -> MeasurementScenarioOutcome { + MeasurementScenarioOutcome { + id: id.to_owned(), + category, + status, + failure_reason: None, + failure: None::, + failure_class: None, + cause: None, + evidence_hash: None, + evidence_path: None, + retried: false, + } + } + + #[test] + fn duplicate_scenario_ids_fail_the_catalog_join() { + let duplicate = scenario("duplicate", ScenarioCategory::SingleCall); + let error = Catalog::from_scenarios(vec![duplicate.clone(), duplicate]) + .err() + .expect("duplicate catalog should fail"); + assert!(error.contains("duplicate scenario id \"duplicate\"")); + } + + #[test] + fn result_scenario_absent_from_catalog_fails_the_join() { + let catalog = + Catalog::from_scenarios(vec![scenario("present", ScenarioCategory::SingleCall)]) + .expect("catalog"); + let error = join_scenarios( + "result.json", + &[outcome("missing", ScenarioCategory::SingleCall)], + Some(&catalog), + ) + .err() + .expect("missing scenario should fail"); + assert!(error.contains("scenario \"missing\" absent from catalog")); + } + + #[test] + fn result_and_catalog_category_disagreement_fails_the_join() { + let catalog = Catalog::from_scenarios(vec![scenario( + "category-mismatch", + ScenarioCategory::Streaming, + )]) + .expect("catalog"); + let error = join_scenarios( + "result.json", + &[outcome("category-mismatch", ScenarioCategory::SingleCall)], + Some(&catalog), + ) + .err() + .expect("category mismatch should fail"); + assert!(error.contains("category disagreement")); + assert!(error.contains("result has single_call, catalog has streaming")); + } + + #[test] + fn supplied_catalog_hash_mismatch_fails_before_descriptions_are_attached() { + let frozen = + Catalog::from_scenarios(vec![scenario("frozen", ScenarioCategory::SingleCall)]) + .expect("frozen catalog"); + let custom = + Catalog::from_scenarios(vec![scenario("custom", ScenarioCategory::SingleCall)]) + .expect("custom catalog"); + let measurement = measurement_with_corpus("sha256:not-the-custom-catalog"); + let error = select_catalog("result.json", &measurement, &frozen, Some(&custom)) + .err() + .expect("hash mismatch should fail"); + assert!(error.contains("corpus hash mismatch")); + assert!(error.contains("sha256:not-the-custom-catalog")); + assert!(error.contains(&custom.sha256)); + } + + #[test] + fn custom_corpus_without_catalog_never_falls_back_to_frozen_catalog() { + let frozen_scenario = scenario("same-id", ScenarioCategory::SingleCall); + let frozen = Catalog::from_scenarios(vec![frozen_scenario]).expect("frozen catalog"); + let measurement = measurement_with_corpus("sha256:custom-corpus"); + let selected = select_catalog("result.json", &measurement, &frozen, None) + .expect("custom corpus without catalog is supported"); + assert!(selected.is_none()); + + let views = join_scenarios( + "result.json", + &[outcome("same-id", ScenarioCategory::SingleCall)], + selected, + ) + .expect("catalog-unavailable view"); + assert!(views[0].definition.is_none()); + assert_eq!(views[0].description, CATALOG_UNAVAILABLE); + assert_eq!(views[0].rationale, CATALOG_UNAVAILABLE); + } + + #[test] + fn category_counts_keep_four_states_and_withhold_macro_for_incomplete_measurement() { + let outcomes = [ + outcome_with_status("pass", ScenarioCategory::SingleCall, Status::Pass), + outcome_with_status("fail", ScenarioCategory::SingleCall, Status::Fail), + outcome_with_status("error", ScenarioCategory::Streaming, Status::Error), + outcome_with_status("skip", ScenarioCategory::MultiTurn, Status::Skipped), + ]; + let scenarios = join_scenarios("result.json", &outcomes, None).expect("scenario views"); + let counts = CategoryCounts::from_scenarios(&scenarios); + assert_eq!( + (counts.passed, counts.failed, counts.errors, counts.skipped), + (1, 1, 1, 1) + ); + assert_eq!(counts.measurement_coverage, 0.5); + assert_eq!(counts.macro_category_pass_rate, None); + } + + #[test] + fn cited_preset_mapping_resolves_only_explicit_entries() { + let registry = DecodeModeRegistry::from_json(DECODE_MODE_REGISTRY) + .expect("published decode mode registry"); + assert_eq!( + effective_decode_mode(DecodeMode::Unknown, "llamacpp", ®istry), + ( + DecodeMode::GrammarConstrained, + DecodeModeSource::PresetMapping + ) + ); + for server in ["ollama", "mlx_lm"] { + assert_eq!( + effective_decode_mode(DecodeMode::Unknown, server, ®istry), + ( + DecodeMode::UnconstrainedPostHoc, + DecodeModeSource::PresetMapping + ) + ); + } + assert_eq!( + effective_decode_mode(DecodeMode::Unknown, "custom", ®istry), + (DecodeMode::Unknown, DecodeModeSource::Unknown) + ); + } + + #[test] + fn decode_mode_mapping_requires_citations() { + let mut document: serde_json::Value = + serde_json::from_slice(DECODE_MODE_REGISTRY).expect("decode mode registry JSON"); + document["entries"][0] + .as_object_mut() + .expect("decode mode entry") + .remove("provenance_refs"); + assert!(DecodeModeRegistry::from_json( + &serde_json::to_vec(&document).expect("encode registry without provenance") + ) + .is_err()); + } + + #[test] + fn recorded_decode_mode_is_not_overridden_by_the_preset_mapping() { + let registry = DecodeModeRegistry::from_json(DECODE_MODE_REGISTRY) + .expect("published decode mode registry"); + assert_eq!( + effective_decode_mode(DecodeMode::UnconstrainedPostHoc, "llamacpp", ®istry), + (DecodeMode::UnconstrainedPostHoc, DecodeModeSource::Recorded) + ); + } + + #[test] + fn published_mlx_rows_use_their_recorded_quirk_fact() { + let results = std::path::Path::new(env!("CARGO_MANIFEST_DIR")).join("../../results"); + let dataset = load(&results, None).expect("published site dataset"); + let mlx_rows = dataset + .rows + .iter() + .filter(|row| row.metadata.server.preset_name == "mlx_lm") + .collect::>(); + assert_eq!(mlx_rows.len(), 6); + assert!(mlx_rows.iter().all(|row| { + row.metadata + .server + .quirk_flags + .iter() + .any(|flag| flag == "unconstrained_post_hoc_parse") + && row.metadata.server.decode_mode == DecodeMode::UnconstrainedPostHoc + && row.decode_mode == DecodeMode::UnconstrainedPostHoc + && row.decode_mode_source == DecodeModeSource::Recorded + })); + } + + #[test] + fn study_view_admits_only_arms_with_five_distinct_consistent_runs() { + let mut rows = (0..4) + .map(|run_index| replicated_row(run_index, "arm-a", "v1:arm-a")) + .collect::>(); + assert!(study_views(&rows).is_empty()); + + rows.push(replicated_row(4, "arm-a", "v1:arm-a")); + let studies = study_views(&rows); + assert_eq!(studies.len(), 1); + assert_eq!(studies[0].study_id, "study"); + assert_eq!(studies[0].arms.len(), 1); + assert_eq!(studies[0].arms[0].arm_id, "arm-a"); + assert_eq!(studies[0].arms[0].row_indices.len(), 5); + } + + fn replicated_row(run_index: u32, arm_id: &str, fingerprint: &str) -> super::StackRow { + let mut measurement = measurement_with_corpus("sha256:custom-corpus"); + measurement.metadata.replication = Some(ReplicationMetadata { + study_id: "study".to_owned(), + arm_id: arm_id.to_owned(), + run_index, + mode: ReplicationMode::GreedyReproducibility, + }); + measurement.metadata.arm_fingerprint = Some(fingerprint.to_owned()); + let decode_modes = DecodeModeRegistry::from_json(DECODE_MODE_REGISTRY) + .expect("published decode mode registry"); + stack_row( + format!("result-{run_index}.json"), + measurement, + None, + &decode_modes, + ) + .expect("stack row") + } + + fn measurement_with_corpus(sha256: &str) -> wic_core::result::Measurement { + let bytes = serde_json::to_vec(&serde_json::json!({ + "schema_version": 3, + "metadata": { + "run_id": "site-data-test", + "timestamp": "2000-01-01T00:00:00Z", + "willitcall_version": "test", + "endpoint": "https://fixture.invalid/v1", + "model": { + "display_name": "fixture", + "family_id": null, + "canonical_id": null, + "parameter_count_b": null, + "endpoint_id": "fixture", + "identity_status": "unresolved", + "artifact": { + "source_kind": "other", + "source_id": null, + "revision": null, + "sha256": null, + "format": "unknown", + "quantization": null + } + }, + "corpus": CorpusMetadata { + id: "custom".to_owned(), + revision: "v1".to_owned(), + sha256: sha256.to_owned(), + scenario_count: 1, + scoring_version: "v1".to_owned() + }, + "server": { + "preset_name": "custom", + "reported_version": null, + "quirk_flags": [], + "decode_mode": "unknown", + "chat_template": null, + "launch_config_sha256": null + }, + "environment": { + "display_label": "test", + "os_name": null, + "os_version": null, + "architecture": null, + "accelerator": null, + "memory_bytes": null + }, + "sampling": { + "temperature": 0.0, + "top_p": 1.0, + "seed": 1, + "max_tokens": 1 + }, + "replication": null, + "arm_fingerprint": null + }, + "scenarios": [{ + "id": "same-id", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "failure": null, + "evidence_hash": null, + "evidence_path": null, + "retried": false + }], + "totals": { + "total": 1, + "passed": 1, + "failed": 0, + "errors": 0, + "skipped": 0 + } + })) + .expect("encode measurement"); + wic_core::result::parse_and_validate_measurement(&bytes).expect("parse measurement") + } +} diff --git a/crates/willitcall/src/site/export.rs b/crates/willitcall/src/site/export.rs new file mode 100644 index 0000000..f51a917 --- /dev/null +++ b/crates/willitcall/src/site/export.rs @@ -0,0 +1,292 @@ +use std::fmt::Write as _; +use std::path::Path; + +use serde_json::{json, Value}; +use wic_core::result::{ + ArtifactFormat, ArtifactSourceKind, CauseKind, DecodeMode, IdentityStatus, Status, +}; + +use super::data::{DecodeModeSource, ScenarioView, SiteDataset, StackRow}; + +pub(super) fn render_json(dataset: &SiteDataset) -> Result { + let stacks = dataset + .rows + .iter() + .enumerate() + .map(|(index, row)| stack_json(dataset, index, row)) + .collect::>(); + let document = json!({ + "schema_version": 1, + "row_count": dataset.rows.len(), + "scenario_count": dataset.scenario_count, + "stacks": stacks, + }); + serde_json::to_string(&document) + .map(|json| ascii_escape(&json)) + .map_err(|error| format!("failed to encode site results.json: {error}")) +} + +fn stack_json(dataset: &SiteDataset, index: usize, row: &StackRow) -> Value { + let metadata = &row.metadata; + let model = &metadata.model; + let artifact = &model.artifact; + let server = &metadata.server; + json!({ + "stack_index": index + 1, + "result_file": row.file_name, + "schema_version": row.schema_version, + "run_id": metadata.run_id, + "timestamp": metadata.timestamp, + "willitcall_version": metadata.willitcall_version, + "display_name": row.display_name, + "endpoint_id": row.endpoint_display, + "canonical_id": row.cross_model_key(), + "family_id": model.family_id, + "parameter_count_b": model.parameter_count_b, + "identity_status": identity_status(model.identity_status), + "artifact": { + "source_kind": artifact_source(artifact.source_kind), + "source_id": artifact.source_id.as_deref().map(safe_path_value), + "revision": artifact.revision, + "sha256": artifact.sha256, + "format": artifact_format(artifact.format), + "quantization": artifact.quantization, + }, + "server": { + "preset_name": server.preset_name, + "reported_version": server.reported_version, + "quirk_flags": server.quirk_flags, + "recorded_decode_mode": decode_mode(server.decode_mode), + "effective_decode_mode": decode_mode(row.decode_mode), + "decode_mode_source": decode_source(row.decode_mode_source), + "chat_template": server.chat_template, + "launch_config_sha256": server.launch_config_sha256, + }, + "corpus": metadata.corpus, + "environment": metadata.environment, + "sampling": metadata.sampling, + "replication": metadata.replication, + "replication_count": dataset.replication_count(index), + "arm_fingerprint": metadata.arm_fingerprint, + "preflight_override": metadata.preflight_override, + "preflight_ignored_ports": metadata.preflight_ignored_ports, + "scenarios": row.scenarios.iter().map(scenario_json).collect::>(), + }) +} + +fn scenario_json(scenario: &ScenarioView) -> Value { + json!({ + "id": scenario.id, + "category": scenario.category.to_string(), + "description": scenario.description, + "rationale": scenario.rationale, + "status": status(scenario.status), + "failure": scenario.failure_detail, + "failure_reason": scenario.failure_reason, + "failure_class": scenario.failure_class, + "cause": scenario.cause, + "evidence_path": scenario.evidence_path, + "evidence_hash": scenario.evidence_hash, + "retried": scenario.retried, + }) +} + +pub(super) fn render_csv(dataset: &SiteDataset) -> String { + const HEADERS: [&str; 31] = [ + "stack_index", + "result_file", + "schema_version", + "run_id", + "timestamp", + "display_name", + "endpoint_id", + "canonical_id", + "identity_status", + "quantization", + "server", + "server_version", + "decode_mode", + "decode_mode_source", + "replication_count", + "scenario_id", + "category", + "description", + "rationale", + "status", + "failure_reason", + "failure_class", + "failure_stage", + "failure_code", + "http_status", + "failed_turn_index", + "cause_kind", + "cause_reference", + "cause_note", + "evidence_path", + "evidence_hash", + ]; + let mut csv = String::new(); + write_csv_row(&mut csv, &HEADERS); + for (index, row) in dataset.rows.iter().enumerate() { + for scenario in &row.scenarios { + let model = &row.metadata.model; + let quantization = model + .artifact + .quantization + .as_ref() + .map(|value| value.label.as_str()) + .unwrap_or(""); + let failure = scenario.failure_detail.as_ref(); + let cause = scenario.cause.as_ref(); + let values = [ + (index + 1).to_string(), + row.file_name.clone(), + row.schema_version.to_string(), + row.metadata.run_id.clone(), + row.metadata.timestamp.clone(), + row.display_name.clone(), + row.endpoint_display.clone(), + row.cross_model_key().unwrap_or("").to_owned(), + identity_status(model.identity_status).to_owned(), + quantization.to_owned(), + row.metadata.server.preset_name.clone(), + row.metadata + .server + .reported_version + .clone() + .unwrap_or_default(), + decode_mode(row.decode_mode).to_owned(), + decode_source(row.decode_mode_source).to_owned(), + dataset.replication_count(index).to_string(), + scenario.id.clone(), + scenario.category.to_string(), + scenario.description.clone(), + scenario.rationale.clone(), + status(scenario.status).to_owned(), + scenario.failure_reason.clone().unwrap_or_default(), + scenario.failure_class.clone().unwrap_or_default(), + failure.map(|value| value.stage.clone()).unwrap_or_default(), + failure.map(|value| value.code.clone()).unwrap_or_default(), + failure + .and_then(|value| value.http_status) + .map(|value| value.to_string()) + .unwrap_or_default(), + failure + .and_then(|value| value.failed_turn_index) + .map(|value| value.to_string()) + .unwrap_or_default(), + cause + .map(|value| cause_kind(value.kind)) + .unwrap_or("") + .to_owned(), + cause + .and_then(|value| value.reference.clone()) + .unwrap_or_default(), + cause + .and_then(|value| value.note.clone()) + .unwrap_or_default(), + scenario.evidence_path.clone().unwrap_or_default(), + scenario.evidence_hash.clone().unwrap_or_default(), + ]; + write_csv_row(&mut csv, &values); + } + } + csv +} + +fn write_csv_row>(csv: &mut String, values: &[S]) { + for (index, value) in values.iter().enumerate() { + if index > 0 { + csv.push(','); + } + csv.push('"'); + csv.push_str(&ascii_escape(value.as_ref()).replace('"', "\"\"")); + csv.push('"'); + } + csv.push('\n'); +} + +fn ascii_escape(value: &str) -> String { + let mut escaped = String::with_capacity(value.len()); + for character in value.chars() { + if character.is_ascii() { + escaped.push(character); + } else { + for unit in character.encode_utf16(&mut [0; 2]) { + write!(escaped, "\\u{unit:04x}").expect("write ASCII escape"); + } + } + } + escaped +} + +fn safe_path_value(value: &str) -> String { + let path = Path::new(value); + if path.is_absolute() { + path.file_name() + .and_then(|component| component.to_str()) + .unwrap_or("local artifact") + .to_owned() + } else { + value.to_owned() + } +} + +fn identity_status(value: IdentityStatus) -> &'static str { + match value { + IdentityStatus::Verified => "verified", + IdentityStatus::Declared => "declared", + IdentityStatus::Unresolved => "unresolved", + } +} + +fn decode_mode(value: DecodeMode) -> &'static str { + match value { + DecodeMode::GrammarConstrained => "grammar_constrained", + DecodeMode::UnconstrainedPostHoc => "unconstrained_post_hoc", + DecodeMode::Unknown => "unknown", + } +} + +fn decode_source(value: DecodeModeSource) -> &'static str { + match value { + DecodeModeSource::Recorded => "recorded", + DecodeModeSource::PresetMapping => "preset_mapping", + DecodeModeSource::Unknown => "unknown", + } +} + +fn status(value: Status) -> &'static str { + match value { + Status::Pass => "pass", + Status::Fail => "fail", + Status::Error => "error", + Status::Skipped => "skipped", + } +} + +fn cause_kind(value: CauseKind) -> &'static str { + match value { + CauseKind::ServerDefect => "server_defect", + CauseKind::Unknown => "unknown", + } +} + +fn artifact_source(value: ArtifactSourceKind) -> &'static str { + match value { + ArtifactSourceKind::Huggingface => "huggingface", + ArtifactSourceKind::Ollama => "ollama", + ArtifactSourceKind::LocalFile => "local_file", + ArtifactSourceKind::Other => "other", + } +} + +fn artifact_format(value: ArtifactFormat) -> &'static str { + match value { + ArtifactFormat::Gguf => "gguf", + ArtifactFormat::Mlx => "mlx", + ArtifactFormat::Safetensors => "safetensors", + ArtifactFormat::OllamaBlob => "ollama_blob", + ArtifactFormat::Unknown => "unknown", + } +} diff --git a/crates/willitcall/src/site/html.rs b/crates/willitcall/src/site/html.rs new file mode 100644 index 0000000..d981c5b --- /dev/null +++ b/crates/willitcall/src/site/html.rs @@ -0,0 +1,1868 @@ +use std::collections::{BTreeMap, BTreeSet}; +use std::fmt::Write as _; +use std::path::Path; + +use wic_core::result::{ + ArtifactFormat, ArtifactSourceKind, CauseKind, DecodeMode, EnvironmentMetadataV3, + IdentityStatus, ReplicationMode, Status, +}; +use wic_core::ScenarioCategory; + +use super::data::{DecodeModeSource, ScenarioView, SiteDataset, StackRow, CATEGORIES}; +use super::svg; + +// The replicated source runs behind these two case studies live off-repository, so +// SiteDataset cannot derive their historical run and arm counts. +const CASE_STUDY_SAMPLE_SUMMARY: &str = + "90 runs across 18 quantization arms, and 40 runs across 8 arms for the peg-native anomaly"; + +#[derive(Clone, Copy, Eq, PartialEq)] +pub(super) enum Page { + Matrix, + Outcomes, + Appendix, + Submit, +} + +pub(super) struct PageShell<'a> { + pub description: &'a str, + pub title: &'a str, + pub current_page: Page, + pub main_class: Option<&'a str>, + pub main: &'a str, + pub footer: &'a str, + pub script: Option<&'a str>, +} + +pub(super) fn render_page_shell(shell: PageShell<'_>) -> String { + let main_class = shell + .main_class + .map(|class| format!(" class=\"{}\"", escape_html(class))) + .unwrap_or_default(); + let script = shell.script.unwrap_or_default(); + format!( + r#" + + + + + + {} + + + +
+ willitcall +{} +
+ +{} + +{}{} + +"#, + escape_html(shell.description), + escape_html(shell.title), + render_nav(shell.current_page), + shell.main, + shell.footer, + script + ) +} + +pub(super) fn render_nav(current_page: Page) -> String { + match current_page { + Page::Matrix => { + r#" "# + } + Page::Outcomes => { + r#" "# + } + Page::Appendix => { + r#" "# + } + Page::Submit => { + r#" "# + } + } + .to_owned() +} + +/// The only HTML-renderer seam for figures. Every SVG variant carries the shared +/// accessibility contract enforced by `svg::render_figure`. +#[allow(dead_code)] // Brief 8 supplies the first figure after this seam is established. +pub(super) fn render_figure(figure: svg::Figure<'_>) -> String { + svg::render_figure(figure) +} + +pub(super) fn render_index(dataset: &SiteDataset, repo_base: &str) -> String { + let results = &dataset.rows; + let uniform_environment = results + .first() + .and_then(|result| result.metadata.environment.as_ref()) + .filter(|environment| { + results.iter().all(|result| { + result + .metadata + .environment + .as_ref() + .is_some_and(|candidate| same_environment(candidate, environment)) + }) + }); + let mut main = String::new(); + write!( + main, + r#"
+

Measurement register

+

Tool-calling support matrix

+

Each row records one tested stack. The stack includes the model and its artifact. It also includes the server and decode mode. The matrix does not rank models or state verdicts.

+
"#, + ) + .expect("write HTML"); + main.push_str(&render_table(dataset, repo_base)); + main.push_str(&super::analysis::render_pass_counts(dataset)); + main.push_str(&render_methodology(dataset, repo_base, uniform_environment)); + + let footer = render_colophon(dataset, &main); + + render_page_shell(PageShell { + description: "Measured tool-calling support by model, quant, server, and server version.", + title: "willitcall support matrix", + current_page: Page::Matrix, + main_class: None, + main: &main, + footer: &footer, + script: Some(" \n"), + }) +} + +pub(super) fn render_outcomes(dataset: &SiteDataset) -> String { + let mut main = String::from( + r#"
+

Evidence layer

+

Observed outcomes

+

These rasters show the scenario-level evidence behind the matrix. They keep the same row order as the matrix. They do not rank observations from a single run.

+

Each mark shows one scenario outcome for one published stack. The data files include scenario descriptions and rationales. They also contain status text and evidence paths. They provide citation details. The appendix includes recorded stack metadata and transcript links.

+
"#, + ); + main.push_str(&super::analysis::render_outcomes(dataset)); + let footer = render_colophon(dataset, &main); + render_page_shell(PageShell { + description: "Scenario-level evidence behind the willitcall support matrix.", + title: "Observed outcomes - willitcall", + current_page: Page::Outcomes, + main_class: None, + main: &main, + footer: &footer, + script: None, + }) +} + +pub(super) fn render_appendix_page(dataset: &SiteDataset, repo_base: &str) -> String { + let mut main = String::from( + r#"
+

Evidence register

+

Per-stack appendix

+

Recorded metadata and scenario-level transcript links for every published stack.

+
"#, + ); + main.push_str(&render_appendix(dataset, repo_base)); + let footer = render_colophon(dataset, &main); + render_page_shell(PageShell { + description: "Recorded metadata and transcript links for published willitcall stacks.", + title: "Per-stack appendix - willitcall", + current_page: Page::Appendix, + main_class: None, + main: &main, + footer: &footer, + script: None, + }) +} + +fn render_reading_legend(dataset: &SiteDataset) -> String { + let replication_note = if !dataset.rows.is_empty() + && (0..dataset.rows.len()).all(|index| dataset.replication_count(index) == 1) + { + "n=1, no verdict: every published arm is a single run, so no cell carries a verdict; hatching marks this." + } else { + "Hatching marks cells with fewer than five runs; those cells carry no verdict." + }; + format!( + r#"
+

How to read: Each ratio shows passed scenarios / all scenarios in that category.

+
+ pass + model / response failure + execution / server error + not tested +
+
+ {replication_note} + identity status +
+
+"#, + ) +} + +pub(super) fn render_table(dataset: &SiteDataset, repo_base: &str) -> String { + let mut html = String::new(); + write!( + html, + r#"
+
+
+

Observed stacks

+

Capability matrix

+
+ +
+{} +

Showing {} stacks.

+
+ + + + +"#, + render_reading_legend(dataset), + dataset.rows.len() + ) + .expect("write HTML"); + for category in CATEGORIES { + writeln!( + html, + " ", + category, + category_label(category) + ) + .expect("write HTML"); + } + html.push_str( + r#" + +"#, + ); + for (model_key, row_indices) in grouped_row_indices(dataset) { + let first = &dataset.rows[row_indices[0]]; + let model_label = model_group_label(first); + let search_text = model_search_text(dataset, &row_indices); + let group_class = if row_indices.len() > 1 { + "multi-row" + } else { + "single-row" + }; + writeln!( + html, + " ", + escape_html(&model_key), + escape_html(&search_text), + ) + .expect("write model group"); + if row_indices.len() > 1 { + writeln!( + html, + " ", + CATEGORIES.len() + 1, + escape_html(&model_label), + ) + .expect("write model heading"); + } + let spans_decode_strata = model_spans_decode_strata(dataset, &row_indices); + let mut current_band = None; + for index in row_indices { + let row = &dataset.rows[index]; + if spans_decode_strata && current_band != Some(row.decode_mode) { + render_decode_boundary(&mut html, row.decode_mode); + current_band = Some(row.decode_mode); + } + render_result_row(&mut html, dataset, index, row, repo_base); + } + html.push_str(" \n"); + } + + html.push_str( + r#"
Model / quant / server{}
Model {}
+
+

Machine-readable observations: JSON and CSV. Scenario rasters are on the outcomes page; recorded metadata and transcripts are in the appendix.

+
"#, + ); + html +} + +fn render_decode_boundary(html: &mut String, decode_mode: DecodeMode) { + let id = decode_mode_id(decode_mode); + let note = match decode_mode { + DecodeMode::GrammarConstrained => "tool grammar constrains generation", + DecodeMode::UnconstrainedPostHoc => "generated text is parsed after decoding", + DecodeMode::Unknown => "decode behavior was not established", + }; + writeln!( + html, + " {id}{note}", + CATEGORIES.len() + 1, + ) + .expect("write decode band"); +} + +fn render_result_row( + html: &mut String, + dataset: &SiteDataset, + index: usize, + result: &StackRow, + repo_base: &str, +) { + let server = &result.metadata.server.preset_name; + let server_display = display_server(server); + let model = &result.metadata.model; + let identity_status = display_identity_status(model.identity_status); + let row_label = if model.identity_status == IdentityStatus::Unresolved { + format!("{} (unverified artifact)", result.display_name) + } else { + result.display_name.clone() + }; + let cross_model_attribute = result + .cross_model_key() + .map(|key| format!(" data-cross-model-key=\"{}\"", escape_html(key))) + .unwrap_or_default(); + let identity_title = match model.identity_status { + IdentityStatus::Unresolved => "Identity status: unresolved. Provenance could not be established; excluded from cross-model comparison.", + IdentityStatus::Declared => "Identity status: declared.", + IdentityStatus::Verified => "Identity status: verified.", + }; + let quant = result + .metadata + .model + .artifact + .quantization + .as_ref() + .map(|quantization| quantization.label.as_str()) + .unwrap_or("not declared"); + let decode_title = format!( + "Decode mode: {}; decode provenance: {}.", + decode_mode_label(result.decode_mode), + decode_mode_source_label(result.decode_mode_source), + ); + write!( + html, + " \n \n {}\n
{} / {}{} ({})id: {}detail
\n \n", + escape_html(server), + identity_status, + decode_mode_id(result.decode_mode), + decode_mode_source_id(result.decode_mode_source), + cross_model_attribute, + escape_html(&row_label), + escape_html(quant), + escape_html(server_display), + decode_mode_id(result.decode_mode), + escape_html(&decode_title), + decode_mode_short_label(result.decode_mode), + decode_mode_source_short_label(result.decode_mode_source), + identity_status, + escape_html(identity_title), + identity_status, + ) + .expect("write HTML"); + + let replication_count = dataset.replication_count(index); + for category in CATEGORIES { + render_category_cell(html, result, category, repo_base, replication_count); + } + html.push_str(" \n"); +} + +fn grouped_row_indices(dataset: &SiteDataset) -> Vec<(String, Vec)> { + let mut groups = BTreeMap::)>::new(); + for (index, row) in dataset.rows.iter().enumerate() { + let label = model_group_label(row); + groups + .entry(label.to_ascii_lowercase()) + .or_insert_with(|| (label, Vec::new())) + .1 + .push(index); + } + groups + .into_values() + .map(|(label, mut indices)| { + indices.sort_by_key(|index| { + let row = &dataset.rows[*index]; + let quant = row + .metadata + .model + .artifact + .quantization + .as_ref() + .map(|value| value.label.to_ascii_lowercase()) + .unwrap_or_default(); + ( + decode_order(row.decode_mode), + row.metadata.server.preset_name.to_ascii_lowercase(), + quant, + row.display_name.to_ascii_lowercase(), + row.file_name.clone(), + ) + }); + (label, indices) + }) + .collect() +} + +fn model_group_label(row: &StackRow) -> String { + row.cross_model_key() + .unwrap_or(&row.display_name) + .to_owned() +} + +fn model_spans_decode_strata(dataset: &SiteDataset, indices: &[usize]) -> bool { + indices.iter().enumerate().any(|(position, left_index)| { + let left = &dataset.rows[*left_index]; + indices[position + 1..].iter().any(|right_index| { + let right = &dataset.rows[*right_index]; + left.decode_mode != right.decode_mode + && left.metadata.model.artifact.source_kind + == right.metadata.model.artifact.source_kind + && left.metadata.model.artifact.source_id == right.metadata.model.artifact.source_id + && left.metadata.model.artifact.revision == right.metadata.model.artifact.revision + && left.metadata.model.artifact.sha256 == right.metadata.model.artifact.sha256 + && left.metadata.model.artifact.format == right.metadata.model.artifact.format + && left + .metadata + .model + .artifact + .quantization + .as_ref() + .map(|value| (&value.label, &value.scheme, value.bits)) + == right + .metadata + .model + .artifact + .quantization + .as_ref() + .map(|value| (&value.label, &value.scheme, value.bits)) + }) + }) +} + +fn model_search_text(dataset: &SiteDataset, indices: &[usize]) -> String { + let mut terms = BTreeSet::new(); + for index in indices { + let row = &dataset.rows[*index]; + terms.insert(row.display_name.to_ascii_lowercase()); + terms.insert(row.endpoint_display.to_ascii_lowercase()); + if let Some(key) = row.cross_model_key() { + terms.insert(key.to_ascii_lowercase()); + } + } + terms.into_iter().collect::>().join(" ") +} + +fn decode_order(mode: DecodeMode) -> u8 { + match mode { + DecodeMode::GrammarConstrained => 0, + DecodeMode::UnconstrainedPostHoc => 1, + DecodeMode::Unknown => 2, + } +} + +fn decode_mode_id(mode: DecodeMode) -> &'static str { + match mode { + DecodeMode::GrammarConstrained => "grammar_constrained", + DecodeMode::UnconstrainedPostHoc => "unconstrained_post_hoc", + DecodeMode::Unknown => "unknown", + } +} + +fn decode_mode_label(mode: DecodeMode) -> &'static str { + match mode { + DecodeMode::GrammarConstrained => "grammar constrained", + DecodeMode::UnconstrainedPostHoc => "unconstrained post-hoc", + DecodeMode::Unknown => "unknown", + } +} + +fn decode_mode_short_label(mode: DecodeMode) -> &'static str { + match mode { + DecodeMode::GrammarConstrained => "grammar", + DecodeMode::UnconstrainedPostHoc => "post-hoc", + DecodeMode::Unknown => "unknown", + } +} + +fn decode_mode_source_id(source: DecodeModeSource) -> &'static str { + match source { + DecodeModeSource::Recorded => "recorded", + DecodeModeSource::PresetMapping => "preset_mapping", + DecodeModeSource::Unknown => "unknown", + } +} + +fn decode_mode_source_label(source: DecodeModeSource) -> &'static str { + match source { + DecodeModeSource::Recorded => "recorded by run", + DecodeModeSource::PresetMapping => "documented preset mapping", + DecodeModeSource::Unknown => "unknown (no cited mapping)", + } +} + +fn decode_mode_source_short_label(source: DecodeModeSource) -> &'static str { + match source { + DecodeModeSource::Recorded => "run", + DecodeModeSource::PresetMapping => "mapping", + DecodeModeSource::Unknown => "source unknown", + } +} + +fn category_label(category: ScenarioCategory) -> &'static str { + match category { + ScenarioCategory::SingleCall => "Single call", + ScenarioCategory::ToolChoiceModes => "Tool choice", + ScenarioCategory::NegativeTrap => "Correctly declines", + ScenarioCategory::MultiTurn => "Multi-turn", + ScenarioCategory::ParallelCalls => "Parallel calls", + ScenarioCategory::Streaming => "Streaming", + } +} + +fn same_environment(left: &EnvironmentMetadataV3, right: &EnvironmentMetadataV3) -> bool { + left.display_label == right.display_label + && left.os_name == right.os_name + && left.os_version == right.os_version + && left.architecture == right.architecture + && left.accelerator == right.accelerator + && left.memory_bytes == right.memory_bytes +} + +fn environment_display_parts(environment: &EnvironmentMetadataV3) -> (&str, &str) { + environment + .display_label + .split_once("; ") + .unwrap_or((&environment.display_label, "not recorded")) +} + +fn render_environment_statement(environment: &EnvironmentMetadataV3) -> String { + format!( + "

All measurements used {}.

", + escape_html(&environment.display_label) + ) +} + +fn render_methodology( + dataset: &SiteDataset, + repo_base: &str, + uniform_environment: Option<&EnvironmentMetadataV3>, +) -> String { + let case_studies_url = format!("{repo_base}/tree/main/docs/case-studies"); + let peg_native_case_study_url = format!( + "{repo_base}/blob/main/docs/case-studies/2026-07-21-llamacpp-500s-on-llama-3.1-tool-calls.md" + ); + let decode_mapping_url = format!("{repo_base}/blob/main/registry/decode-modes-v1.json"); + let environment_statement = uniform_environment + .map(render_environment_statement) + .unwrap_or_default(); + format!( + r#"
+

Calibration notes

+

Method and limitations

+

Each cell measures one full stack. The stack combines a model, quant, server, and server version. A cell does not describe the model alone.

+

A failed observation applies only to the tested combination. It does not mean the weights are bad. The same weights can pass on one server and fail on another. When the evidence proves that difference, the cell includes a cause annotation.

+

When the result includes a transcript path, a failing observation links to the full request and response transcript. Legacy schema v1 results do not record transcript paths. Read the case studies in docs/case-studies/ for controlled comparisons.

+

The servers use different decode methods. llama.cpp compiles the supplied tool definitions into a GBNF grammar. It uses that grammar to constrain decoding. Ollama and MLX LM generate unconstrained text. They parse the tool call after decoding. Cross-band differences therefore reflect the full stack. Compare adjacent models only when they use the same server. Each row states whether the run recorded its decode mode or the site read the mode from the cited preset mapping. An unmapped preset has an unknown mode.

+

The site includes {} distinct scenarios. Each published cell represents one run. Hatching marks that the cell has no verdict.

+

The case studies draw a verdict only after at least five runs per arm. The current case studies cover {}.

+

Excluded rows

+

The quantization conclusion excludes Meta-Llama-3.1-8B-Instruct on llama.cpp (Q8_0, Q4_K_M, Q3_K_M). For this model, llama.cpp returns HTTP 500 on 7-9 of 50 scenarios per run ("does not match the expected peg-native format"). These results are server errors. They are not model failures and cannot be compared across arms. Read the peg-native case study.

+{} +
+"#, + escape_html(&case_studies_url), + escape_html(&decode_mapping_url), + dataset.scenario_count, + CASE_STUDY_SAMPLE_SUMMARY, + escape_html(&peg_native_case_study_url), + environment_statement, + ) +} + +fn render_appendix(dataset: &SiteDataset, repo_base: &str) -> String { + let mut html = String::from( + "
\n

Recorded stacks

\n

Full recorded metadata and scenario-level evidence, in the same model and decode order as the matrix. Machine-readable observations are available as JSON and CSV.

\n", + ); + for (_, row_indices) in grouped_row_indices(dataset) { + for index in row_indices { + render_stack_detail(&mut html, dataset, index, &dataset.rows[index], repo_base); + } + } + html.push_str("
\n"); + html +} + +fn render_stack_detail( + html: &mut String, + dataset: &SiteDataset, + index: usize, + result: &StackRow, + repo_base: &str, +) { + let metadata = &result.metadata; + let model = &metadata.model; + let artifact = &model.artifact; + let server = &metadata.server; + let environment = metadata.environment.as_ref(); + writeln!( + html, + "
\n

Stack {:02}

{}

{}
\n
\n
", + index + 1, + escape_html(&result.display_name), + decode_mode_id(result.decode_mode), + decode_mode_id(result.decode_mode), + ) + .expect("write stack detail"); + metadata_item(html, "Result file", &result.file_name, true); + metadata_item( + html, + "Schema", + &format!("v{}", result.schema_version), + false, + ); + metadata_item(html, "Run id", recorded(&metadata.run_id), true); + metadata_item(html, "Run time", &metadata.timestamp, false); + metadata_item( + html, + "willitcall version", + &metadata.willitcall_version, + true, + ); + metadata_item(html, "Endpoint id", &result.endpoint_display, true); + metadata_item( + html, + "Canonical id", + result.cross_model_key().unwrap_or("not established"), + true, + ); + metadata_item( + html, + "Family id", + model.family_id.as_deref().unwrap_or("not recorded"), + true, + ); + metadata_item( + html, + "Parameters", + &model + .parameter_count_b + .map(|value| format!("{value}B")) + .unwrap_or_else(|| "not recorded".to_owned()), + false, + ); + metadata_item( + html, + "Identity status", + display_identity_status(model.identity_status), + false, + ); + metadata_item( + html, + "Artifact source", + display_artifact_source(artifact.source_kind), + true, + ); + metadata_item( + html, + "Artifact source id", + artifact + .source_id + .as_deref() + .map(safe_path_value) + .as_deref() + .unwrap_or("not recorded"), + true, + ); + metadata_item( + html, + "Artifact revision", + artifact.revision.as_deref().unwrap_or("not recorded"), + true, + ); + metadata_item( + html, + "Artifact sha256", + artifact.sha256.as_deref().unwrap_or("not recorded"), + true, + ); + metadata_item( + html, + "Artifact format", + display_artifact_format(artifact.format), + true, + ); + let quant = artifact.quantization.as_ref(); + metadata_item( + html, + "Artifact quant", + quant + .map(|value| value.label.as_str()) + .unwrap_or("not declared"), + true, + ); + metadata_item( + html, + "Quant scheme", + quant + .and_then(|value| value.scheme.as_deref()) + .unwrap_or("not recorded"), + true, + ); + metadata_item( + html, + "Quant bits", + &quant + .and_then(|value| value.bits) + .map(|value| value.to_string()) + .unwrap_or_else(|| "not recorded".to_owned()), + false, + ); + metadata_item(html, "Server", display_server(&server.preset_name), false); + metadata_item( + html, + "Server version", + server + .reported_version + .as_deref() + .unwrap_or("version not reported"), + true, + ); + metadata_item( + html, + "Decode class", + decode_mode_id(result.decode_mode), + true, + ); + metadata_item( + html, + "Decode provenance", + decode_mode_source_label(result.decode_mode_source), + false, + ); + metadata_item( + html, + "Recorded decode mode", + decode_mode_id(server.decode_mode), + true, + ); + metadata_item( + html, + "Quirk flags", + &list_or_not_recorded(&server.quirk_flags), + true, + ); + metadata_item( + html, + "Chat template id", + server + .chat_template + .as_ref() + .and_then(|template| template.id.as_deref()) + .unwrap_or("not recorded"), + true, + ); + metadata_item( + html, + "Chat template sha256", + server + .chat_template + .as_ref() + .and_then(|template| template.sha256.as_deref()) + .unwrap_or("not recorded"), + true, + ); + metadata_item( + html, + "Launch config sha256", + server + .launch_config_sha256 + .as_deref() + .unwrap_or("not recorded"), + true, + ); + if let Some(corpus) = metadata.corpus.as_ref() { + metadata_item(html, "Corpus id", &corpus.id, true); + metadata_item(html, "Corpus revision", &corpus.revision, true); + metadata_item(html, "Corpus sha256", &corpus.sha256, true); + metadata_item( + html, + "Corpus scenarios", + &corpus.scenario_count.to_string(), + false, + ); + metadata_item(html, "Scoring version", &corpus.scoring_version, true); + } else { + metadata_item(html, "Corpus", "not recorded", false); + } + metadata_item( + html, + "Host hardware", + environment + .map(|value| value.display_label.as_str()) + .unwrap_or("not recorded"), + false, + ); + metadata_item( + html, + "Host OS", + &environment + .map(environment_display_parts) + .map(|(_, os)| os.to_owned()) + .unwrap_or_else(|| "not recorded".to_owned()), + false, + ); + metadata_item( + html, + "Architecture", + environment + .and_then(|value| value.architecture.as_deref()) + .unwrap_or("not recorded"), + true, + ); + metadata_item( + html, + "Accelerator", + environment + .and_then(|value| value.accelerator.as_deref()) + .unwrap_or("not recorded"), + true, + ); + metadata_item( + html, + "Memory bytes", + &environment + .and_then(|value| value.memory_bytes) + .map(|value| value.to_string()) + .unwrap_or_else(|| "not recorded".to_owned()), + false, + ); + metadata_item( + html, + "Temperature", + &metadata + .sampling + .temperature + .map(|value| value.to_string()) + .unwrap_or_else(|| "not recorded".to_owned()), + false, + ); + metadata_item( + html, + "Top p", + &metadata + .sampling + .top_p + .map(|value| value.to_string()) + .unwrap_or_else(|| "not recorded".to_owned()), + false, + ); + metadata_item( + html, + "Seed", + &metadata + .sampling + .seed + .map(|value| value.to_string()) + .unwrap_or_else(|| "not recorded".to_owned()), + false, + ); + metadata_item( + html, + "Max tokens", + &metadata + .sampling + .max_tokens + .map(|value| value.to_string()) + .unwrap_or_else(|| "not recorded".to_owned()), + false, + ); + metadata_item( + html, + "Replication", + &replication_summary(result, dataset.replication_count(index)), + false, + ); + metadata_item( + html, + "Arm fingerprint", + metadata + .arm_fingerprint + .as_deref() + .unwrap_or("not recorded"), + true, + ); + metadata_item( + html, + "Preflight override", + &metadata + .preflight_override + .as_ref() + .map(|override_| { + format!( + "forced={}; foreign endpoints={}", + override_.forced, + list_or_not_recorded(&override_.foreign_endpoints) + ) + }) + .unwrap_or_else(|| "none".to_owned()), + false, + ); + metadata_item( + html, + "Ignored ports", + &metadata + .preflight_ignored_ports + .as_ref() + .map(|ports| { + ports + .iter() + .map(u16::to_string) + .collect::>() + .join(", ") + }) + .filter(|ports| !ports.is_empty()) + .unwrap_or_else(|| "none".to_owned()), + false, + ); + html.push_str("
\n
    \n"); + for scenario in &result.scenarios { + render_scenario_detail(html, result.schema_version, scenario, repo_base); + } + html.push_str("
\n Back to matrix\n
\n
\n"); +} + +fn metadata_item(html: &mut String, label: &str, value: &str, code: bool) { + if code { + writeln!( + html, + "
{}
{}
", + escape_html(label), + escape_html(value) + ) + .expect("write metadata item"); + } else { + writeln!( + html, + "
{}
{}
", + escape_html(label), + escape_html(value) + ) + .expect("write metadata item"); + } +} + +fn render_scenario_detail( + html: &mut String, + schema_version: u32, + scenario: &ScenarioView, + repo_base: &str, +) { + let status = display_status(scenario.status); + write!( + html, + "
  • {}{status}

    {}

    Rationale: {}

    ", + escape_html(&scenario.id), + escape_html(&scenario.description), + escape_html(&scenario.rationale), + ) + .expect("write scenario detail"); + if let Some(failure) = scenario.failure_detail.as_ref() { + write!( + html, + " stage: {}; code: {}; HTTP: {}; failed turn: {}", + escape_html(&failure.stage), + escape_html(&failure.code), + failure + .http_status + .map(|value| value.to_string()) + .unwrap_or_else(|| "not recorded".to_owned()), + failure + .failed_turn_index + .map(|value| value.to_string()) + .unwrap_or_else(|| "not recorded".to_owned()), + ) + .expect("write scenario failure"); + } + if let Some(reason) = scenario.failure_reason.as_deref() { + write!( + html, + " {}", + escape_html(reason) + ) + .expect("write failure reason"); + } + if let Some(evidence_path) = scenario.evidence_path.as_deref() { + write!( + html, + " transcript", + escape_html(&evidence_url(repo_base, evidence_path)) + ) + .expect("write transcript link"); + } + if let Some(evidence_hash) = scenario.evidence_hash.as_deref() { + write!( + html, + " {}", + escape_html(evidence_hash) + ) + .expect("write evidence hash"); + } + if scenario.retried { + html.push_str(" retried"); + } + render_annotation(html, schema_version, scenario, repo_base); + html.push_str("
    • \n"); +} + +fn recorded(value: &str) -> &str { + if value.is_empty() { + "not recorded" + } else { + value + } +} + +fn safe_path_value(value: &str) -> String { + let path = Path::new(value); + if path.is_absolute() { + path.file_name() + .and_then(|component| component.to_str()) + .unwrap_or("local artifact") + .to_owned() + } else { + value.to_owned() + } +} + +fn list_or_not_recorded(values: &[String]) -> String { + if values.is_empty() { + "not recorded".to_owned() + } else { + values.join(", ") + } +} + +fn replication_summary(result: &StackRow, count: usize) -> String { + result.metadata.replication.as_ref().map_or_else( + || format!("n={count}, no declared study arm"), + |replication| { + format!( + "n={count}; study={}; arm={}; run={}; mode={}", + replication.study_id, + replication.arm_id, + replication.run_index, + display_replication_mode(replication.mode) + ) + }, + ) +} + +fn display_replication_mode(mode: ReplicationMode) -> &'static str { + match mode { + ReplicationMode::GreedyReproducibility => "greedy_reproducibility", + ReplicationMode::SeedVariedVariance => "seed_varied_variance", + } +} + +fn display_artifact_source(source: ArtifactSourceKind) -> &'static str { + match source { + ArtifactSourceKind::Huggingface => "huggingface", + ArtifactSourceKind::Ollama => "ollama", + ArtifactSourceKind::LocalFile => "local_file", + ArtifactSourceKind::Other => "other", + } +} + +fn display_artifact_format(format: ArtifactFormat) -> &'static str { + match format { + ArtifactFormat::Gguf => "gguf", + ArtifactFormat::Mlx => "mlx", + ArtifactFormat::Safetensors => "safetensors", + ArtifactFormat::OllamaBlob => "ollama_blob", + ArtifactFormat::Unknown => "unknown", + } +} + +fn render_colophon(dataset: &SiteDataset, main: &str) -> String { + let mut revisions = dataset + .rows + .iter() + .filter_map(|row| row.metadata.corpus.as_ref()) + .map(|corpus| format!("{} {}", corpus.id, corpus.revision)) + .collect::>(); + let revision = if revisions.is_empty() { + "not recorded".to_owned() + } else { + revisions.pop_first().expect("non-empty revisions") + + &revisions + .into_iter() + .map(|value| format!(", {value}")) + .collect::() + }; + let date = dataset + .rows + .iter() + .filter_map(|row| row.metadata.timestamp.get(..10)) + .max() + .unwrap_or("not recorded"); + let mut hash = 0xcbf29ce484222325_u64; + for byte in main.bytes().chain(STYLE.bytes()).chain(SCRIPT.bytes()) { + hash ^= u64::from(byte); + hash = hash.wrapping_mul(0x100000001b3); + } + format!( + "

    Build hash {hash:016x} - corpus revision {} - data date

    \n", + escape_html(&revision), + escape_html(date), + escape_html(date), + ) +} + +fn render_category_cell( + html: &mut String, + result: &StackRow, + category: ScenarioCategory, + repo_base: &str, + replication_count: usize, +) { + let scenarios = result + .scenarios + .iter() + .filter(|scenario| scenario.category == category) + .collect::>(); + let counts = result.category_counts.for_category(category); + let total = counts.total(); + let passed = counts.passed; + let failed = counts.failed; + let errors = counts.errors; + let skipped = counts.skipped; + let not_measurable = errors + skipped > 0 && passed + failed == 0; + let class = if total == 0 { + "untested" + } else if not_measurable { + "not-measurable" + } else if passed == total { + "all-pass" + } else if passed == 0 { + "none-pass" + } else { + "partial" + }; + let first_evidence = scenarios + .iter() + .find(|scenario| { + scenario.status != Status::Pass && scenario.evidence_path.as_deref().is_some() + }) + .and_then(|scenario| scenario.evidence_path.as_deref()); + let replication_class = if replication_count < 5 { + " low-replication" + } else { + "" + }; + write!( + html, + " ", + category_label(category), + category, + if replication_count < 5 { "; no verdict" } else { "" }, + ) + .expect("write HTML"); + if let Some(evidence_path) = first_evidence { + write!( + html, + "{passed}/{total}", + escape_html(&evidence_url(repo_base, evidence_path)) + ) + .expect("write HTML"); + } else { + write!(html, "{passed}/{total}").expect("write HTML"); + if total > 0 && passed < total && result.schema_version == 1 { + html.push_str("schema v1: no transcript path"); + } + } + if not_measurable { + html.push_str("not measurable"); + } + html.push_str("\n"); +} + +fn render_annotation( + html: &mut String, + schema_version: u32, + scenario: &ScenarioView, + repo_base: &str, +) { + if let Some(cause) = scenario.cause.as_ref() { + let label = match cause.kind { + CauseKind::ServerDefect => "server defect", + CauseKind::Unknown => "cause unknown", + }; + if let Some(reference) = cause.reference.as_deref() { + let reference = reference_url(repo_base, reference); + write!( + html, + " {label}", + escape_html(&reference), + ) + .expect("write HTML"); + } else { + write!(html, " {label}").expect("write HTML"); + } + } + if scenario.failure_class.as_deref() == Some("empty_response") { + html.push_str(" empty response"); + } else if scenario.failure_class.as_deref() == Some("unparsed_tool_call") { + html.push_str(" unparsed tool call"); + } else if schema_version == 1 + && scenario.cause.is_none() + && scenario.status != Status::Pass + && scenario.evidence_hash.is_some() + { + html.push_str(" legacy evidence hash only"); + } +} + +fn display_identity_status(status: IdentityStatus) -> &'static str { + match status { + IdentityStatus::Verified => "verified", + IdentityStatus::Declared => "declared", + IdentityStatus::Unresolved => "unresolved", + } +} + +pub(super) fn render_submit(repo_base: &str) -> String { + let contributing_url = format!("{repo_base}/blob/main/CONTRIBUTING.md"); + let main = format!( + r#"
    +

    Submission method

    +

    Create a result

    +

    Run one model at a time. Store its result file and evidence directory together.

    +
    +
    +

    Ollama

    + 
    MODEL=qwen2.5:7b-instruct
+OUT=results/ollama-qwen2.5-7b-instruct.json
+cargo run -p willitcall -- run \
+  --model "$MODEL" \
+  --server ollama \
+  --out "$OUT"
+cargo run -p willitcall -- validate "$OUT"
    +
    +
    +

    llama.cpp

    + 
    MODEL_PATH=/absolute/path/to/model.Q4_K_M.gguf
+OUT=results/llamacpp-model-q4_k_m.json
+cargo run -p willitcall -- run \
+  --model "$MODEL_PATH" \
+  --server llamacpp \
+  --out "$OUT"
+cargo run -p willitcall -- validate "$OUT"
    +
    +
    +

    Pull request checklist

    +
      +
    • Confirm that preflight is clean and has no contention override. Explain any override.
      • +
    • Validate the result file against the schema.
      • +
    • Include the evidence transcripts.
      • +
    • Cross-check empty responses on a second server as required by the seeding protocol.
      • +
    +

    Read CONTRIBUTING.md for the complete contribution rules.

    +
    "#, + escape_html(&contributing_url) + ); + render_page_shell(PageShell { + description: "Commands and checks for submitting a willitcall result.", + title: "Submit a result - willitcall", + current_page: Page::Submit, + main_class: Some("submit-page"), + main: &main, + footer: "

    Reviewers treat each result as measured stack behavior. They do not treat it as a model-only claim.

    \n", + script: None, + }) +} + +fn display_server(server: &str) -> &str { + if server == "llamacpp" { + "llama.cpp" + } else if server == "mlx_lm" { + "MLX LM" + } else { + server + } +} + +fn display_status(status: Status) -> &'static str { + match status { + Status::Pass => "pass", + Status::Fail => "fail", + Status::Error => "error", + Status::Skipped => "skipped", + } +} + +fn evidence_url(repo_base: &str, evidence_path: &str) -> String { + format!( + "{repo_base}/blob/main/results/{}", + evidence_path.trim_start_matches('/') + ) +} + +fn reference_url(repo_base: &str, reference: &str) -> String { + if reference.starts_with("https://") || reference.starts_with("http://") { + reference.to_owned() + } else { + format!( + "{repo_base}/blob/main/{}", + reference.trim_start_matches('/') + ) + } +} + +pub(super) fn escape_html(value: &str) -> String { + let mut escaped = String::with_capacity(value.len()); + for character in value.chars() { + match character { + '&' => escaped.push_str("&"), + '<' => escaped.push_str("<"), + '>' => escaped.push_str(">"), + '"' => escaped.push_str("""), + '\'' => escaped.push_str("'"), + character if character.is_ascii() => escaped.push(character), + character => write!(escaped, "&#{};", character as u32).expect("write entity"), + } + } + escaped +} + +pub(super) const SCRIPT: &str = r#"const search = document.getElementById("model-search"); +const groups = Array.from(document.querySelectorAll(".model-group")); +const status = document.getElementById("filter-status"); + +search.addEventListener("input", () => { + const query = search.value.trim().toLocaleLowerCase(); + let shown = 0; + for (const group of groups) { + const visible = !query || group.dataset.modelSearch.includes(query); + group.hidden = !visible; + if (visible) shown += group.querySelectorAll(".result-row").length; + } + status.textContent = `Showing ${shown} ${shown === 1 ? "stack" : "stacks"}.`; +}); +"#; + +pub(super) const STYLE: &str = r#":root { + color-scheme: light; + /* Instrument-document foundation. These remain unused until the new figures land. */ + --design-paper: #fcfbf9; + --design-ink: #1c1c1c; + --design-grey-1: #4f4e4b; + --design-grey-2: #8c8a85; + --design-grey-3: #d5d2cc; + --design-pass: #00513a; /* Okabe-Ito green #009e73, darkened toward L30. */ + --design-fail: #d55e00; + --design-link-accent: #0f6d6d; /* Underlined links only. */ + --type-prose: 16px; + --type-table: 13px; + --type-raster-label: 11px; + --type-heading-1: 20px; + --type-heading-2: 17px; + --type-heading-weight: 600; + --measure-prose: 46rem; + --measure-figure: 76rem; + --ink: #17212b; + --muted: #52606d; + --line: #c9d2da; + --paper: #f7f8f9; + --panel: #ffffff; + --accent: #135f69; + --pass-bg: #d8efdf; + --pass-ink: #17452a; + --partial-bg: #fff0bf; + --partial-ink: #594200; + --none-bg: #f5d8dc; + --none-ink: #681f29; + --neutral-bg: #e8edf1; + --neutral-ink: #35434f; + font-family: "IBM Plex Sans", "Helvetica Neue", Helvetica, Arial, sans-serif; + font-size: 16px; + line-height: 1.55; +} + +* { box-sizing: border-box; } + +body { + margin: 0; + color: var(--ink); + background: var(--paper); +} + +a { color: #075f8a; text-underline-offset: 0.16em; } +a:hover { text-decoration-thickness: 2px; } +a:focus-visible, select:focus-visible, summary:focus-visible { + outline: 3px solid #e5901a; + outline-offset: 3px; +} + +.site-header { + display: flex; + align-items: center; + justify-content: space-between; + gap: 2rem; + padding: 1rem max(1.25rem, calc((100vw - 90rem) / 2)); + color: #ffffff; + background: #15313a; + border-bottom: 4px solid #4ca1a9; +} + +.wordmark { + color: #ffffff; + font-family: ui-monospace, SFMono-Regular, Menlo, Consolas, monospace; + font-size: 1.1rem; + font-weight: 800; + text-decoration: none; + letter-spacing: 0.04em; +} + +nav { display: flex; flex-wrap: wrap; gap: 1.25rem; } +nav a { color: #dcebed; font-weight: 650; text-decoration: none; } +nav a[aria-current="page"] { color: #ffffff; text-decoration: underline; } + +main, footer { + width: min(90rem, calc(100% - 2.5rem)); + margin-inline: auto; +} + +.methods { + max-width: 75rem; + padding: 4rem 0 2.5rem; +} + +.methods p:not(.eyebrow) { max-width: 76ch; font-size: 1.06rem; } + +.eyebrow { + margin: 0 0 0.4rem; + color: var(--accent); + font-size: 0.78rem; + font-weight: 800; + letter-spacing: 0.12em; + text-transform: uppercase; +} + +h1, h2 { margin: 0 0 1rem; line-height: 1.12; } +h1 { font-size: clamp(2.2rem, 5vw, 4.4rem); letter-spacing: -0.045em; } +h2 { font-size: clamp(1.45rem, 2.5vw, 2.1rem); letter-spacing: -0.025em; } + +.analysis { max-width: var(--measure-figure); margin-bottom: 4rem; } +.analysis > p { max-width: var(--measure-prose); } +.svg-figure { margin: 2.5rem 0 3.5rem; color: var(--design-ink); } +.svg-figure figcaption { max-width: var(--measure-prose); margin-bottom: 1rem; } +.svg-figure figcaption > span { display: block; margin-top: 0.45rem; } +.svg-figure figcaption > strong { font-size: var(--type-heading-2); font-weight: var(--type-heading-weight); } +.does-not-show { color: var(--design-grey-1); } +.figure-scroll { overflow-x: auto; padding: 0.5rem; background: var(--design-paper); border: 1px solid var(--design-grey-3); } +.svg-figure svg { display: block; max-width: none; font-family: ui-monospace, SFMono-Regular, Menlo, Consolas, monospace; } +.raster-row-label, .signature-stack-label, +.plot-panel-label, .not-measurable-label, .strip-tick-label { fill: var(--design-ink); } +.raster-row-label, .signature-stack-label, .raster-column-label, +.not-measurable-label { font-size: var(--type-raster-label); } +.raster-column-label { fill: var(--design-grey-1); } +.raster-group-label, .plot-panel-label { fill: var(--design-ink); font-size: 12px; font-weight: 700; } +.strip-tick-label { font-size: 11px; } + +.matrix { + margin-bottom: 4rem; + padding: 1.5rem; + background: var(--panel); + border: 1px solid var(--line); + box-shadow: 0 10px 30px rgb(23 33 43 / 8%); +} + +.matrix-heading { + display: flex; + align-items: end; + justify-content: space-between; + gap: 2rem; +} + +label { color: var(--muted); font-size: 0.84rem; font-weight: 750; } +select { + display: block; + min-width: 11rem; + margin-top: 0.35rem; + padding: 0.65rem 2.25rem 0.65rem 0.75rem; + color: var(--ink); + background: #ffffff; + border: 1px solid #81909c; + border-radius: 0.2rem; + font: inherit; +} + +.legend { display: flex; flex-wrap: wrap; gap: 1.25rem; margin: 1.25rem 0 0; color: var(--muted); font-size: 0.82rem; } +.legend span { display: inline-flex; align-items: center; gap: 0.4rem; } +.swatch { width: 0.85rem; height: 0.85rem; border: 1px solid rgb(23 33 43 / 25%); } +.swatch.all-pass, .score.all-pass { color: var(--pass-ink); background: var(--pass-bg); } +.swatch.partial, .score.partial { color: var(--partial-ink); background: var(--partial-bg); } +.swatch.none-pass, .score.none-pass { color: var(--none-ink); background: var(--none-bg); } +.score.untested { color: var(--neutral-ink); background: var(--neutral-bg); } +/* Not measurable is neutral, never red: the combination produced no measurement, + which is not the same claim as failing. The hatch keeps it distinct from the + other states without relying on colour. */ +.swatch.not-measurable, .score.not-measurable { + color: var(--neutral-ink); + background: repeating-linear-gradient( + 45deg, var(--neutral-bg), var(--neutral-bg) 3px, + rgb(23 33 43 / 12%) 3px, rgb(23 33 43 / 12%) 6px); +} +.measurement-state { display: block; margin-top: 0.35rem; font-size: 0.68rem; line-height: 1.25; } + +.filter-status { margin: 0.7rem 0 1rem; color: var(--muted); font-size: 0.86rem; } +.table-scroll { overflow-x: auto; border: 1px solid var(--line); } +table { width: 100%; min-width: 70rem; border-collapse: collapse; } +th, td { padding: 0.85rem; text-align: left; border: 1px solid var(--line); } +thead th { color: #ffffff; background: #284852; font-size: 0.78rem; } +thead code { color: inherit; } +.result-row > th { width: 18rem; background: #f1f4f6; } +.result-row > th strong, .result-row > th span { display: block; } +.result-row > th strong { margin-bottom: 0.35rem; font-size: 0.98rem; } +.result-row > th span { color: var(--muted); font-size: 0.78rem; font-weight: 500; } +.score { min-width: 8rem; text-align: center; } +.ratio { display: block; color: inherit; font: 800 1.05rem/1.2 ui-monospace, SFMono-Regular, Menlo, Consolas, monospace; } +.legacy-evidence { display: block; margin-top: 0.35rem; font-size: 0.68rem; line-height: 1.25; } +.detail-row > td { padding: 0; background: #fbfcfc; } +.detail-row details { padding: 0.8rem 1rem; } +.detail-row summary { width: fit-content; color: #075f8a; cursor: pointer; font-weight: 700; } + +.metadata { + display: grid; + grid-template-columns: repeat(auto-fit, minmax(13rem, 1fr)); + gap: 0.75rem; + margin: 1rem 0; +} +.metadata div { min-width: 0; padding: 0.7rem; background: #eef2f4; } +.metadata dt { color: var(--muted); font-size: 0.7rem; font-weight: 800; text-transform: uppercase; } +.metadata dd { margin: 0.2rem 0 0; overflow-wrap: anywhere; } +.scenario-list { margin: 1rem 0 0; padding-left: 1.75rem; } +.scenario { padding: 0.5rem 0 0.5rem 0.25rem; border-bottom: 1px solid #e0e5e9; } +.scenario:last-child { border-bottom: 0; } +.status-label { margin-left: 0.4rem; font-size: 0.72rem; font-weight: 850; text-transform: uppercase; } +.status-pass .status-label { color: #236d3d; } +.status-fail .status-label, .status-error .status-label { color: #9a2535; } +.failure-reason { display: inline; color: var(--muted); } +.failure-reason::before { content: "- "; } +.transcript { margin-left: 0.55rem; font-size: 0.85rem; } +.annotation { margin-left: 0.45rem; font-size: 0.72rem; } + +.submit-page { max-width: 58rem; } +.submit-page section { margin-bottom: 2.5rem; } +pre { overflow-x: auto; padding: 1.25rem; color: #eef7f8; background: #18343d; border-left: 4px solid #4ca1a9; } +code { font-family: ui-monospace, SFMono-Regular, Menlo, Consolas, monospace; } +.checklist { padding-left: 1.3rem; } +.checklist li { margin-bottom: 0.65rem; } + +footer { padding: 1.5rem 0 3rem; color: var(--muted); border-top: 1px solid var(--line); font-size: 0.86rem; } + +@media (prefers-color-scheme: dark) { + :root { + --design-paper: #131417; + --design-ink: #e8e6e1; + --design-grey-1: #c8c5bf; + --design-grey-2: #8d8c89; + --design-grey-3: #4b4c50; + --design-pass: #39b990; + --design-fail: #f1843d; + --design-link-accent: #69b9b9; + } +} + +@media (max-width: 44rem) { + .site-header, .matrix-heading { align-items: flex-start; flex-direction: column; gap: 1rem; } + .site-header { padding-inline: 1.25rem; } + main, footer { width: min(100% - 1.5rem, 90rem); } + .methods { padding-top: 2.5rem; } + .matrix { padding: 1rem; } + nav { gap: 1rem; } +} + +@media print { + body { background: #ffffff; } + .site-header { color: #000000; background: #ffffff; border-color: #000000; } + .wordmark, nav a { color: #000000; } + .matrix { box-shadow: none; } + label, .filter-status { display: none; } + .table-scroll { overflow: visible; } + table { min-width: 0; font-size: 9pt; } + a { color: inherit; } +} + +/* Brief 9: calibration-sheet presentation. Colour is reserved for data marks. */ +:root { + color-scheme: light; + --paper: #f8f7f3; + --panel: #f8f7f3; + --ink: var(--design-ink); + --muted: var(--design-grey-1); + --line: var(--design-grey-3); + --neutral-bg: #ebe9e3; + --neutral-ink: var(--design-ink); + --pass-bg: #d8e7df; + --pass-ink: #073e2e; + --partial-bg: #eee7d1; + --partial-ink: #403b28; + --none-bg: #efdcd3; + --none-ink: #552b1b; + font-family: "IBM Plex Sans", "Helvetica Neue", Helvetica, Arial, sans-serif; + font-size: var(--type-prose); +} + +body { background: var(--paper); color: var(--ink); } +a { color: inherit; text-decoration-thickness: 1px; } +a:focus-visible, input:focus-visible, summary:focus-visible { + outline: 2px solid var(--paper); + outline-offset: 2px; + box-shadow: 0 0 0 4px var(--ink); +} +code, .wordmark, .ratio, input { font-family: "IBM Plex Mono", "SFMono-Regular", Consolas, monospace; } + +.site-header { + width: min(var(--measure-figure), calc(100% - 2.5rem)); + margin-inline: auto; + padding: 0.8rem 0; + color: var(--ink); + background: transparent; + border-bottom: 1px solid var(--ink); +} +.wordmark, nav a, nav a[aria-current="page"] { color: var(--ink); } +.wordmark { font-size: 0.92rem; font-weight: 600; letter-spacing: 0.08em; } +nav a { font-size: 0.82rem; font-weight: 500; } + +main, footer { width: 100%; } +.register-intro, .methods, .submit-page, .submit-page section { + width: min(var(--measure-prose), calc(100% - 2.5rem)); + margin-inline: auto; +} +.register-intro { padding: 3.25rem 0 1.5rem; } +.register-intro h1 { max-width: 38rem; } +.framing { max-width: var(--measure-prose); margin: 0; font-size: 1.05rem; } +.eyebrow { color: var(--ink); font-size: 0.7rem; font-weight: 600; } +h1 { font-size: clamp(2rem, 5vw, 3.65rem); font-weight: 500; } +h2 { font-size: var(--type-heading-1); font-weight: var(--type-heading-weight); letter-spacing: 0; } +h3 { font-size: var(--type-heading-2); font-weight: var(--type-heading-weight); } + +.reading-key { margin-top: 0.7rem; padding: 0.6rem 0; border-block: 1px solid var(--line); } +.reading-caption { margin: 0 0 0.5rem; font-size: 0.75rem; } +.mark-key, .method-key { display: flex; flex-wrap: wrap; gap: 0.55rem 1.15rem; } +.method-key { margin-top: 0.5rem; padding-top: 0.5rem; border-top: 1px dotted var(--line); } +.mark-key span, .method-key > span { display: inline-flex; align-items: center; gap: 0.4rem; font-size: 0.72rem; } +.state-key, .mini-mark { position: relative; display: inline-block; width: 0.78rem; height: 0.78rem; flex: none; } +.state-key.state-pass, .mini-mark.pass { background: var(--design-pass); } +.state-key.state-fail, .mini-mark.fail { background: var(--design-fail); } +.state-key.state-fail::after, .mini-mark.fail::after, +.state-key.state-error::after, .mini-mark.error::after { + content: ""; position: absolute; inset: 48% -1px auto; border-top: 1.5px solid var(--design-paper); transform: rotate(45deg); +} +.state-key.state-error, .mini-mark.error { border: 1px solid var(--design-ink); } +.state-key.state-error::after, .mini-mark.error::after { border-color: var(--design-ink); border-width: 1px; } +.state-key.state-skipped, .mini-mark.skipped { border: 1px dotted var(--design-grey-2); opacity: 0.6; } +.hatch-key { width: 1.15rem; height: 0.78rem; border: 1px dashed var(--ink); background: repeating-linear-gradient(135deg, transparent 0 3px, rgb(28 28 28 / 18%) 3px 4px); } +.decode-badge { + display: inline-block; + padding: 0.13rem 0.38rem; + border: 1px solid var(--ink); + border-radius: 0; + color: var(--ink); + background: transparent; + font: 500 0.67rem/1.2 "IBM Plex Mono", "SFMono-Regular", Consolas, monospace; +} +.decode-badge.grammar_constrained { color: var(--paper); background: var(--ink); } +.decode-badge.unknown { border-style: dashed; } +.identity-status { border-left: 2px solid var(--ink); } +i.identity-status { display: inline-block; width: 0.55rem; height: 0.7rem; } +.identity-status.unresolved { border-left-style: dashed; } + +.matrix, .analysis, .appendix { + width: min(var(--measure-figure), calc(100% - 2.5rem)); + max-width: none; + margin: 0 auto 3.5rem; +} +.matrix { padding: 1rem 0 0; background: transparent; border: 0; border-top: 2px solid var(--ink); box-shadow: none; } +.matrix-heading { align-items: start; } +.matrix-heading h2 { margin-bottom: 0; } +label { color: var(--ink); font-size: 0.72rem; font-weight: 600; } +input[type="search"] { + display: block; + width: min(20rem, 70vw); + margin-top: 0.3rem; + padding: 0.5rem 0.6rem; + color: var(--ink); + background: var(--paper); + border: 1px solid var(--ink); + border-radius: 0; + font-size: 0.82rem; +} +.matrix-note, .filter-status { max-width: var(--measure-prose); color: var(--muted); font-size: 0.75rem; } +.matrix-note { margin: 0.55rem 0 0; } +.filter-status { margin: 0.2rem 0 0.45rem; } +.table-scroll { border: 1px solid var(--ink); } +.matrix-table { min-width: 68rem; font-size: var(--type-table); } +.matrix-table th, .matrix-table td { padding: 0.44rem 0.5rem; vertical-align: middle; border-color: var(--line); } +.matrix-table thead th { color: var(--paper); background: var(--ink); font-size: 0.7rem; font-weight: 500; } +.matrix-table thead th:first-child, .result-row > th { position: sticky; left: 0; z-index: 2; } +.matrix-table thead th:first-child { z-index: 4; } +.model-heading th { padding: 0.55rem 0.5rem 0.3rem; color: var(--ink); background: var(--paper); border-top: 2px solid var(--ink); border-bottom: 0; font: 500 0.9rem/1.3 "IBM Plex Mono", "SFMono-Regular", Consolas, monospace; } +.model-heading th > span { margin-right: 0.6rem; color: var(--muted); font: 500 0.62rem/1 "IBM Plex Sans", sans-serif; letter-spacing: 0.08em; text-transform: uppercase; } +.decode-band th { padding: 0.35rem 0.6rem; color: var(--muted); background: var(--neutral-bg); border-block: 1px solid var(--ink); font-size: 0.68rem; font-weight: 400; } +.decode-band .decode-badge { margin-right: 0.7rem; } +.result-row > th { width: 22rem; background: var(--paper); } +.result-row > th strong { margin: 0; overflow: hidden; font: 500 0.79rem/1.3 "IBM Plex Mono", "SFMono-Regular", Consolas, monospace; text-overflow: ellipsis; white-space: nowrap; } +.row-meta { display: flex; align-items: center; gap: 0.45rem; margin-top: 0.18rem; white-space: nowrap; } +.result-row > th .row-meta > span { display: inline-block; color: var(--muted); font-size: 0.63rem; font-weight: 400; } +.result-row > th .decode-badge { display: inline-block; margin: 0; color: var(--ink); font-size: 0.61rem; } +.result-row > th .decode-badge.grammar_constrained { color: var(--paper); } +.result-row > th .identity-status { margin: 0; padding-left: 0.3rem; } +.detail-link { display: inline-block; margin: 0; font-size: 0.63rem; font-weight: 500; } +.score { position: relative; min-width: 7.5rem; text-align: center; } +.score.low-replication { + background-image: repeating-linear-gradient(135deg, transparent 0 5px, rgb(28 28 28 / 12%) 5px 6px); + background-blend-mode: multiply; +} +.ratio { font-size: 0.9rem; font-weight: 500; } +.measurement-state, .legacy-evidence { display: block; margin-top: 0.2rem; font-size: 0.58rem; line-height: 1.2; } + +.analysis { border-top: 2px solid var(--ink); padding-top: 1rem; } +.analysis-primary { margin-top: -1.5rem; } +.analysis > p { max-width: var(--measure-prose); } +.svg-figure { margin: 1.5rem 0 3.25rem; } +.svg-figure figcaption { max-width: var(--measure-prose); } +.figure-scroll { padding: 0.5rem; background: var(--design-paper); border-color: var(--ink); } +.figure-links { font-size: 0.75rem; } +.desktop-raster { display: block; } +.mobile-raster { display: none; } + +.methods { max-width: var(--measure-prose); padding: 1rem 0 3.5rem; border-top: 2px solid var(--ink); } +.methods p:not(.eyebrow) { max-width: var(--measure-prose); font-size: 1rem; } +.appendix { padding-top: 1rem; border-top: 2px solid var(--ink); } +.appendix > p { max-width: var(--measure-prose); } +.stack-detail { margin: 2rem 0 3rem; padding-top: 0.8rem; border-top: 1px solid var(--ink); } +.stack-detail header { display: flex; align-items: baseline; flex-wrap: wrap; gap: 0.5rem 0.8rem; } +.stack-detail header h3 { margin: 0; } +.stack-detail header h3 a { text-underline-offset: 0.15em; } +.stack-content { display: none; } +.stack-detail:target { scroll-margin-top: 1rem; border-top-width: 2px; } +.stack-detail:target .stack-content { display: block; } +.detail-index { margin: 0; color: var(--muted); font: 500 0.65rem/1 "IBM Plex Mono", monospace; text-transform: uppercase; } +.metadata { grid-template-columns: repeat(auto-fit, minmax(12rem, 1fr)); gap: 1px; border: 1px solid var(--line); background: var(--line); } +.metadata div { padding: 0.55rem; background: var(--paper); } +.metadata dt { color: var(--muted); font-size: 0.6rem; font-weight: 600; } +.metadata dd { font-size: 0.74rem; } +.scenario-list { padding-left: 1.5rem; } +.scenario { border-color: var(--line); } +.scenario > p { margin: 0.25rem 0; max-width: var(--measure-prose); font-size: 0.78rem; } +.scenario .rationale { color: var(--muted); } +.failure-detail, .evidence-hash { display: inline-block; margin: 0.3rem 0 0 0.45rem; color: var(--muted); font-size: 0.68rem; } +.evidence-hash { max-width: 100%; overflow-wrap: anywhere; word-break: break-all; } +.back-link { font-size: 0.75rem; } + +.colophon { + width: min(var(--measure-figure), calc(100% - 2.5rem)); + margin-inline: auto; + padding: 0.7rem 0; + color: var(--muted); + border-top: 1px solid var(--ink); + font-size: 0.66rem; +} +.colophon p { margin: 0; } + +pre { color: var(--paper); background: var(--ink); border-left: 0; } + +@media (prefers-color-scheme: dark) { + :root { + color-scheme: dark; + --paper: var(--design-paper); + --panel: var(--design-paper); + --ink: var(--design-ink); + --muted: var(--design-grey-1); + --line: var(--design-grey-3); + --neutral-bg: #24262a; + --neutral-ink: var(--design-ink); + --pass-bg: #18372e; + --pass-ink: #c9f0df; + --partial-bg: #393426; + --partial-ink: #eee3bd; + --none-bg: #422a23; + --none-ink: #f4d2c5; + } + .score.low-replication { background-blend-mode: screen; } + .hatch-key { background-image: repeating-linear-gradient(135deg, transparent 0 3px, rgb(232 230 225 / 22%) 3px 4px); } +} + +@media (max-width: 46rem) { + .site-header, .matrix-heading { align-items: flex-start; flex-direction: column; gap: 0.8rem; } + .site-header, .register-intro, .methods, .matrix, .analysis, .appendix, .colophon { + width: min(100% - 1.5rem, var(--measure-figure)); + } + .register-intro { padding-top: 2.25rem; } + .mark-key, .method-key { display: grid; grid-template-columns: 1fr 1fr; } + input[type="search"] { width: min(100%, 22rem); } + .table-scroll { overflow: visible; border: 0; } + .matrix-table, .matrix-table tbody, .matrix-table tr { display: block; min-width: 0; width: 100%; } + .matrix-table thead { position: absolute; width: 1px; height: 1px; overflow: hidden; clip-path: inset(50%); } + .model-group { margin-bottom: 1rem; border: 1px solid var(--ink); } + .model-group.single-row { margin-bottom: 0; border: 0; } + .model-heading th, .decode-band th { display: block; width: 100%; border-inline: 0; } + .result-row { display: grid !important; grid-template-columns: repeat(3, minmax(0, 1fr)); border-bottom: 1px solid var(--ink); } + .result-row:last-child { border-bottom: 0; } + .result-row > th { position: static; grid-column: 1 / -1; width: auto; border: 0; border-bottom: 1px solid var(--line); } + .row-meta { gap: 0.3rem; } + .result-row > th .row-meta > span, .row-meta .detail-link { font-size: 0.58rem; } + .result-row > td { display: block; min-width: 0; padding: 0.5rem 0.25rem; border-width: 0 1px 1px 0; } + .score::before { content: attr(data-label); display: block; min-height: 2.2em; margin-bottom: 0.25rem; color: currentColor; font-size: 0.57rem; line-height: 1.1; } + .ratio { font-size: 0.78rem; } + .analysis-primary .figure-scroll { display: none; } + .mobile-raster { display: block; } + .mini-raster-panel { margin: 1.25rem 0; overflow-x: auto; } + .mini-raster-panel h3 { position: sticky; left: 0; margin-bottom: 0.4rem; font-size: 0.78rem; } + .mini-raster-grid { display: grid; align-items: center; gap: 2px; width: max-content; font: 0.58rem/1.1 "IBM Plex Mono", monospace; } + .mini-raster-corner, .mini-row-label { position: sticky; left: 0; z-index: 2; width: 8.5rem; padding-right: 0.35rem; overflow: hidden; background: var(--paper); text-align: right; text-overflow: ellipsis; white-space: nowrap; } + .mini-column-label { width: 0.78rem; overflow: hidden; writing-mode: vertical-rl; transform: rotate(180deg); white-space: nowrap; } +} + +@media print { + :root { color-scheme: light; --paper: #fff; --ink: #000; --muted: #333; --line: #aaa; } + .site-header { color: #000; background: #fff; border-color: #000; } + .wordmark, nav a { color: #000; } + label, .filter-status, .detail-link, .back-link { display: none; } + .table-scroll { overflow: visible; } + .matrix-table { min-width: 0; font-size: 7pt; } + .matrix-table thead th:first-child, .result-row > th { position: static; } + .analysis-primary .figure-scroll { display: block; } + .mobile-raster { display: none; } + .stack-content { display: block; } + a { color: inherit; } +} +"#; diff --git a/crates/willitcall/src/site/svg.rs b/crates/willitcall/src/site/svg.rs new file mode 100644 index 0000000..a840fa6 --- /dev/null +++ b/crates/willitcall/src/site/svg.rs @@ -0,0 +1,631 @@ +use std::fmt::Write as _; + +const DENSE_MARK_SIZE: u32 = 13; +const AGGREGATE_MARK_SIZE: u32 = 44; + +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub(super) struct FigureAccessibility<'a> { + pub number: u8, + pub title: &'a str, + pub description: &'a str, + pub caption: &'a str, + pub does_not_show: &'a str, +} + +/// A closed set of figure primitives. Each variant requires the same accessibility +/// contract, and the module exposes no alternate complete-SVG renderer. +#[allow(dead_code)] // Brief 8 constructs the first production figure. +pub(super) enum Figure<'a> { + StatusMarks { + accessibility: FigureAccessibility<'a>, + width: u32, + height: u32, + marks: &'a [StatusMark<'a>], + }, + AggregateMarks { + accessibility: FigureAccessibility<'a>, + width: u32, + height: u32, + marks: &'a [AggregateMark<'a>], + }, + StatusRaster { + accessibility: FigureAccessibility<'a>, + width: u32, + height: u32, + marks: &'a [StatusMark<'a>], + texts: &'a [FigureText<'a>], + rules: &'a [FigureRule], + }, + StripPlot { + accessibility: FigureAccessibility<'a>, + width: u32, + height: u32, + dots: &'a [StripDot<'a>], + not_fully_measurable: &'a [NotFullyMeasurableMark<'a>], + texts: &'a [FigureText<'a>], + rules: &'a [FigureRule], + }, +} + +pub(super) fn render_figure(figure: Figure<'_>) -> String { + match figure { + Figure::StatusMarks { + accessibility, + width, + height, + marks, + } => render_status_marks(accessibility, width, height, marks), + Figure::AggregateMarks { + accessibility, + width, + height, + marks, + } => render_aggregate_marks(accessibility, width, height, marks), + Figure::StatusRaster { + accessibility, + width, + height, + marks, + texts, + rules, + } => render_status_raster(accessibility, width, height, marks, texts, rules), + Figure::StripPlot { + accessibility, + width, + height, + dots, + not_fully_measurable, + texts, + rules, + } => render_strip_plot( + accessibility, + (width, height), + dots, + not_fully_measurable, + texts, + rules, + ), + } +} + +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub(super) enum TextAnchor { + Start, + Middle, + End, +} + +impl TextAnchor { + fn as_str(self) -> &'static str { + match self { + Self::Start => "start", + Self::Middle => "middle", + Self::End => "end", + } + } +} + +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub(super) struct FigureText<'a> { + pub x: u32, + pub y: u32, + pub text: &'a str, + pub class: &'static str, + pub anchor: TextAnchor, + pub rotation: Option, +} + +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub(super) struct FigureRule { + pub x1: u32, + pub y1: u32, + pub x2: u32, + pub y2: u32, + pub class: &'static str, +} + +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub(super) struct StripDot<'a> { + pub x: u32, + pub y: u32, + pub pass_count: usize, + pub label: &'a str, +} + +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub(super) struct NotFullyMeasurableMark<'a> { + pub x: u32, + pub y: u32, + pub label: &'a str, + pub detail: &'a str, +} + +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +#[allow(dead_code)] // Brief 8 maps result statuses into these production marks. +pub(super) enum MarkState { + Pass, + Fail, + Error, + Skipped, +} + +impl MarkState { + fn label(self) -> &'static str { + match self { + Self::Pass => "pass", + Self::Fail => "fail", + Self::Error => "error", + Self::Skipped => "skipped", + } + } +} + +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub(super) struct StatusMark<'a> { + pub x: u32, + pub y: u32, + pub label: &'a str, + pub state: MarkState, +} + +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub(super) struct AggregateMark<'a> { + pub x: u32, + pub y: u32, + pub label: &'a str, + pub state: MarkState, + pub passed: u32, + pub measured: u32, + pub no_verdict: bool, +} + +/// Renders prepared 13 px status marks. Positions and statuses are supplied by the caller. +fn render_status_marks( + accessibility: FigureAccessibility<'_>, + width: u32, + height: u32, + marks: &[StatusMark<'_>], +) -> String { + let mut body = String::new(); + + for mark in marks { + writeln!( + body, + " ", + mark.state.label(), + escape_html(mark.label), + mark.state.label() + ) + .expect("write SVG"); + write_mark_shape(&mut body, mark.state, mark.x, mark.y, DENSE_MARK_SIZE); + body.push_str(" \n"); + } + + render_accessible_figure(accessibility, width, height, &body) +} + +/// Renders prepared 44 px aggregate cells. The caller explicitly selects `no_verdict` per cell. +fn render_aggregate_marks( + accessibility: FigureAccessibility<'_>, + width: u32, + height: u32, + marks: &[AggregateMark<'_>], +) -> String { + let mut body = String::new(); + + for mark in marks { + writeln!( + body, + " ", + mark.state.label(), + escape_html(mark.label), + mark.passed, + mark.measured, + mark.state.label() + ) + .expect("write SVG"); + write_mark_shape(&mut body, mark.state, mark.x, mark.y, AGGREGATE_MARK_SIZE); + let text_ink = match mark.state { + MarkState::Pass | MarkState::Fail => "var(--design-paper, #fcfbf9)", + MarkState::Error | MarkState::Skipped => "var(--design-ink, #1c1c1c)", + }; + writeln!( + body, + " {}/{}", + mark.x + AGGREGATE_MARK_SIZE / 2, + mark.y + AGGREGATE_MARK_SIZE / 2 + 3, + mark.passed, + mark.measured + ) + .expect("write SVG"); + if mark.no_verdict { + body.push_str(&render_no_verdict_overlay(mark.x, mark.y)); + } + body.push_str(" \n"); + } + + render_accessible_figure(accessibility, width, height, &body) +} + +fn render_status_raster( + accessibility: FigureAccessibility<'_>, + width: u32, + height: u32, + marks: &[StatusMark<'_>], + texts: &[FigureText<'_>], + rules: &[FigureRule], +) -> String { + let mut body = String::new(); + write_rules(&mut body, rules); + write_texts(&mut body, texts); + for mark in marks { + write_status_mark(&mut body, mark); + } + render_accessible_figure(accessibility, width, height, &body) +} + +fn render_strip_plot( + accessibility: FigureAccessibility<'_>, + size: (u32, u32), + dots: &[StripDot<'_>], + not_fully_measurable: &[NotFullyMeasurableMark<'_>], + texts: &[FigureText<'_>], + rules: &[FigureRule], +) -> String { + let (width, height) = size; + let mut body = String::new(); + write_rules(&mut body, rules); + write_texts(&mut body, texts); + for dot in dots { + writeln!( + body, + " \n {}\n \n ", + escape_html(dot.label), + escape_html(dot.label), + dot.pass_count, + dot.x, + dot.y, + ) + .expect("write SVG strip dot"); + } + for mark in not_fully_measurable { + writeln!( + body, + " \n {}: {}", + escape_html(mark.label), + escape_html(mark.detail), + escape_html(mark.label), + escape_html(mark.detail), + ) + .expect("write SVG not-fully-measurable mark"); + write_mark_shape(&mut body, MarkState::Error, mark.x, mark.y, DENSE_MARK_SIZE); + body.push_str(" \n"); + } + render_accessible_figure(accessibility, width, height, &body) +} + +fn write_status_mark(svg: &mut String, mark: &StatusMark<'_>) { + writeln!( + svg, + " \n {}: {}", + mark.state.label(), + escape_html(mark.label), + mark.state.label(), + escape_html(mark.label), + mark.state.label(), + ) + .expect("write SVG status mark"); + write_mark_shape(svg, mark.state, mark.x, mark.y, DENSE_MARK_SIZE); + svg.push_str(" \n"); +} + +fn write_texts(svg: &mut String, texts: &[FigureText<'_>]) { + for text in texts { + write!( + svg, + " {}", escape_html(text.text)).expect("write SVG text"); + } +} + +fn write_rules(svg: &mut String, rules: &[FigureRule]) { + for rule in rules { + writeln!( + svg, + " ", + rule.class, rule.x1, rule.y1, rule.x2, rule.y2 + ) + .expect("write SVG rule"); + } +} + +/// Returns only the aggregate-cell overlay so dense raster renderers cannot apply it implicitly. +fn render_no_verdict_overlay(x: u32, y: u32) -> String { + let mut overlay = + String::from(" \n"); + let end = AGGREGATE_MARK_SIZE; + for offset in (6..end).step_by(7) { + writeln!( + overlay, + " ", + x + offset, + y, + x + end, + y + end - offset, + x, + y + offset, + x + end - offset, + y + end + ) + .expect("write SVG"); + } + write!( + overlay, + " \n n=1\n \n", + x, + y, + x + end / 2, + y + end - 5 + ) + .expect("write SVG"); + overlay +} + +fn write_mark_shape(svg: &mut String, state: MarkState, x: u32, y: u32, size: u32) { + let inset = 1; + let mark_size = size - inset * 2; + let start = inset + 1; + let end = size - inset - 1; + match state { + MarkState::Pass => { + writeln!( + svg, + " ", + x + inset, + y + inset + ) + .expect("write SVG"); + } + MarkState::Fail => { + write!( + svg, + " \n \n", + x + inset, + y + inset, + x + start, + y + start, + x + end, + y + end + ) + .expect("write SVG"); + } + MarkState::Error => { + write!( + svg, + " \n \n", + x + inset, + y + inset, + x + start, + y + start, + x + end, + y + end + ) + .expect("write SVG"); + } + MarkState::Skipped => { + writeln!( + svg, + " ", + x + inset, + y + inset + ) + .expect("write SVG"); + } + } +} + +fn render_accessible_figure( + accessibility: FigureAccessibility<'_>, + width: u32, + height: u32, + body: &str, +) -> String { + let mut figure = String::new(); + write!( + figure, + "
    \n
    \n Figure {}. {}\n {}\n What this does not show: {}\n Method and limitations - Data: JSON, CSV\n
    \n
    \n \n Figure {}. {}\n {}\n{body} \n
    \n
    \n", + accessibility.number, + accessibility.number, + escape_html(accessibility.title), + escape_html(accessibility.caption), + escape_html(accessibility.does_not_show), + accessibility.number, + escape_html(accessibility.title), + escape_html(accessibility.description), + ) + .expect("write SVG figure"); + figure +} + +fn escape_html(value: &str) -> String { + let mut escaped = String::with_capacity(value.len()); + for character in value.chars() { + match character { + '&' => escaped.push_str("&"), + '<' => escaped.push_str("<"), + '>' => escaped.push_str(">"), + '"' => escaped.push_str("""), + '\'' => escaped.push_str("'"), + character if character.is_ascii() => escaped.push(character), + character => write!(escaped, "&#{};", character as u32).expect("write entity"), + } + } + escaped +} + +#[cfg(test)] +mod tests { + use super::*; + + fn render_one_status(state: MarkState, label: &str) -> String { + render_figure(Figure::StatusMarks { + accessibility: FigureAccessibility { + number: 1, + title: "Status figure", + description: "One prepared status mark.", + caption: "One status observation.", + does_not_show: "Repeated-run behavior.", + }, + width: DENSE_MARK_SIZE, + height: DENSE_MARK_SIZE, + marks: &[StatusMark { + x: 0, + y: 0, + label, + state, + }], + }) + } + + fn aggregate(no_verdict: bool) -> String { + render_figure(Figure::AggregateMarks { + accessibility: FigureAccessibility { + number: 2, + title: "Aggregate figure", + description: "One prepared aggregate mark.", + caption: "One aggregate observation.", + does_not_show: "A ranking.", + }, + width: AGGREGATE_MARK_SIZE, + height: AGGREGATE_MARK_SIZE, + marks: &[AggregateMark { + x: 0, + y: 0, + label: "single call", + state: MarkState::Pass, + passed: 1, + measured: 1, + no_verdict, + }], + }) + } + + #[test] + fn four_states_have_distinct_non_colour_shapes() { + let pass = render_one_status(MarkState::Pass, "pass result"); + assert!(pass.contains("class=\"state-shape state-pass\"")); + assert!(pass.contains("stroke=\"none\"")); + assert!(!pass.contains("state-slash")); + + let fail = render_one_status(MarkState::Fail, "fail result"); + assert!(fail.contains("class=\"state-shape state-fail\"")); + assert!(fail.contains("class=\"state-slash knockout-slash\"")); + assert!(fail.contains("stroke-width=\"1.5\"")); + + let error = render_one_status(MarkState::Error, "error result"); + assert!(error.contains("class=\"state-shape state-error\"")); + assert!(error.contains("class=\"state-slash ink-slash\"")); + assert!(error.contains("fill=\"none\"")); + + let skipped = render_one_status(MarkState::Skipped, "skipped result"); + assert!(skipped.contains("class=\"state-shape state-skipped\"")); + assert!(skipped.contains("stroke-dasharray=\"1 1.75\"")); + assert!(skipped.contains("opacity=\"0.45\"")); + assert!(!skipped.contains("state-slash")); + } + + #[test] + fn every_renderer_includes_title_description_and_data_links() { + for figure in [ + render_one_status(MarkState::Pass, "result"), + aggregate(false), + ] { + assert!(figure.contains("")); + assert!(figure.contains("")); + assert!(figure.contains("")); + assert!(figure.contains("")); + assert!(figure.contains("href=\"results.json\"")); + assert!(figure.contains("href=\"results.csv\"")); + assert!(!figure.contains("svg-text-fallback")); + } + } + + #[test] + fn interpolated_labels_are_escaped_everywhere() { + let figure = render_one_status(MarkState::Pass, "model & \"quoted\""); + assert!(figure.contains("model <x> & "quoted"")); + assert!(!figure.contains("model ")); + assert!(!figure.contains("\"quoted\"")); + } + + #[test] + fn no_verdict_overlay_is_opt_in_and_aggregate_only() { + let with_overlay = aggregate(true); + assert!(with_overlay.contains("class=\"no-verdict-overlay\"")); + assert!(with_overlay.contains("class=\"no-verdict-hatch\"")); + assert!(with_overlay.contains("stroke-dasharray=\"3 2\"")); + assert!(with_overlay.contains("baseline-shift=\"super\">n=1")); + + assert!(!aggregate(false).contains("no-verdict-overlay")); + assert!(!render_one_status(MarkState::Pass, "result").contains("no-verdict-overlay")); + } + + #[test] + fn output_contains_no_external_url() { + let figures = [ + render_one_status(MarkState::Fail, "result"), + aggregate(true), + ]; + for figure in figures { + assert!(!figure.contains("http://")); + assert!(!figure.contains("https://")); + } + } + + #[test] + fn stylesheet_defines_light_and_dark_instrument_tokens() { + let style = super::super::html::STYLE; + for declaration in [ + "--design-paper: #fcfbf9", + "--design-ink: #1c1c1c", + "--design-grey-1: #4f4e4b", + "--design-grey-2: #8c8a85", + "--design-grey-3: #d5d2cc", + "--design-pass: #00513a", + "--design-fail: #d55e00", + "--design-link-accent: #0f6d6d", + "--type-prose: 16px", + "--type-table: 13px", + "--type-raster-label: 11px", + "--type-heading-1: 20px", + "--type-heading-2: 17px", + "--type-heading-weight: 600", + "--measure-prose: 46rem", + "--measure-figure: 76rem", + "@media (prefers-color-scheme: dark)", + "--design-paper: #131417", + "--design-ink: #e8e6e1", + ] { + assert!(style.contains(declaration), "missing {declaration}"); + } + assert!(style.contains("font-family: \"IBM Plex Sans\", \"Helvetica Neue\"")); + assert!(style.contains("font-family: \"IBM Plex Mono\"")); + assert!(!style.contains("@import")); + assert!(!style.contains("fonts.googleapis")); + assert!(!style.contains("fonts.gstatic")); + } +} diff --git a/crates/willitcall/tests/fixtures/result-v2.json b/crates/willitcall/tests/fixtures/result-v2.json new file mode 100644 index 0000000..298d125 --- /dev/null +++ b/crates/willitcall/tests/fixtures/result-v2.json @@ -0,0 +1,44 @@ +{ + "schema_version": 2, + "metadata": { + "run_id": "20260720T190335Z-fixture", + "timestamp": "2026-07-20T19:03:35Z", + "willitcall_version": "0.1.0", + "endpoint": "http://127.0.0.1:8080/v1", + "model_id": "fixture-model", + "declared_quant": "Q3_K_M", + "server": { + "preset_name": "llamacpp", + "reported_version": "fixture-version", + "quirk_flags": [] + }, + "environment": { + "host_hardware_class": "Fixture workstation, 64GB", + "host_os": "Fixture OS" + }, + "sampling": { + "temperature": 0.0, + "top_p": 1.0, + "seed": 42, + "max_tokens": 1024 + } + }, + "scenarios": [ + { + "id": "single-weather", + "category": "single_call", + "status": "error", + "failure_reason": "turn 1: fixture server error", + "evidence_hash": "sha256:fixture", + "evidence_path": "evidence/fixture/single-weather.json", + "retried": false + } + ], + "totals": { + "total": 1, + "passed": 0, + "failed": 0, + "errors": 1, + "skipped": 0 + } +} diff --git a/crates/willitcall/tests/fixtures/site-contract-results/01-v1-unresolved.json b/crates/willitcall/tests/fixtures/site-contract-results/01-v1-unresolved.json new file mode 100644 index 0000000..90d5d31 --- /dev/null +++ b/crates/willitcall/tests/fixtures/site-contract-results/01-v1-unresolved.json @@ -0,0 +1,54 @@ +{ + "schema_version": 1, + "metadata": { + "timestamp": "2000-01-01T00:00:00Z", + "willitcall_version": "site-contract-fixture", + "endpoint": "https://fixture.invalid/v1", + "model_id": "/fixtures/models/legacy-unresolved.gguf", + "declared_quant": null, + "server": { + "preset_name": "ollama", + "reported_version": null, + "quirk_flags": [] + }, + "sampling": { + "temperature": 0.0, + "top_p": 1.0, + "seed": 1, + "max_tokens": 64 + } + }, + "scenarios": [ + { + "id": "v1-pass", + "category": "single_call", + "status": "pass", + "failure_reason": null, + "evidence_hash": null, + "retried": false + }, + { + "id": "v1-fail", + "category": "parallel_calls", + "status": "fail", + "failure_reason": "legacy wrong & call", + "evidence_hash": "sha256:legacy-fixture", + "retried": false + }, + { + "id": "v1-skipped", + "category": "tool_choice_modes", + "status": "skipped", + "failure_reason": "fixture intentionally skipped", + "evidence_hash": null, + "retried": false + } + ], + "totals": { + "total": 3, + "passed": 1, + "failed": 1, + "errors": 0, + "skipped": 1 + } +} diff --git a/crates/willitcall/tests/fixtures/site-contract-results/02-v2-unresolved.json b/crates/willitcall/tests/fixtures/site-contract-results/02-v2-unresolved.json new file mode 100644 index 0000000..60f15d7 --- /dev/null +++ b/crates/willitcall/tests/fixtures/site-contract-results/02-v2-unresolved.json @@ -0,0 +1,83 @@ +{ + "schema_version": 2, + "metadata": { + "run_id": "site-contract-v2", + "timestamp": "2000-01-02T00:00:00Z", + "willitcall_version": "site-contract-fixture", + "endpoint": "https://fixture.invalid/v1", + "model_id": "v2-unresolved:model", + "declared_quant": "Q5_K_M", + "server": { + "preset_name": "mlx_lm", + "reported_version": "fixture-v2", + "quirk_flags": [] + }, + "environment": { + "host_hardware_class": "Fixture workstation", + "host_os": "Fixture OS v2" + }, + "sampling": { + "temperature": 0.0, + "top_p": 1.0, + "seed": 2, + "max_tokens": 64 + } + }, + "scenarios": [ + { + "id": "v2-pass", + "category": "multi_turn", + "status": "pass", + "failure_reason": null, + "evidence_hash": null, + "evidence_path": null, + "retried": false + }, + { + "id": "v2-fail-with-cause", + "category": "multi_turn", + "status": "fail", + "failure_reason": "assistant returned no content ", + "failure_class": "empty_response", + "cause": { + "kind": "server-defect", + "reference": "docs/site-contract-cause.md", + "note": "fixture cause & note" + }, + "evidence_hash": "sha256:v2-fail-fixture", + "evidence_path": "/evidence/site-contract/v2-fail.json", + "retried": false + }, + { + "id": "v2-error", + "category": "streaming", + "status": "error", + "failure_reason": "fixture server error", + "evidence_hash": null, + "evidence_path": null, + "retried": false + }, + { + "id": "v2-unparsed", + "category": "negative_trap", + "status": "fail", + "failure_reason": "fixture tool call was not parsed", + "failure_class": "unparsed_tool_call", + "cause": { + "kind": "unknown", + "reference": null, + "note": null + }, + "evidence_hash": null, + "evidence_path": null, + "retried": false + } + ], + "totals": { + "total": 4, + "passed": 1, + "failed": 2, + "errors": 1, + "skipped": 0 + } +} diff --git a/crates/willitcall/tests/fixtures/site-contract-results/03-v3-declared.json b/crates/willitcall/tests/fixtures/site-contract-results/03-v3-declared.json new file mode 100644 index 0000000..4c6db8c --- /dev/null +++ b/crates/willitcall/tests/fixtures/site-contract-results/03-v3-declared.json @@ -0,0 +1,79 @@ +{ + "schema_version": 3, + "metadata": { + "run_id": "site-contract-v3-declared", + "timestamp": "2000-01-03T00:00:00Z", + "willitcall_version": "site-contract-fixture", + "endpoint": "https://fixture.invalid/v1", + "model": { + "display_name": "/fixtures/models/declared-fixture.gguf", + "family_id": "fixture-declared", + "canonical_id": "Fixture/Declared", + "parameter_count_b": 1.0, + "endpoint_id": "/fixtures/models/declared-fixture.gguf", + "identity_status": "declared", + "artifact": { + "source_kind": "huggingface", + "source_id": "Fixture/Declared-GGUF", + "revision": "fixture-revision", + "sha256": null, + "format": "gguf", + "quantization": { + "label": "Q4_K_M", + "scheme": "k-quant", + "bits": 4 + } + } + }, + "corpus": { + "id": "site-contract", + "revision": "fixture-v1", + "sha256": "sha256:fixture-corpus", + "scenario_count": 1, + "scoring_version": "fixture-v1" + }, + "server": { + "preset_name": "llamacpp", + "reported_version": "fixture-v3-declared", + "quirk_flags": ["grammar_constrained_decoding"], + "decode_mode": "grammar_constrained", + "chat_template": null, + "launch_config_sha256": null + }, + "environment": { + "display_label": "Declared fixture host; Fixture OS v3", + "os_name": "Fixture OS", + "os_version": "v3", + "architecture": "fixture-arch", + "accelerator": null, + "memory_bytes": null + }, + "sampling": { + "temperature": 0.0, + "top_p": 1.0, + "seed": 3, + "max_tokens": 64 + }, + "replication": null, + "arm_fingerprint": null + }, + "scenarios": [ + { + "id": "v3-declared-pass", + "category": "tool_choice_modes", + "status": "pass", + "failure_reason": null, + "failure": null, + "evidence_hash": null, + "evidence_path": null, + "retried": false + } + ], + "totals": { + "total": 1, + "passed": 1, + "failed": 0, + "errors": 0, + "skipped": 0 + } +} diff --git a/crates/willitcall/tests/fixtures/site-contract-results/04-v3-verified.json b/crates/willitcall/tests/fixtures/site-contract-results/04-v3-verified.json new file mode 100644 index 0000000..0cc985d --- /dev/null +++ b/crates/willitcall/tests/fixtures/site-contract-results/04-v3-verified.json @@ -0,0 +1,83 @@ +{ + "schema_version": 3, + "metadata": { + "run_id": "site-contract-v3-verified", + "timestamp": "2000-01-04T00:00:00Z", + "willitcall_version": "site-contract-fixture", + "endpoint": "https://fixture.invalid/v1", + "model": { + "display_name": "Verified & Fixture", + "family_id": "fixture-verified", + "canonical_id": "Fixture/Verified", + "parameter_count_b": 2.0, + "endpoint_id": "verified-fixture:latest", + "identity_status": "verified", + "artifact": { + "source_kind": "local_file", + "source_id": "verified-fixture.gguf", + "revision": null, + "sha256": "sha256:verified-fixture", + "format": "gguf", + "quantization": null + } + }, + "corpus": { + "id": "site-contract", + "revision": "fixture-v1", + "sha256": "sha256:fixture-corpus", + "scenario_count": 1, + "scoring_version": "fixture-v1" + }, + "server": { + "preset_name": "ollama", + "reported_version": "fixture-v3-verified", + "quirk_flags": [], + "decode_mode": "unconstrained_post_hoc", + "chat_template": { + "id": "fixture-template", + "sha256": "sha256:fixture-template" + }, + "launch_config_sha256": "sha256:fixture-launch" + }, + "environment": { + "display_label": "Verified fixture host; Fixture OS v3", + "os_name": "Fixture OS", + "os_version": "v3", + "architecture": "fixture-arch", + "accelerator": "fixture-accelerator", + "memory_bytes": 1024 + }, + "sampling": { + "temperature": 0.0, + "top_p": 1.0, + "seed": 4, + "max_tokens": 64 + }, + "replication": { + "study_id": "site-contract", + "arm_id": "verified", + "run_index": 0, + "mode": "greedy_reproducibility" + }, + "arm_fingerprint": "v1:fixture" + }, + "scenarios": [ + { + "id": "v3-verified-pass", + "category": "negative_trap", + "status": "pass", + "failure_reason": null, + "failure": null, + "evidence_hash": null, + "evidence_path": null, + "retried": false + } + ], + "totals": { + "total": 1, + "passed": 1, + "failed": 0, + "errors": 0, + "skipped": 0 + } +} diff --git a/crates/willitcall/tests/golden/site-contract-index.html b/crates/willitcall/tests/golden/site-contract-index.html new file mode 100644 index 0000000..dc747f8 --- /dev/null +++ b/crates/willitcall/tests/golden/site-contract-index.html @@ -0,0 +1,191 @@ + + + + + + + willitcall support matrix + + + +
    + willitcall + +
    +
    +
    +

    Measurement register

    +

    Tool-calling support matrix

    +

    Each row records one tested stack. The stack includes the model and its artifact. It also includes the server and decode mode. The matrix does not rank models or state verdicts.

    +
    +
    +
    +

    Observed stacks

    +

    Capability matrix

    +
    + +
    +
    +

    How to read: Each ratio shows passed scenarios / all scenarios in that category.

    +
    + pass + model / response failure + execution / server error + not tested +
    +
    + n=1, no verdict: every published arm is a single run, so no cell carries a verdict; hatching marks this. + identity status +
    +
    + +

    Showing 4 stacks.

    +
    + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
    Model / quant / serverSingle callParallel callsStreamingTool choiceMulti-turnCorrectly declines
    + declared-fixture.gguf +
    Q4_K_M / llama.cppgrammar (run)id: declareddetail
    +     		0/00/00/01/10/00/0
    + Verified & Fixture +
    not declared / ollamapost-hoc (run)id: verifieddetail
    +     		0/00/00/00/00/01/1
    + legacy-unresolved.gguf (unverified artifact) +
    not declared / ollamapost-hoc (mapping)id: unresolveddetail
    +     		1/10/1schema v1: no transcript path0/00/1schema v1: no transcript pathnot measurable0/00/0
    + v2-unresolved:model (unverified artifact) +
    Q5_K_M / MLX LMpost-hoc (mapping)id: unresolveddetail
    +     		0/00/00/1not measurable0/01/20/1
    +
    +

    Machine-readable observations: JSON and CSV. Scenario rasters are on the outcomes page; recorded metadata and transcripts are in the appendix.

    +
    +

    One-run observations

    +

    Observed pass counts

    +

    Pass counts are shown in fixed stack order and withheld for rows with errors or skips. They are not scores or a ranking.

    +
    +
    + Figure 4. Observed pass-count strip plot + Each dot shows one published stack observation from a single run. We plot a pass count only when every scenario has a pass or fail verdict. A separate panel lists rows with errors or skips. + What this does not show: This figure does not show a model score distribution or rank models. It does not measure uncertainty or variation across repeated runs. + Method and limitations - Data: JSON, CSV +
    +
    + + Figure 4. Observed pass-count strip plot + One pass-count dot per fully measurable published stack observation, with error-bearing or skipped rows listed separately and not assigned a pass count. + + + + + + + Fully measurable observations + 0 + 1 + 2 + 3 + Not fully measurable - pass counts withheld + legacy-unresolved.gguf | quant not declared | ollama - 0 errors, 1 skipped; pass count not plotted + v2-unresolved:model | Q5_K_M | MLX LM - 1 errors, 0 skipped; pass count not plotted + + declared-fixture.gguf | Q4_K_M | llama.cpp: 1 passes + + + + Verified & Fixture | quant not declared | ollama: 1 passes + + + + legacy-unresolved.gguf | quant not declared | ollama: 0 errors, 1 skipped; pass count not plotted + + + + + v2-unresolved:model | Q5_K_M | MLX LM: 1 errors, 0 skipped; pass count not plotted + + + + +
    +
    +
    +
    +

    Calibration notes

    +

    Method and limitations

    +

    Each cell measures one full stack. The stack combines a model, quant, server, and server version. A cell does not describe the model alone.

    +

    A failed observation applies only to the tested combination. It does not mean the weights are bad. The same weights can pass on one server and fail on another. When the evidence proves that difference, the cell includes a cause annotation.

    +

    When the result includes a transcript path, a failing observation links to the full request and response transcript. Legacy schema v1 results do not record transcript paths. Read the case studies in docs/case-studies/ for controlled comparisons.

    +

    The servers use different decode methods. llama.cpp compiles the supplied tool definitions into a GBNF grammar. It uses that grammar to constrain decoding. Ollama and MLX LM generate unconstrained text. They parse the tool call after decoding. Cross-band differences therefore reflect the full stack. Compare adjacent models only when they use the same server. Each row states whether the run recorded its decode mode or the site read the mode from the cited preset mapping. An unmapped preset has an unknown mode.

    +

    The site includes 9 distinct scenarios. Each published cell represents one run. Hatching marks that the cell has no verdict.

    +

    The case studies draw a verdict only after at least five runs per arm. The current case studies cover 90 runs across 18 quantization arms, and 40 runs across 8 arms for the peg-native anomaly.

    +

    Excluded rows

    +

    The quantization conclusion excludes Meta-Llama-3.1-8B-Instruct on llama.cpp (Q8_0, Q4_K_M, Q3_K_M). For this model, llama.cpp returns HTTP 500 on 7-9 of 50 scenarios per run ("does not match the expected peg-native format"). These results are server errors. They are not model failures and cannot be compared across arms. Read the peg-native case study.

    + +
    + +
    +

    Build hash 3e9b9a5806493d81 - corpus revision site-contract fixture-v1 - data date

    + + + diff --git a/crates/willitcall/tests/migrate_cli.rs b/crates/willitcall/tests/migrate_cli.rs new file mode 100644 index 0000000..2edfc7b --- /dev/null +++ b/crates/willitcall/tests/migrate_cli.rs @@ -0,0 +1,277 @@ +use std::fs; +use std::path::PathBuf; +use std::process::{Command, Output}; + +use serde_json::{json, Value}; + +const REGISTRY: &[u8] = include_bytes!("../../../registry/models-v1.json"); +const V2_RESULT_FIXTURE: &[u8] = include_bytes!("fixtures/result-v2.json"); + +struct Fixture { + directory: tempfile::TempDir, + result_path: PathBuf, + manifest_path: PathBuf, + registry_path: PathBuf, +} + +impl Fixture { + fn new(source: &[u8], selector: &str, existing_model_id: Option<&str>) -> Self { + let directory = tempfile::tempdir().expect("temp directory"); + let results = directory.path().join("results"); + fs::create_dir(&results).expect("results directory"); + let result_path = results.join("misleading-model-name.json"); + let mut source: Value = serde_json::from_slice(source).expect("source result JSON"); + if let Some(model_id) = existing_model_id { + source["metadata"]["model_id"] = Value::String(model_id.to_owned()); + } + fs::write( + &result_path, + serde_json::to_vec_pretty(&source).expect("encode source result"), + ) + .expect("write source result"); + + let manifest_path = directory.path().join("manifest.json"); + fs::write( + &manifest_path, + serde_json::to_vec_pretty(&json!({ + "schema_version": 1, + "source_schema_version": 2, + "target_schema_version": 3, + "entries": [{ + "result_path": "results/misleading-model-name.json", + "registry_selector": selector, + "provenance_ref": "registry/evidence/huggingface-recovery.json#/records/6" + }] + })) + .expect("encode manifest"), + ) + .expect("write manifest"); + let registry_path = directory.path().join("registry.json"); + fs::write(®istry_path, REGISTRY).expect("write registry"); + Self { + directory, + result_path, + manifest_path, + registry_path, + } + } + + fn run(&self, extra: &[&str]) -> Output { + let mut command = Command::new(env!("CARGO_BIN_EXE_willitcall")); + command + .current_dir(self.directory.path()) + .arg("migrate-v3") + .arg("--manifest") + .arg(&self.manifest_path) + .arg("--registry") + .arg(&self.registry_path) + .args(extra); + command.output().expect("run migrate-v3") + } + + fn bytes(&self) -> Vec { + fs::read(&self.result_path).expect("read result") + } + + fn document(&self) -> Value { + serde_json::from_slice(&self.bytes()).expect("result JSON") + } +} + +fn assert_success(output: &Output) { + assert!( + output.status.success(), + "migrate-v3 failed\nstdout: {}\nstderr: {}", + String::from_utf8_lossy(&output.stdout), + String::from_utf8_lossy(&output.stderr) + ); +} + +fn assert_measured_facts_preserved(before: &Value, after: &Value) { + for field in [ + "run_id", + "timestamp", + "willitcall_version", + "endpoint", + "sampling", + "preflight_override", + "preflight_ignored_ports", + ] { + assert_eq!(before["metadata"][field], after["metadata"][field]); + } + for field in ["preset_name", "reported_version", "quirk_flags"] { + assert_eq!( + before["metadata"]["server"][field], + after["metadata"]["server"][field] + ); + } + assert_eq!(before["totals"], after["totals"]); + assert_eq!( + before["scenarios"] + .as_array() + .expect("source scenarios") + .len(), + after["scenarios"] + .as_array() + .expect("migrated scenarios") + .len() + ); + for (source, migrated) in before["scenarios"] + .as_array() + .expect("source scenarios") + .iter() + .zip(after["scenarios"].as_array().expect("migrated scenarios")) + { + for field in [ + "id", + "category", + "status", + "failure_reason", + "failure_class", + "cause", + "evidence_path", + "evidence_hash", + "retried", + ] { + assert_eq!(source[field], migrated[field], "changed {field}"); + } + assert!(migrated["failure"].is_null()); + } +} + +#[test] +fn migrates_v2_from_manifest_and_check_never_writes() { + let fixture = Fixture::new( + V2_RESULT_FIXTURE, + "bartowski/Meta-Llama-3.1-8B-Instruct-GGUF:Q3_K_M", + Some("must-not-drive-identity"), + ); + let before_bytes = fixture.bytes(); + let before: Value = serde_json::from_slice(&before_bytes).expect("source result JSON"); + + let check = fixture.run(&["--check"]); + assert_success(&check); + assert_eq!(before_bytes, fixture.bytes()); + assert!(String::from_utf8_lossy(&check.stdout).contains("no files written")); + + let migration = fixture.run(&[]); + assert_success(&migration); + let after = fixture.document(); + assert_eq!(after["schema_version"], 3); + assert_eq!( + after["metadata"]["model"]["canonical_id"], + "meta-llama/Meta-Llama-3.1-8B-Instruct" + ); + assert_eq!( + after["metadata"]["model"]["endpoint_id"], + "bartowski/Meta-Llama-3.1-8B-Instruct-GGUF:Q3_K_M" + ); + assert_ne!( + after["metadata"]["model"]["endpoint_id"], + before["metadata"]["model_id"] + ); + assert!(after["metadata"]["replication"].is_null()); + assert!(after["metadata"]["arm_fingerprint"].is_null()); + assert!(after["metadata"]["server"]["chat_template"].is_null()); + assert!(after["metadata"]["server"]["launch_config_sha256"].is_null()); + assert_measured_facts_preserved(&before, &after); +} + +#[test] +fn second_migration_is_a_byte_identical_no_op() { + let fixture = Fixture::new( + V2_RESULT_FIXTURE, + "bartowski/Meta-Llama-3.1-8B-Instruct-GGUF:Q3_K_M", + None, + ); + assert_success(&fixture.run(&[])); + let once = fixture.bytes(); + + let second = fixture.run(&[]); + assert_success(&second); + assert_eq!(once, fixture.bytes()); + assert!(String::from_utf8_lossy(&second.stdout).contains("no changes required")); +} + +#[test] +fn quirk_flag_carries_decode_mode_forward_and_repairs_the_old_v3_gap() { + let fixture = Fixture::new( + V2_RESULT_FIXTURE, + "mlx-community/Meta-Llama-3.1-8B-Instruct-4bit", + None, + ); + let mut source = fixture.document(); + source["metadata"]["server"]["quirk_flags"] = json!(["unconstrained_post_hoc_parse"]); + fs::write( + &fixture.result_path, + serde_json::to_vec_pretty(&source).expect("encode v2 quirk fixture"), + ) + .expect("write v2 quirk fixture"); + + assert_success(&fixture.run(&[])); + let migrated = fixture.document(); + assert_eq!( + migrated["metadata"]["server"]["decode_mode"], + "unconstrained_post_hoc" + ); + + let mut old_v3 = migrated; + old_v3["metadata"]["server"]["decode_mode"] = json!("unknown"); + fs::write( + &fixture.result_path, + serde_json::to_vec_pretty(&old_v3).expect("encode old v3 fixture"), + ) + .expect("write old v3 fixture"); + + assert_success(&fixture.run(&[])); + assert_eq!( + fixture.document()["metadata"]["server"]["decode_mode"], + "unconstrained_post_hoc" + ); + let repaired = fixture.bytes(); + let second = fixture.run(&[]); + assert_success(&second); + assert_eq!(repaired, fixture.bytes()); + assert!(String::from_utf8_lossy(&second.stdout).contains("no changes required")); +} + +#[test] +fn refuses_an_unlisted_result_before_writing() { + let fixture = Fixture::new( + V2_RESULT_FIXTURE, + "bartowski/Meta-Llama-3.1-8B-Instruct-GGUF:Q3_K_M", + None, + ); + let before = fixture.bytes(); + let unlisted = fixture + .result_path + .parent() + .expect("result parent") + .join("unlisted.json"); + fs::write(&unlisted, V2_RESULT_FIXTURE).expect("write unlisted result"); + + let output = fixture.run(&[]); + assert_eq!(output.status.code(), Some(2)); + assert!(String::from_utf8_lossy(&output.stderr).contains("refusing unlisted result file")); + assert_eq!(before, fixture.bytes()); +} + +#[test] +fn unresolved_registry_row_migrates_successfully() { + let fixture = Fixture::new(V2_RESULT_FIXTURE, "gemma3:4b", Some("looks-resolved:99b")); + + let output = fixture.run(&[]); + assert_success(&output); + let migrated = fixture.document(); + assert_eq!( + migrated["metadata"]["model"]["identity_status"], + "unresolved" + ); + assert!(migrated["metadata"]["model"]["canonical_id"].is_null()); + assert!(migrated["metadata"]["arm_fingerprint"].is_null()); + assert!(migrated["scenarios"] + .as_array() + .expect("scenarios") + .iter() + .all(|scenario| scenario["failure"].is_null())); +} diff --git a/crates/willitcall/tests/run_cli.rs b/crates/willitcall/tests/run_cli.rs index 2c1a625..0f5be6a 100644 --- a/crates/willitcall/tests/run_cli.rs +++ b/crates/willitcall/tests/run_cli.rs @@ -6,7 +6,7 @@ use std::process::Command; use serde_json::{json, Value}; use support::{MockServer, ScriptedResponse}; -use wic_core::result::{RunResult, Status}; +use wic_core::result::{parse_and_validate_measurement, IdentityStatus, Status}; fn completion(calls: Value, content: Value) -> String { json!({ @@ -123,6 +123,82 @@ fn write_result_fixture(path: &std::path::Path, schema_version: u32, mut scenari .expect("write result fixture"); } +fn write_v3_result_fixture(path: &std::path::Path, mut scenarios: Vec) { + for scenario in &mut scenarios { + scenario["failure"] = Value::Null; + } + let total = scenarios.len() as u32; + let result = json!({ + "schema_version": 3, + "metadata": { + "run_id": "019fd4e8-8e84-7a90-a686-f3caf2e147ef", + "timestamp": "2026-08-05T12:00:00Z", + "willitcall_version": "0.1.0", + "endpoint": "http://127.0.0.1:8080/v1", + "model": { + "display_name": "Unresolved model", + "family_id": null, + "canonical_id": null, + "parameter_count_b": null, + "endpoint_id": "fixture-model", + "identity_status": "unresolved", + "artifact": { + "source_kind": "other", + "source_id": null, + "revision": null, + "sha256": null, + "format": "unknown", + "quantization": null + } + }, + "corpus": { + "id": "wic-50", + "revision": "v1", + "sha256": "sha256:fixture", + "scenario_count": total, + "scoring_version": "v1" + }, + "server": { + "preset_name": "custom", + "reported_version": null, + "quirk_flags": [], + "decode_mode": "unknown", + "chat_template": null, + "launch_config_sha256": null + }, + "environment": { + "display_label": "Fixture workstation; Fixture OS", + "os_name": "Fixture OS", + "os_version": null, + "architecture": null, + "accelerator": null, + "memory_bytes": null + }, + "sampling": { + "temperature": 0.0, + "top_p": 1.0, + "seed": 42, + "max_tokens": 1024 + }, + "replication": null, + "arm_fingerprint": null + }, + "scenarios": scenarios, + "totals": { + "total": total, + "passed": 0, + "failed": total, + "errors": 0, + "skipped": 0 + } + }); + fs::write( + path, + serde_json::to_vec_pretty(&result).expect("encode v3 result fixture"), + ) + .expect("write v3 result fixture"); +} + fn write_transcript_fixture( result_path: &std::path::Path, evidence_path: &str, @@ -509,6 +585,180 @@ async fn rescore_leaves_v1_without_evidence_paths_alone() { assert_eq!(fs::read(result_path).expect("result bytes"), before); } +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn new_runs_emit_v3_and_legacy_edits_preserve_their_version() { + let server = MockServer::start_scripted( + "qwen2.5:7b-instruct", + vec![ScriptedResponse::Json(completion( + json!([]), + json!("ready"), + ))], + ) + .await; + let directory = tempfile::tempdir().expect("temp directory"); + let scenario_path = directory.path().join("scenarios"); + fs::create_dir(&scenario_path).expect("scenario directory"); + fs::write( + scenario_path.join("v3-run.toml"), + r#" +id = "v3-run" +category = "negative_trap" +description = "Produce no tool call." +rationale = "This fixture asserts v3 emission; tool_choice none forbids the offered tool." + +[[tools]] +name = "get_weather" +description = "Get weather." + +[tools.parameters] +type = "object" + +[tool_choice] +mode = "none" + +[[turns]] +[[turns.messages]] +role = "user" +content = "Reply ready." +"#, + ) + .expect("write scenario"); + let v3_path = directory.path().join("new-run.json"); + + let run = run_binary(vec![ + "run".to_owned(), + "--endpoint".to_owned(), + server.endpoint(), + "--model".to_owned(), + "qwen2.5:7b-instruct".to_owned(), + "--force".to_owned(), + "--scenarios".to_owned(), + scenario_path.display().to_string(), + "--out".to_owned(), + v3_path.display().to_string(), + ]) + .await; + + assert_eq!( + run.status.code(), + Some(0), + "{}", + String::from_utf8_lossy(&run.stderr) + ); + let v3_bytes = fs::read(&v3_path).expect("v3 result"); + let v3 = parse_and_validate_measurement(&v3_bytes).expect("valid v3 measurement"); + assert_eq!(v3.schema_version, 3); + assert_eq!(v3.metadata.model.endpoint_id, "qwen2.5:7b-instruct"); + assert_eq!(v3.metadata.model.identity_status, IdentityStatus::Declared); + assert_eq!( + v3.metadata + .corpus + .as_ref() + .expect("v3 corpus metadata") + .scoring_version, + "v2" + ); + assert!(v3.metadata.replication.is_none()); + assert!(v3.metadata.arm_fingerprint.is_none()); + + let validation = run_binary(vec!["validate".to_owned(), v3_path.display().to_string()]).await; + assert_eq!( + validation.status.code(), + Some(0), + "{}", + String::from_utf8_lossy(&validation.stderr) + ); + + let v2_path = directory.path().join("legacy-v2.json"); + write_result_fixture( + &v2_path, + 2, + vec![ + scenario_fixture( + "annotate-me", + Some("empty_response"), + Some("evidence/annotate.json"), + ), + scenario_fixture("rescore-me", None, Some("evidence/rescore.json")), + ], + ); + write_transcript_fixture( + &v2_path, + "evidence/rescore.json", + &completion(json!([]), Value::Null), + json!([]), + ); + let before: Value = + serde_json::from_slice(&fs::read(&v2_path).expect("v2 result")).expect("valid v2 JSON"); + + let annotation = run_binary(vec![ + "annotate".to_owned(), + "--result".to_owned(), + v2_path.display().to_string(), + "--scenario".to_owned(), + "annotate-me".to_owned(), + "--cause".to_owned(), + "unknown".to_owned(), + ]) + .await; + assert_eq!( + annotation.status.code(), + Some(0), + "{}", + String::from_utf8_lossy(&annotation.stderr) + ); + let annotated: Value = serde_json::from_slice(&fs::read(&v2_path).expect("annotated v2")) + .expect("valid annotated v2 JSON"); + assert_eq!(annotated["schema_version"], 2); + assert!(annotated["metadata"].get("model").is_none()); + + let rescored = run_binary(vec![ + "rescore".to_owned(), + "--result".to_owned(), + v2_path.display().to_string(), + ]) + .await; + assert_eq!( + rescored.status.code(), + Some(0), + "{}", + String::from_utf8_lossy(&rescored.stderr) + ); + let after: Value = serde_json::from_slice(&fs::read(&v2_path).expect("rescored v2")) + .expect("valid rescored v2 JSON"); + assert_eq!(after["schema_version"], 2); + assert!(after["metadata"].get("model").is_none()); + + let mut expected = before; + expected["scenarios"][0]["cause"] = json!({ + "kind": "unknown", + "reference": null, + "note": null + }); + expected["scenarios"][1]["failure_class"] = json!("empty_response"); + assert_eq!(after, expected, "only the requested v2 fields may change"); + + let site_output = directory.path().join("site"); + let site = run_binary(vec![ + "site".to_owned(), + "--results".to_owned(), + directory.path().display().to_string(), + "--out".to_owned(), + site_output.display().to_string(), + ]) + .await; + assert_eq!( + site.status.code(), + Some(0), + "{}", + String::from_utf8_lossy(&site.stderr) + ); + let appendix = + fs::read_to_string(site_output.join("appendix.html")).expect("generated appendix"); + assert!(appendix.contains("Schema
    v2")); + assert!(appendix.contains("Schema
    v3")); +} + fn write_m1a_scenarios(path: &std::path::Path) { fs::create_dir(path).expect("scenario directory"); for (name, contents) in [ @@ -596,7 +846,7 @@ async fn happy_run_passes_all_scenarios_and_writes_a_valid_result() { .await; let document = fs::read(&output_path).expect("result file"); - let result: RunResult = serde_json::from_slice(&document).expect("schema-valid result"); + let result = parse_and_validate_measurement(&document).expect("schema-valid result"); assert_eq!( output.status.code(), Some(0), @@ -604,8 +854,12 @@ async fn happy_run_passes_all_scenarios_and_writes_a_valid_result() { String::from_utf8_lossy(&output.stderr), result.scenarios ); - assert_eq!(result.schema_version, 2); - assert_eq!(result.metadata.declared_quant, None); + assert_eq!(result.schema_version, 3); + assert!(result.metadata.model.artifact.quantization.is_none()); + assert_eq!( + result.metadata.model.identity_status, + IdentityStatus::Unresolved + ); assert_eq!(result.metadata.sampling.seed, Some(42)); assert_eq!(result.metadata.sampling.temperature, Some(0.0)); assert_eq!(result.totals.passed, 5); @@ -711,10 +965,17 @@ content = "Reply ready." "{}", String::from_utf8_lossy(&output.stderr) ); - let result: RunResult = serde_json::from_slice(&output.stdout).expect("stdout is only JSON"); + let result = + parse_and_validate_measurement(&output.stdout).expect("stdout is schema-valid JSON"); assert_eq!(result.metadata.server.preset_name, "ollama"); assert_eq!( - result.metadata.declared_quant.as_deref(), + result + .metadata + .model + .artifact + .quantization + .as_ref() + .map(|quantization| quantization.label.as_str()), Some("Q4_K_M-imatrix") ); assert_eq!(result.metadata.sampling.seed, Some(8675309)); @@ -732,8 +993,9 @@ content = "Reply ready." .environment .as_ref() .expect("measurement environment"); - assert_eq!(environment.host_hardware_class, "Fixture workstation, 32GB"); - assert!(!environment.host_os.is_empty()); + assert!(environment + .display_label + .starts_with("Fixture workstation, 32GB; ")); assert_eq!( fs::read(&output_path).expect("result file"), output.stdout, @@ -772,8 +1034,8 @@ content = "Reply ready." "{}", String::from_utf8_lossy(&output.stderr) ); - let result: RunResult = - serde_json::from_slice(&output.stdout).expect("stdout is only JSON"); + let result = + parse_and_validate_measurement(&output.stdout).expect("stdout is schema-valid JSON"); assert_eq!(result.metadata.server.preset_name, recorded_name); assert_eq!(result.metadata.server.quirk_flags, expected_quirks); } @@ -803,7 +1065,7 @@ async fn validate_directory_ignores_archive() { } #[tokio::test(flavor = "multi_thread", worker_threads = 2)] -async fn validate_accepts_v1_and_v2_fixtures_and_rejects_v3() { +async fn validate_accepts_v1_v2_and_v3_fixtures() { let directory = tempfile::tempdir().expect("temp directory"); let v1_path = std::path::Path::new(env!("CARGO_MANIFEST_DIR")) .join("../../results/ollama-qwen2.5-7b-instruct.json"); @@ -843,7 +1105,10 @@ async fn validate_accepts_v1_and_v2_fixtures_and_rejects_v3() { ) .expect("write v2 result"); - for fixture in [&v1_path, &v2_path] { + let v3_path = directory.path().join("v3.json"); + write_v3_result_fixture(&v3_path, Vec::new()); + + for fixture in [&v1_path, &v2_path, &v3_path] { let output = run_binary(vec!["validate".to_owned(), fixture.display().to_string()]).await; assert_eq!( output.status.code(), @@ -853,19 +1118,51 @@ async fn validate_accepts_v1_and_v2_fixtures_and_rejects_v3() { ); } - let v3_path = directory.path().join("v3.json"); - let mut v3: Value = + let unsupported_path = directory.path().join("v4.json"); + let mut unsupported: Value = serde_json::from_slice(&fs::read(&v2_path).expect("v2 bytes")).expect("v2 JSON"); - v3["schema_version"] = json!(3); - fs::write(&v3_path, serde_json::to_vec_pretty(&v3).expect("encode v3")) - .expect("write v3 result"); - let output = run_binary(vec!["validate".to_owned(), v3_path.display().to_string()]).await; + unsupported["schema_version"] = json!(4); + fs::write( + &unsupported_path, + serde_json::to_vec_pretty(&unsupported).expect("encode unsupported result"), + ) + .expect("write unsupported result"); + let output = run_binary(vec![ + "validate".to_owned(), + unsupported_path.display().to_string(), + ]) + .await; + + assert_eq!(output.status.code(), Some(2)); + assert!(output.stdout.is_empty()); + let stderr = String::from_utf8_lossy(&output.stderr); + assert!( + stderr.contains("unsupported schema_version 4; expected 1, 2, or 3"), + "{stderr}" + ); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn validate_rejects_a_corrupt_document_with_a_nonzero_exit() { + let directory = tempfile::tempdir().expect("temp directory"); + let path = directory.path().join("corrupt.json"); + write_v3_result_fixture(&path, Vec::new()); + let mut corrupt: Value = + serde_json::from_slice(&fs::read(&path).expect("v3 bytes")).expect("v3 JSON"); + corrupt["totals"]["total"] = json!(1); + fs::write( + &path, + serde_json::to_vec_pretty(&corrupt).expect("encode corrupt result"), + ) + .expect("write corrupt result"); + + let output = run_binary(vec!["validate".to_owned(), path.display().to_string()]).await; assert_eq!(output.status.code(), Some(2)); assert!(output.stdout.is_empty()); let stderr = String::from_utf8_lossy(&output.stderr); assert!( - stderr.contains("unsupported schema_version 3; expected 1 or 2"), + stderr.contains("totals.total is 1 but scenarios contains 0 outcomes"), "{stderr}" ); } @@ -942,8 +1239,8 @@ city = "Boston" "{}", String::from_utf8_lossy(&output.stderr) ); - let result: RunResult = - serde_json::from_slice(&fs::read(output_path).expect("result file")).expect("valid result"); + let result = parse_and_validate_measurement(&fs::read(output_path).expect("result file")) + .expect("valid result"); assert_eq!(result.scenarios[0].status, Status::Fail); assert_eq!( result.scenarios[0].failure_reason.as_deref(), @@ -1014,8 +1311,8 @@ content = "Say hello." .await; assert_eq!(output.status.code(), Some(0)); - let result: RunResult = - serde_json::from_slice(&fs::read(output_path).expect("result file")).expect("valid result"); + let result = parse_and_validate_measurement(&fs::read(output_path).expect("result file")) + .expect("valid result"); assert!(result.scenarios[0].retried); let evidence_path = directory.path().join( result.scenarios[0] diff --git a/crates/willitcall/tests/site_cli.rs b/crates/willitcall/tests/site_cli.rs index 582651c..d40cd63 100644 --- a/crates/willitcall/tests/site_cli.rs +++ b/crates/willitcall/tests/site_cli.rs @@ -4,6 +4,9 @@ use std::process::Command; use serde_json::{json, Value}; +// Briefs 8 and 9 intentionally change the renderer and will update this golden. +const SITE_CONTRACT_GOLDEN: &[u8] = include_bytes!("golden/site-contract-index.html"); + fn scenario( id: &str, category: &str, @@ -118,7 +121,195 @@ fn run_site(results: &Path, output: &Path, repo_base: Option<&str>) -> std::proc } #[test] -fn site_generates_v1_and_v2_rows_ratios_links_and_badges() { +fn site_contract_golden_matches_post_migration_renderer() { + let directory = tempfile::tempdir().expect("temp directory"); + let results = + Path::new(env!("CARGO_MANIFEST_DIR")).join("tests/fixtures/site-contract-results"); + let output = directory.path().join("site"); + + let generated = run_site( + &results, + &output, + Some("https://example.invalid/willitcall"), + ); + assert_eq!( + generated.status.code(), + Some(0), + "{}", + String::from_utf8_lossy(&generated.stderr) + ); + + let index = fs::read(output.join("index.html")).expect("generated index"); + assert_eq!( + normalize_build_hash(&index), + normalize_build_hash(SITE_CONTRACT_GOLDEN) + ); +} + +/// The colophon states which build produced the page, so it changes whenever the +/// generator does. Comparing it here would make this golden fail on every code +/// change and train whoever hits it to regenerate without reading the diff, which +/// is the one thing a golden exists to prevent. +fn normalize_build_hash(html: &[u8]) -> String { + let text = String::from_utf8_lossy(html); + let mut out = String::with_capacity(text.len()); + let mut rest: &str = &text; + while let Some(start) = rest.find("Build hash ") { + let after = start + "Build hash ".len(); + let Some(end) = rest[after..].find("") else { + break; + }; + out.push_str(&rest[..after]); + out.push_str("NORMALIZED"); + rest = &rest[after + end..]; + } + out.push_str(rest); + out +} + +#[test] +fn analysis_views_render_the_published_observation_contract() { + let directory = tempfile::tempdir().expect("temp directory"); + let repo = Path::new(env!("CARGO_MANIFEST_DIR")).join("../.."); + let output = directory.path().join("site"); + + let generated = run_site(&repo.join("results"), &output, None); + assert_eq!( + generated.status.code(), + Some(0), + "{}", + String::from_utf8_lossy(&generated.stderr) + ); + + let index = fs::read_to_string(output.join("index.html")).expect("generated index"); + let outcomes = + fs::read_to_string(output.join("outcomes.html")).expect("generated outcomes page"); + let appendix = + fs::read_to_string(output.join("appendix.html")).expect("generated appendix page"); + let results_json = + fs::read_to_string(output.join("results.json")).expect("generated JSON data"); + let results_csv = fs::read_to_string(output.join("results.csv")).expect("generated CSV data"); + assert_eq!(index.matches("class=\"result-row\"").count(), 32); + assert_eq!(index.matches("(unverified artifact)").count(), 4); + assert_eq!(appendix.matches("class=\"stack-detail\"").count(), 32); + assert_eq!(index.matches("").expect("closed svg").0) + .collect::>(); + assert_eq!(svgs.len(), 4); + for svg in &svgs { + assert!(svg.contains("")); + assert!(svg.contains("<desc>")); + assert!(!svg.contains("http://")); + assert!(!svg.contains("https://")); + } + for generated in [&index, &outcomes, &appendix, &results_json, &results_csv] { + assert!(!generated.contains("/Users/")); + assert!(generated.is_ascii()); + } + assert!(!index.contains("<script src=\"http")); + assert!(!index.contains("View 50 scenarios and row metadata")); + assert!(!index.contains("class=\"detail-row\"")); + assert_eq!(index.matches("class=\"detail-link\"").count(), 32); + assert_eq!(outcomes.matches("class=\"mini-raster-panel\"").count(), 6); + assert_eq!(index.matches("class=\"replication-note\"").count(), 0); + assert_eq!(index.matches("n=1, no verdict").count(), 1); + assert_eq!(index.matches("class=\"model-heading\"").count(), 5); + assert_eq!(index.matches("result-group multi-row").count(), 5); + assert_eq!(index.matches("result-group single-row").count(), 13); + assert_eq!(index.matches("class=\"row-meta\"").count(), 32); + assert!(index.contains("id=\"model-search\" type=\"search\"")); + assert!(index.contains("data-decode-mode=\"grammar_constrained\"")); + assert!(index.contains("data-decode-mode=\"unconstrained_post_hoc\"")); + assert_eq!(index.matches("data-decode-source=\"recorded\"").count(), 6); + assert_eq!( + index + .matches("data-decode-source=\"preset_mapping\"") + .count(), + 26 + ); + assert_eq!(index.matches("data-decode-source=\"unknown\"").count(), 0); + assert_eq!(index.matches("class=\"decode-band").count(), 0); + assert_eq!(index.matches("class=\"decode-badge").count(), 32); + assert_eq!(index.matches("decode provenance:").count(), 32); + for (id, label) in [ + ("single_call", "Single call"), + ("tool_choice_modes", "Tool choice"), + ("negative_trap", "Correctly declines"), + ("multi_turn", "Multi-turn"), + ("parallel_calls", "Parallel calls"), + ("streaming", "Streaming"), + ] { + assert!(index.contains(&format!("title=\"{id}\">{label}</th>"))); + } + + assert!(svgs[0].contains("Figure 1. Scenario-status raster")); + assert!(svgs[0].contains("<title>single-weather: pass")); + assert!(!svgs[0].contains("Call one weather tool with a city argument.")); + assert!(!svgs[0].contains("Rationale:")); + assert!(!svgs[0].contains("no-verdict-overlay")); + + assert!(outcomes.contains( + "In these published observations, multi_turn passes 37/224 (17%). Of 32 rows, 22 pass none of the multi_turn scenarios." + )); + assert!(outcomes.contains( + "9 stacks share the repeated 7-pass signature. The stacks include the granite observations. Every row has the same 5 negative_trap passes and 2 tool_choice_modes passes." + )); + assert_eq!( + figure_pages + .matches("Data: JSON, JSON")); + let json: Value = serde_json::from_str(&results_json).expect("valid site results JSON"); + assert_eq!(json["row_count"], 32); + assert_eq!(json["stacks"].as_array().expect("stacks array").len(), 32); + assert_eq!(results_csv.lines().count(), 1_601); + + assert_eq!(svgs[3].matches("class=\"strip-observation\"").count(), 27); + assert_eq!( + svgs[3] + .matches("class=\"not-fully-measurable-item\"") + .count(), + 5 + ); + for gemma in ["gemma3:4b", "gemma3:12b"] { + let item = svgs[3] + .split(&format!("aria-label=\"{gemma}")) + .nth(1) + .expect("gemma row in not-fully-measurable panel") + .split_once("") + .expect("closed gemma panel item") + .0; + assert!(item.contains("50 errors, 0 skipped; pass count not plotted")); + assert!(!item.contains("data-pass-count")); + } +} + +#[test] +fn site_generates_v1_and_v2_rows_ratios_links_and_plain_annotations() { let directory = tempfile::tempdir().expect("temp directory"); let results = directory.path().join("results"); let output = directory.path().join("site"); @@ -221,56 +412,186 @@ fn site_generates_v1_and_v2_rows_ratios_links_and_badges() { ); let index = fs::read_to_string(output.join("index.html")).expect("generated index"); + let appendix = + fs::read_to_string(output.join("appendix.html")).expect("generated appendix page"); let submit = fs::read_to_string(output.join("submit.html")).expect("generated submit page"); assert!(output.join("style.css").is_file()); assert!(output.join("site.js").is_file()); assert_eq!(index.matches("class=\"result-row\"").count(), 3); + assert!(index.contains("id=\"model-search\" type=\"search\"")); assert!(index.contains("data-server=\"ollama\"")); assert!(index.contains("data-server=\"llamacpp\"")); - assert!(index.contains("")); assert!(index.contains("data-server=\"mlx_lm\"")); - assert!(index.contains("server: MLX LM")); - assert!(index.contains("blob-model")); - assert!(index.contains("quant: Q4_K_M")); - assert!(index.contains("server: llama.cpp")); - assert!(index.contains("/models/blobs/sha256-deadbeef")); + assert!(index.contains("data-decode-mode=\"grammar_constrained\"")); + assert!(index.contains("data-decode-mode=\"unconstrained_post_hoc\"")); + assert!(index.contains(" / MLX LM")); + assert!(appendix.contains("blob-model")); + assert!(index.contains(">Q4_K_M / llama.cpp")); + assert!(appendix.contains("sha256-deadbeef")); + assert!(!appendix.contains("/models/blobs/sha256-deadbeef")); assert!(index.contains(">1/2<")); assert!(index.contains(">0/1<")); assert!(index.contains(">1/2<")); assert!(index.contains( "https://github.com/devYRPauli/willitcall/blob/main/results/evidence/fixture/parallel-bad.json" )); - assert!(index.contains( + assert!(appendix.contains( "https://github.com/devYRPauli/willitcall/blob/main/docs/case-studies/server-defect.md" )); - assert!(index.contains("server defect")); - assert!(index.contains("empty response")); - assert!(index.contains("unparsed tool call")); - assert!(index.contains("A cell measures the whole stack")); + assert!(appendix.contains("server defect")); + assert!(appendix.contains("empty response")); + assert!(appendix.contains("unparsed tool call")); + assert!(!appendix.contains("class=\"badge")); + assert!(index.contains("Each cell measures one full stack")); assert!(index.contains("GBNF grammar")); - assert!(index.contains("Each published cell is one run.")); - assert!(index.contains("replicated across at least five runs per arm")); + assert!(index.contains("Each published cell represents one run.")); + assert!( + index.contains("The case studies draw a verdict only after at least five runs per arm.") + ); assert!(index.contains("90 runs across 18 quantization arms")); assert!(index.contains("40 runs across 8 arms for the peg-native anomaly")); assert!(index.contains("Meta-Llama-3.1-8B-Instruct")); assert!(index.contains( "https://github.com/devYRPauli/willitcall/blob/main/docs/case-studies/2026-07-21-llamacpp-500s-on-llama-3.1-tool-calls.md" )); - assert!(index.contains("Host hardware")); - assert!(index.contains("Apple M4 Max, 64GB")); - assert!(index.contains("Host OS")); - assert!(index.contains("macOS 15.5")); + assert!(appendix.contains("Host hardware")); + assert!(appendix.contains("Apple M4 Max, 64GB")); + assert!(appendix.contains("Host OS")); + assert!(appendix.contains("macOS 15.5")); assert!(index.contains("docs/case-studies/")); + assert!(!index.contains("class=\"detail-row\"")); + assert!(!index.contains("View 50 scenarios and row metadata")); + assert_eq!(index.matches("class=\"detail-link\"").count(), 3); + assert_eq!(appendix.matches("class=\"stack-detail\"").count(), 3); + assert!(output.join("results.json").is_file()); + assert!(output.join("results.csv").is_file()); assert!(!index.contains("