From 77b59978a5084fa509c90269950b07eea8158eaf Mon Sep 17 00:00:00 2001 From: Clement Pakkam Isaac Date: Wed, 30 Sep 2026 14:20:57 -0700 Subject: [PATCH 1/3] feat(evals): add offline TypeSafe replay harness Signed-off-by: Clement Pakkam Isaac --- benchmark/README.md | 3 + benchmark/typesafe/README.md | 48 ++++ .../typesafe/fixtures/synthetic-routing.json | 70 +++++ benchmark/typesafe_replay.py | 269 ++++++++++++++++++ tests/test_typesafe_replay.py | 113 ++++++++ 5 files changed, 503 insertions(+) create mode 100644 benchmark/typesafe/README.md create mode 100644 benchmark/typesafe/fixtures/synthetic-routing.json create mode 100644 benchmark/typesafe_replay.py create mode 100644 tests/test_typesafe_replay.py diff --git a/benchmark/README.md b/benchmark/README.md index 2b8f9e696..f29b0db82 100644 --- a/benchmark/README.md +++ b/benchmark/README.md @@ -16,6 +16,9 @@ Harbor directly at the upstream provider. For a small automated MMLU-Redux example using NeMo Gym instead of Harbor, see [Evaluate Switchyard routing with NeMo Gym](nemo_gym/README.md). +To replay recorded TypeSafe/Jev routing evidence without credentials or provider calls, see +[Offline TypeSafe/Jev Replay](typesafe/README.md). + ## Prerequisites From the repo root: diff --git a/benchmark/typesafe/README.md b/benchmark/typesafe/README.md new file mode 100644 index 000000000..50ce73c08 --- /dev/null +++ b/benchmark/typesafe/README.md @@ -0,0 +1,48 @@ + + + +# Offline TypeSafe/Jev Replay + +Use the replay tool to evaluate recorded TypeSafe routing evidence without credentials, network +access, or new model calls: + +```bash +uv run --no-sync python benchmark/typesafe_replay.py \ + benchmark/typesafe/fixtures/synthetic-routing.json +``` + +Pass `--output report.json` to save the stable, machine-readable report. Replaying the same fixture +always produces the same bytes. + +## Fixture format + +Every fixture is one JSON document with `schema_version: 1`. It records: + +- a suite name and pinned Jev model identifier; +- two or more candidate labels and descriptions; +- the confidence threshold and fallback target; +- one or more unique candidate orders per case, with a complete probability distribution for each; +- measured quality in `[0, 1]` and nonnegative cost for every candidate outcome. + +The replay averages the recorded distributions, normalizes the average, and chooses the largest +probability. Configured candidate order breaks an exact tie. Confidence is the selected +probability's improvement over a uniform distribution, scaled to `[0, 1]`. A result below +`base_threshold` selects `default_target`. + +`best_fixed_target` is the candidate with the highest total quality when used for every case. Lower +cost and then configured candidate order break ties. The report compares the routed totals with +that fixed baseline. Positive `quality_regret_vs_best_fixed` means routing lost quality; a negative +value means it beat every fixed target. `cost_delta_vs_best_fixed` uses the same sign convention. +`order_sensitive_case_count` counts cases whose per-order top candidate changes, while +`maximum_probability_movement` reports the largest probability shift seen across orders. + +## Sanitization boundary + +The schema deliberately excludes request text, credentials, response bodies, and provider error +details. Use opaque case IDs. Candidate descriptions and model identifiers can still contain +deployment information, so replace them with non-sensitive equivalents before committing a +fixture. The checked-in fixture is entirely synthetic and does not claim production routing +quality. + +This is an offline evaluator, not a provider collector. Collection of live evidence must remain an +explicit, separately reviewed workflow. diff --git a/benchmark/typesafe/fixtures/synthetic-routing.json b/benchmark/typesafe/fixtures/synthetic-routing.json new file mode 100644 index 000000000..15c78a242 --- /dev/null +++ b/benchmark/typesafe/fixtures/synthetic-routing.json @@ -0,0 +1,70 @@ +{ + "schema_version": 1, + "suite": "synthetic-two-target-routing", + "model": "jev-pinned-example", + "candidates": [ + { + "label": "capable", + "description": "Best for complex, multi-step work." + }, + { + "label": "efficient", + "description": "Best for short, well-specified work." + } + ], + "base_threshold": 0.25, + "default_target": "efficient", + "cases": [ + { + "id": "simple-request", + "orders": [ + { + "candidate_order": ["capable", "efficient"], + "probabilities": {"capable": 0.1, "efficient": 0.9} + }, + { + "candidate_order": ["efficient", "capable"], + "probabilities": {"capable": 0.2, "efficient": 0.8} + } + ], + "outcomes": { + "capable": {"quality": 1.0, "cost": 10.0}, + "efficient": {"quality": 1.0, "cost": 2.0} + } + }, + { + "id": "complex-request", + "orders": [ + { + "candidate_order": ["capable", "efficient"], + "probabilities": {"capable": 0.8, "efficient": 0.2} + }, + { + "candidate_order": ["efficient", "capable"], + "probabilities": {"capable": 0.6, "efficient": 0.4} + } + ], + "outcomes": { + "capable": {"quality": 1.0, "cost": 10.0}, + "efficient": {"quality": 0.0, "cost": 2.0} + } + }, + { + "id": "order-sensitive-request", + "orders": [ + { + "candidate_order": ["capable", "efficient"], + "probabilities": {"capable": 0.7, "efficient": 0.3} + }, + { + "candidate_order": ["efficient", "capable"], + "probabilities": {"capable": 0.3, "efficient": 0.7} + } + ], + "outcomes": { + "capable": {"quality": 1.0, "cost": 10.0}, + "efficient": {"quality": 0.0, "cost": 2.0} + } + } + ] +} diff --git a/benchmark/typesafe_replay.py b/benchmark/typesafe_replay.py new file mode 100644 index 000000000..cf1188889 --- /dev/null +++ b/benchmark/typesafe_replay.py @@ -0,0 +1,269 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Replay recorded TypeSafe/Jev routing evidence without provider calls.""" + +from __future__ import annotations + +import argparse +import json +import math +import sys +from pathlib import Path +from typing import Any, cast + +SCHEMA_VERSION = 1 +PROBABILITY_SUM_TOLERANCE = 0.02 + + +class FixtureError(ValueError): + """Report invalid or unsupported replay evidence.""" + + +def require(condition: bool, message: str) -> None: + """Reject incomplete evidence before calculating metrics.""" + if not condition: + raise FixtureError(message) + + +def object_value(value: Any, name: str) -> dict[str, Any]: + """Read one JSON object with a useful field diagnostic.""" + require(isinstance(value, dict), f"{name} must be an object") + return cast(dict[str, Any], value) + + +def string_value(value: Any, name: str) -> str: + """Read one non-empty string field.""" + require(isinstance(value, str) and bool(value.strip()), f"{name} must be a non-empty string") + return value + + +def number_value(value: Any, name: str, *, maximum: float | None = None) -> float: + """Read one finite, nonnegative numeric field.""" + require( + type(value) in (int, float) and math.isfinite(value) and value >= 0, + f"{name} must be a finite, nonnegative number", + ) + number = float(value) + if maximum is not None: + require(number <= maximum, f"{name} must be at most {maximum}") + return number + + +def string_list(value: Any, name: str) -> list[str]: + """Read one non-empty list of unique strings.""" + require(isinstance(value, list) and bool(value), f"{name} must be a non-empty list") + values = [string_value(item, f"{name}[]") for item in value] + require(len(values) == len(set(values)), f"{name} must not contain duplicates") + return values + + +def probabilities(value: Any, labels: list[str], name: str) -> dict[str, float]: + """Validate one complete probability distribution.""" + document = object_value(value, name) + require(set(document) == set(labels), f"{name} must contain every candidate exactly once") + parsed = { + label: number_value(document[label], f"{name}.{label}", maximum=1.0) for label in labels + } + total = sum(parsed.values()) + require( + abs(total - 1.0) <= PROBABILITY_SUM_TOLERANCE, + f"{name} must sum to one within {PROBABILITY_SUM_TOLERANCE}", + ) + return parsed + + +def selected_label(labels: list[str], values: dict[str, float]) -> str: + """Select the highest probability, breaking ties by configured candidate order.""" + return max(labels, key=values.__getitem__) + + +def rounded(value: float) -> float: + """Keep reports readable and stable across harmless floating-point noise.""" + return round(value, 12) + + +def replay(document: dict[str, Any]) -> dict[str, Any]: + """Validate one fixture document and return its deterministic replay report.""" + require( + document.get("schema_version") == SCHEMA_VERSION, + f"schema_version must be {SCHEMA_VERSION}", + ) + suite = string_value(document.get("suite"), "suite") + model = string_value(document.get("model"), "model") + candidate_rows = document.get("candidates") + require( + isinstance(candidate_rows, list) and len(candidate_rows) >= 2, + "candidates needs at least two entries", + ) + + candidates = [object_value(row, "candidates[]") for row in candidate_rows] + labels = [string_value(row.get("label"), "candidates[].label") for row in candidates] + require(len(labels) == len(set(labels)), "candidate labels must be unique") + for row in candidates: + string_value(row.get("description"), "candidates[].description") + + default_target = string_value(document.get("default_target"), "default_target") + require(default_target in labels, "default_target must name a candidate") + threshold = number_value(document.get("base_threshold"), "base_threshold", maximum=1.0) + case_rows = document.get("cases") + require(isinstance(case_rows, list) and bool(case_rows), "cases must be a non-empty list") + + fixed = {label: {"quality": 0.0, "cost": 0.0} for label in labels} + selected_counts = dict.fromkeys(labels, 0) + routed_quality = 0.0 + routed_cost = 0.0 + fallback_count = 0 + order_sensitive_count = 0 + maximum_movement = 0.0 + case_ids: set[str] = set() + case_reports: list[dict[str, Any]] = [] + + for case_index, case_value in enumerate(case_rows): + case = object_value(case_value, f"cases[{case_index}]") + case_id = string_value(case.get("id"), f"cases[{case_index}].id") + require(case_id not in case_ids, f"duplicate case id {case_id!r}") + case_ids.add(case_id) + + order_rows = case.get("orders") + require( + isinstance(order_rows, list) and bool(order_rows), + f"case {case_id!r} needs at least one candidate order", + ) + seen_orders: set[tuple[str, ...]] = set() + distributions: list[dict[str, float]] = [] + order_winners: list[str] = [] + for order_index, order_value in enumerate(order_rows): + order = object_value(order_value, f"case {case_id!r} order {order_index}") + order_labels = string_list( + order.get("candidate_order"), + f"case {case_id!r} order {order_index}.candidate_order", + ) + require( + set(order_labels) == set(labels) and len(order_labels) == len(labels), + f"case {case_id!r} order {order_index} must contain every candidate", + ) + order_key = tuple(order_labels) + require(order_key not in seen_orders, f"case {case_id!r} repeats a candidate order") + seen_orders.add(order_key) + distribution = probabilities( + order.get("probabilities"), + labels, + f"case {case_id!r} order {order_index}.probabilities", + ) + distributions.append(distribution) + order_winners.append(selected_label(order_labels, distribution)) + + averaged = { + label: sum(distribution[label] for distribution in distributions) / len(distributions) + for label in labels + } + average_total = sum(averaged.values()) + averaged = {label: value / average_total for label, value in averaged.items()} + classifier_target = selected_label(labels, averaged) + uniform = 1.0 / len(labels) + unbounded_confidence = (averaged[classifier_target] - uniform) / (1.0 - uniform) + confidence = max(0.0, min(1.0, unbounded_confidence)) + + fallback = confidence < threshold + final_target = default_target if fallback else classifier_target + fallback_count += int(fallback) + selected_counts[final_target] += 1 + + order_sensitive = len(set(order_winners)) > 1 + order_sensitive_count += int(order_sensitive) + movement = max( + max(distribution[label] for distribution in distributions) + - min(distribution[label] for distribution in distributions) + for label in labels + ) + maximum_movement = max(maximum_movement, movement) + + outcomes = object_value(case.get("outcomes"), f"case {case_id!r}.outcomes") + require( + set(outcomes) == set(labels), + f"case {case_id!r}.outcomes must contain every candidate exactly once", + ) + parsed_outcomes: dict[str, dict[str, float]] = {} + for label in labels: + outcome = object_value(outcomes[label], f"case {case_id!r}.outcomes.{label}") + quality = number_value( + outcome.get("quality"), f"case {case_id!r}.outcomes.{label}.quality", maximum=1.0 + ) + cost = number_value(outcome.get("cost"), f"case {case_id!r}.outcomes.{label}.cost") + parsed_outcomes[label] = {"quality": quality, "cost": cost} + fixed[label]["quality"] += quality + fixed[label]["cost"] += cost + + routed_quality += parsed_outcomes[final_target]["quality"] + routed_cost += parsed_outcomes[final_target]["cost"] + case_reports.append( + { + "id": case_id, + "average_probabilities": {label: rounded(averaged[label]) for label in labels}, + "classifier_target": classifier_target, + "confidence": rounded(confidence), + "fallback": fallback, + "selected_target": final_target, + "order_sensitive": order_sensitive, + "maximum_probability_movement": rounded(movement), + } + ) + + best_fixed = min( + labels, + key=lambda label: (-fixed[label]["quality"], fixed[label]["cost"], labels.index(label)), + ) + fixed_report = { + label: { + "quality": rounded(fixed[label]["quality"]), + "cost": rounded(fixed[label]["cost"]), + } + for label in labels + } + return { + "schema_version": SCHEMA_VERSION, + "suite": suite, + "model": model, + "base_threshold": threshold, + "default_target": default_target, + "case_count": len(case_reports), + "selected_counts": selected_counts, + "fallback_count": fallback_count, + "order_sensitive_case_count": order_sensitive_count, + "maximum_probability_movement": rounded(maximum_movement), + "routed": {"quality": rounded(routed_quality), "cost": rounded(routed_cost)}, + "fixed_targets": fixed_report, + "best_fixed_target": best_fixed, + "quality_regret_vs_best_fixed": rounded(fixed[best_fixed]["quality"] - routed_quality), + "cost_delta_vs_best_fixed": rounded(routed_cost - fixed[best_fixed]["cost"]), + "cases": case_reports, + } + + +def read_fixture(path: Path) -> dict[str, Any]: + """Read one replay fixture from disk.""" + return object_value(json.loads(path.read_text(encoding="utf-8")), str(path)) + + +def main(argv: list[str] | None = None) -> int: + """Replay one fixture and write stable JSON to stdout or a file.""" + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("fixture", type=Path, help="Versioned TypeSafe replay fixture") + parser.add_argument("--output", type=Path, help="Write the JSON report to this path") + args = parser.parse_args(argv) + try: + report = replay(read_fixture(args.fixture)) + encoded = json.dumps(report, indent=2, sort_keys=True, allow_nan=False) + "\n" + if args.output is None: + sys.stdout.write(encoded) + else: + args.output.write_text(encoded, encoding="utf-8") + except (FixtureError, OSError, UnicodeError, json.JSONDecodeError) as error: + print(f"Cannot replay fixture: {error}", file=sys.stderr) + return 1 + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tests/test_typesafe_replay.py b/tests/test_typesafe_replay.py new file mode 100644 index 000000000..9df8556a3 --- /dev/null +++ b/tests/test_typesafe_replay.py @@ -0,0 +1,113 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +from __future__ import annotations + +import importlib.util +import json +import subprocess +import sys +from copy import deepcopy +from pathlib import Path +from types import ModuleType + +import pytest + +REPO = Path(__file__).resolve().parents[1] +SCRIPT = REPO / "benchmark/typesafe_replay.py" +FIXTURE = REPO / "benchmark/typesafe/fixtures/synthetic-routing.json" + + +@pytest.fixture +def replay_module() -> ModuleType: + """Load the replay tool without installing Switchyard or provider dependencies.""" + spec = importlib.util.spec_from_file_location("switchyard_typesafe_replay", SCRIPT) + assert spec is not None and spec.loader is not None + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +@pytest.fixture +def fixture_document() -> dict[str, object]: + """Return an isolated copy of the checked-in synthetic fixture.""" + return json.loads(FIXTURE.read_text(encoding="utf-8")) + + +def test_synthetic_fixture_reports_routing_quality_cost_and_order_sensitivity( + replay_module: ModuleType, fixture_document: dict[str, object] +) -> None: + """Replay aggregation, fallback, and fixed-baseline comparisons together.""" + report = replay_module.replay(fixture_document) + + assert report["case_count"] == 3 + assert report["selected_counts"] == {"capable": 1, "efficient": 2} + assert report["fallback_count"] == 1 + assert report["order_sensitive_case_count"] == 1 + assert report["maximum_probability_movement"] == 0.4 + assert report["routed"] == {"quality": 2.0, "cost": 14.0} + assert report["fixed_targets"] == { + "capable": {"quality": 3.0, "cost": 30.0}, + "efficient": {"quality": 1.0, "cost": 6.0}, + } + assert report["best_fixed_target"] == "capable" + assert report["quality_regret_vs_best_fixed"] == 1.0 + assert report["cost_delta_vs_best_fixed"] == -16.0 + assert report["cases"][2] == { + "id": "order-sensitive-request", + "average_probabilities": {"capable": 0.5, "efficient": 0.5}, + "classifier_target": "capable", + "confidence": 0.0, + "fallback": True, + "selected_target": "efficient", + "order_sensitive": True, + "maximum_probability_movement": 0.4, + } + + +def test_cli_output_is_byte_stable_and_offline() -> None: + """Run twice with an empty environment and require byte-identical JSON.""" + command = [sys.executable, str(SCRIPT), str(FIXTURE)] + first = subprocess.run(command, check=False, capture_output=True, env={}) + second = subprocess.run(command, check=False, capture_output=True, env={}) + + assert first.returncode == second.returncode == 0 + assert first.stderr == second.stderr == b"" + assert first.stdout == second.stdout + assert json.loads(first.stdout)["suite"] == "synthetic-two-target-routing" + + +def test_unsupported_schema_version_is_rejected( + replay_module: ModuleType, fixture_document: dict[str, object] +) -> None: + """Reject fixture semantics the evaluator does not understand.""" + fixture_document["schema_version"] = 2 + + with pytest.raises(replay_module.FixtureError, match="schema_version must be 1"): + replay_module.replay(fixture_document) + + +def test_invalid_probability_distribution_is_rejected( + replay_module: ModuleType, fixture_document: dict[str, object] +) -> None: + """Do not calculate metrics from incomplete provider evidence.""" + document = deepcopy(fixture_document) + document["cases"][0]["orders"][0]["probabilities"] = { + "capable": 0.8, + "efficient": 0.8, + } + + with pytest.raises(replay_module.FixtureError, match="must sum to one"): + replay_module.replay(document) + + +def test_duplicate_candidate_order_is_rejected( + replay_module: ModuleType, fixture_document: dict[str, object] +) -> None: + """Prevent one repeated ordering from receiving accidental extra weight.""" + document = deepcopy(fixture_document) + first_order = document["cases"][0]["orders"][0] + document["cases"][0]["orders"][1] = deepcopy(first_order) + + with pytest.raises(replay_module.FixtureError, match="repeats a candidate order"): + replay_module.replay(document) From d858a3e69f33a85bbb27909e62832604ed93e571 Mon Sep 17 00:00:00 2001 From: Clement Pakkam Isaac Date: Wed, 30 Sep 2026 14:46:13 -0700 Subject: [PATCH 2/3] fix(evals): support cost-free replay fixtures Signed-off-by: Clement Pakkam Isaac --- benchmark/typesafe/README.md | 22 +++++++----- benchmark/typesafe_replay.py | 63 ++++++++++++++++++++++++----------- tests/test_typesafe_replay.py | 43 ++++++++++++++++++++++++ 3 files changed, 100 insertions(+), 28 deletions(-) diff --git a/benchmark/typesafe/README.md b/benchmark/typesafe/README.md index 50ce73c08..4c533cd69 100644 --- a/benchmark/typesafe/README.md +++ b/benchmark/typesafe/README.md @@ -16,25 +16,31 @@ always produces the same bytes. ## Fixture format -Every fixture is one JSON document with `schema_version: 1`. It records: +Every fixture is one JSON document with numeric `schema_version: 1`; JSON Booleans are not valid +versions. It records: - a suite name and pinned Jev model identifier; - two or more candidate labels and descriptions; - the confidence threshold and fallback target; - one or more unique candidate orders per case, with a complete probability distribution for each; -- measured quality in `[0, 1]` and nonnegative cost for every candidate outcome. +- measured quality in `[0, 1]` for every candidate outcome; +- optional nonnegative cost for each candidate outcome. The replay averages the recorded distributions, normalizes the average, and chooses the largest probability. Configured candidate order breaks an exact tie. Confidence is the selected probability's improvement over a uniform distribution, scaled to `[0, 1]`. A result below `base_threshold` selects `default_target`. -`best_fixed_target` is the candidate with the highest total quality when used for every case. Lower -cost and then configured candidate order break ties. The report compares the routed totals with -that fixed baseline. Positive `quality_regret_vs_best_fixed` means routing lost quality; a negative -value means it beat every fixed target. `cost_delta_vs_best_fixed` uses the same sign convention. -`order_sensitive_case_count` counts cases whose per-order top candidate changes, while -`maximum_probability_movement` reports the largest probability shift seen across orders. +`best_fixed_target` is the candidate with the highest total quality when used for every case. When +every quality-tied candidate has complete costs, lower cost breaks the tie. If any tied candidate +has incomplete costs, configured candidate order breaks the tie instead of treating an unknown +cost as zero. The report compares the routed totals with that fixed baseline. Positive +`quality_regret_vs_best_fixed` means routing lost quality; a negative value means it beat every +fixed target. A cost total is `null` if any outcome contributing to it has no cost, and +`cost_delta_vs_best_fixed` is `null` unless both compared totals are available. Otherwise the cost +delta uses the same sign convention. `order_sensitive_case_count` counts cases whose per-order top +candidate changes, while `maximum_probability_movement` reports the largest probability shift +seen across orders. ## Sanitization boundary diff --git a/benchmark/typesafe_replay.py b/benchmark/typesafe_replay.py index cf1188889..4889e7fbe 100644 --- a/benchmark/typesafe_replay.py +++ b/benchmark/typesafe_replay.py @@ -85,8 +85,9 @@ def rounded(value: float) -> float: def replay(document: dict[str, Any]) -> dict[str, Any]: """Validate one fixture document and return its deterministic replay report.""" + schema_version = document.get("schema_version") require( - document.get("schema_version") == SCHEMA_VERSION, + type(schema_version) in (int, float) and schema_version == SCHEMA_VERSION, f"schema_version must be {SCHEMA_VERSION}", ) suite = string_value(document.get("suite"), "suite") @@ -109,10 +110,11 @@ def replay(document: dict[str, Any]) -> dict[str, Any]: case_rows = document.get("cases") require(isinstance(case_rows, list) and bool(case_rows), "cases must be a non-empty list") - fixed = {label: {"quality": 0.0, "cost": 0.0} for label in labels} + fixed_quality = dict.fromkeys(labels, 0.0) + fixed_cost: dict[str, float | None] = dict.fromkeys(labels, 0.0) selected_counts = dict.fromkeys(labels, 0) routed_quality = 0.0 - routed_cost = 0.0 + routed_cost: float | None = 0.0 fallback_count = 0 order_sensitive_count = 0 maximum_movement = 0.0 @@ -184,19 +186,29 @@ def replay(document: dict[str, Any]) -> dict[str, Any]: set(outcomes) == set(labels), f"case {case_id!r}.outcomes must contain every candidate exactly once", ) - parsed_outcomes: dict[str, dict[str, float]] = {} + parsed_quality: dict[str, float] = {} + parsed_cost: dict[str, float | None] = {} for label in labels: outcome = object_value(outcomes[label], f"case {case_id!r}.outcomes.{label}") quality = number_value( outcome.get("quality"), f"case {case_id!r}.outcomes.{label}.quality", maximum=1.0 ) - cost = number_value(outcome.get("cost"), f"case {case_id!r}.outcomes.{label}.cost") - parsed_outcomes[label] = {"quality": quality, "cost": cost} - fixed[label]["quality"] += quality - fixed[label]["cost"] += cost - - routed_quality += parsed_outcomes[final_target]["quality"] - routed_cost += parsed_outcomes[final_target]["cost"] + cost_value = outcome.get("cost") + cost = ( + None + if cost_value is None + else number_value(cost_value, f"case {case_id!r}.outcomes.{label}.cost") + ) + parsed_quality[label] = quality + parsed_cost[label] = cost + fixed_quality[label] += quality + if fixed_cost[label] is not None: + fixed_cost[label] = None if cost is None else fixed_cost[label] + cost + + routed_quality += parsed_quality[final_target] + selected_cost = parsed_cost[final_target] + if routed_cost is not None: + routed_cost = None if selected_cost is None else routed_cost + selected_cost case_reports.append( { "id": case_id, @@ -210,17 +222,25 @@ def replay(document: dict[str, Any]) -> dict[str, Any]: } ) - best_fixed = min( - labels, - key=lambda label: (-fixed[label]["quality"], fixed[label]["cost"], labels.index(label)), - ) + best_quality = max(fixed_quality.values()) + quality_ties = [label for label in labels if fixed_quality[label] == best_quality] + if all(fixed_cost[label] is not None for label in quality_ties): + best_fixed = min(quality_ties, key=lambda label: fixed_cost[label]) + else: + best_fixed = quality_ties[0] fixed_report = { label: { - "quality": rounded(fixed[label]["quality"]), - "cost": rounded(fixed[label]["cost"]), + "quality": rounded(fixed_quality[label]), + "cost": None if fixed_cost[label] is None else rounded(fixed_cost[label]), } for label in labels } + best_fixed_cost = fixed_cost[best_fixed] + cost_delta = ( + None + if routed_cost is None or best_fixed_cost is None + else rounded(routed_cost - best_fixed_cost) + ) return { "schema_version": SCHEMA_VERSION, "suite": suite, @@ -232,11 +252,14 @@ def replay(document: dict[str, Any]) -> dict[str, Any]: "fallback_count": fallback_count, "order_sensitive_case_count": order_sensitive_count, "maximum_probability_movement": rounded(maximum_movement), - "routed": {"quality": rounded(routed_quality), "cost": rounded(routed_cost)}, + "routed": { + "quality": rounded(routed_quality), + "cost": None if routed_cost is None else rounded(routed_cost), + }, "fixed_targets": fixed_report, "best_fixed_target": best_fixed, - "quality_regret_vs_best_fixed": rounded(fixed[best_fixed]["quality"] - routed_quality), - "cost_delta_vs_best_fixed": rounded(routed_cost - fixed[best_fixed]["cost"]), + "quality_regret_vs_best_fixed": rounded(fixed_quality[best_fixed] - routed_quality), + "cost_delta_vs_best_fixed": cost_delta, "cases": case_reports, } diff --git a/tests/test_typesafe_replay.py b/tests/test_typesafe_replay.py index 9df8556a3..e3f51a07c 100644 --- a/tests/test_typesafe_replay.py +++ b/tests/test_typesafe_replay.py @@ -87,6 +87,49 @@ def test_unsupported_schema_version_is_rejected( replay_module.replay(fixture_document) +def test_boolean_schema_version_is_rejected_but_integral_float_is_accepted( + replay_module: ModuleType, fixture_document: dict[str, object] +) -> None: + """Keep JSON Booleans out of the numeric schema-version contract.""" + fixture_document["schema_version"] = True + with pytest.raises(replay_module.FixtureError, match="schema_version must be 1"): + replay_module.replay(fixture_document) + + fixture_document["schema_version"] = 1.0 + assert replay_module.replay(fixture_document)["schema_version"] == 1 + + +def test_missing_costs_keep_quality_metrics_and_deterministic_output( + tmp_path: Path, replay_module: ModuleType, fixture_document: dict[str, object] +) -> None: + """Evaluate quality without inventing cost totals or cost-based tie breaks.""" + document = deepcopy(fixture_document) + document["candidates"].reverse() + for case in document["cases"]: + for outcome in case["outcomes"].values(): + outcome["quality"] = 1.0 + outcome.pop("cost") + + report = replay_module.replay(document) + assert report["routed"] == {"quality": 3.0, "cost": None} + assert report["fixed_targets"] == { + "efficient": {"quality": 3.0, "cost": None}, + "capable": {"quality": 3.0, "cost": None}, + } + assert report["best_fixed_target"] == "efficient" + assert report["quality_regret_vs_best_fixed"] == 0.0 + assert report["cost_delta_vs_best_fixed"] is None + + path = tmp_path / "cost-free.json" + path.write_text(json.dumps(document), encoding="utf-8") + command = [sys.executable, str(SCRIPT), str(path)] + first = subprocess.run(command, check=False, capture_output=True, env={}) + second = subprocess.run(command, check=False, capture_output=True, env={}) + assert first.returncode == second.returncode == 0 + assert first.stderr == second.stderr == b"" + assert first.stdout == second.stdout + + def test_invalid_probability_distribution_is_rejected( replay_module: ModuleType, fixture_document: dict[str, object] ) -> None: From 3d6212546c96e61c2aaad3f32c005192bc05351b Mon Sep 17 00:00:00 2001 From: Clement Pakkam Isaac Date: Thu, 1 Oct 2026 09:26:05 -0700 Subject: [PATCH 3/3] fix(evals): reject mixed resolved models in replay Signed-off-by: Clement Pakkam Isaac --- benchmark/typesafe/README.md | 11 ++++++-- .../typesafe/fixtures/synthetic-routing.json | 8 +++++- benchmark/typesafe_replay.py | 11 ++++++++ tests/test_typesafe_replay.py | 26 +++++++++++++++++++ 4 files changed, 53 insertions(+), 3 deletions(-) diff --git a/benchmark/typesafe/README.md b/benchmark/typesafe/README.md index 4c533cd69..120d1ced7 100644 --- a/benchmark/typesafe/README.md +++ b/benchmark/typesafe/README.md @@ -19,13 +19,20 @@ always produces the same bytes. Every fixture is one JSON document with numeric `schema_version: 1`; JSON Booleans are not valid versions. It records: -- a suite name and pinned Jev model identifier; +- a suite name and requested Jev model identifier, which may be an alias; - two or more candidate labels and descriptions; - the confidence threshold and fallback target; -- one or more unique candidate orders per case, with a complete probability distribution for each; +- one or more unique candidate orders per case, each with the provider-reported `resolved_model` + and a complete probability distribution; - measured quality in `[0, 1]` for every candidate outcome; - optional nonnegative cost for each candidate outcome. +The `resolved_model` must be the provider-reported model version that answered each order probe, +not a copy of the requested alias. If the provider only echoes an alias and does not identify the +resolved version, the case cannot establish same-version order sensitivity. All orders in a case +must have the same resolved model; otherwise the replay rejects the case because model drift could +be mistaken for candidate-order sensitivity. The report includes that identity for each case. + The replay averages the recorded distributions, normalizes the average, and chooses the largest probability. Configured candidate order breaks an exact tie. Confidence is the selected probability's improvement over a uniform distribution, scaled to `[0, 1]`. A result below diff --git a/benchmark/typesafe/fixtures/synthetic-routing.json b/benchmark/typesafe/fixtures/synthetic-routing.json index 15c78a242..850317bf6 100644 --- a/benchmark/typesafe/fixtures/synthetic-routing.json +++ b/benchmark/typesafe/fixtures/synthetic-routing.json @@ -1,7 +1,7 @@ { "schema_version": 1, "suite": "synthetic-two-target-routing", - "model": "jev-pinned-example", + "model": "jev-requested-example", "candidates": [ { "label": "capable", @@ -19,10 +19,12 @@ "id": "simple-request", "orders": [ { + "resolved_model": "jev-resolved-example-r1", "candidate_order": ["capable", "efficient"], "probabilities": {"capable": 0.1, "efficient": 0.9} }, { + "resolved_model": "jev-resolved-example-r1", "candidate_order": ["efficient", "capable"], "probabilities": {"capable": 0.2, "efficient": 0.8} } @@ -36,10 +38,12 @@ "id": "complex-request", "orders": [ { + "resolved_model": "jev-resolved-example-r1", "candidate_order": ["capable", "efficient"], "probabilities": {"capable": 0.8, "efficient": 0.2} }, { + "resolved_model": "jev-resolved-example-r1", "candidate_order": ["efficient", "capable"], "probabilities": {"capable": 0.6, "efficient": 0.4} } @@ -53,10 +57,12 @@ "id": "order-sensitive-request", "orders": [ { + "resolved_model": "jev-resolved-example-r1", "candidate_order": ["capable", "efficient"], "probabilities": {"capable": 0.7, "efficient": 0.3} }, { + "resolved_model": "jev-resolved-example-r1", "candidate_order": ["efficient", "capable"], "probabilities": {"capable": 0.3, "efficient": 0.7} } diff --git a/benchmark/typesafe_replay.py b/benchmark/typesafe_replay.py index 4889e7fbe..6236684e8 100644 --- a/benchmark/typesafe_replay.py +++ b/benchmark/typesafe_replay.py @@ -135,8 +135,18 @@ def replay(document: dict[str, Any]) -> dict[str, Any]: seen_orders: set[tuple[str, ...]] = set() distributions: list[dict[str, float]] = [] order_winners: list[str] = [] + resolved_model: str | None = None for order_index, order_value in enumerate(order_rows): order = object_value(order_value, f"case {case_id!r} order {order_index}") + order_model = string_value( + order.get("resolved_model"), f"case {case_id!r} order {order_index}.resolved_model" + ) + if resolved_model is None: + resolved_model = order_model + else: + require( + order_model == resolved_model, f"case {case_id!r} has mixed resolved models" + ) order_labels = string_list( order.get("candidate_order"), f"case {case_id!r} order {order_index}.candidate_order", @@ -212,6 +222,7 @@ def replay(document: dict[str, Any]) -> dict[str, Any]: case_reports.append( { "id": case_id, + "resolved_model": resolved_model, "average_probabilities": {label: rounded(averaged[label]) for label in labels}, "classifier_target": classifier_target, "confidence": rounded(confidence), diff --git a/tests/test_typesafe_replay.py b/tests/test_typesafe_replay.py index e3f51a07c..484fdcead 100644 --- a/tests/test_typesafe_replay.py +++ b/tests/test_typesafe_replay.py @@ -55,6 +55,7 @@ def test_synthetic_fixture_reports_routing_quality_cost_and_order_sensitivity( assert report["cost_delta_vs_best_fixed"] == -16.0 assert report["cases"][2] == { "id": "order-sensitive-request", + "resolved_model": "jev-resolved-example-r1", "average_probabilities": {"capable": 0.5, "efficient": 0.5}, "classifier_target": "capable", "confidence": 0.0, @@ -154,3 +155,28 @@ def test_duplicate_candidate_order_is_rejected( with pytest.raises(replay_module.FixtureError, match="repeats a candidate order"): replay_module.replay(document) + + +def test_mixed_resolved_models_are_rejected( + replay_module: ModuleType, fixture_document: dict[str, object] +) -> None: + """Do not attribute a model revision change to candidate order.""" + fixture_document["model"] = "jev-latest" + orders = fixture_document["cases"][0]["orders"] + orders[0]["resolved_model"] = "jev-revision-a" + orders[1]["resolved_model"] = "jev-revision-b" + + with pytest.raises(replay_module.FixtureError, match="mixed resolved models"): + replay_module.replay(fixture_document) + + +def test_missing_resolved_model_is_rejected( + replay_module: ModuleType, fixture_document: dict[str, object] +) -> None: + """Require provenance for every order probe, not just the first one.""" + fixture_document["cases"][0]["orders"][1].pop("resolved_model") + + with pytest.raises( + replay_module.FixtureError, match="resolved_model must be a non-empty string" + ): + replay_module.replay(fixture_document)