From a68d670166ff98e6857fc76dcc2581d649dcf5d3 Mon Sep 17 00:00:00 2001 From: ayushag Date: Tue, 29 Sep 2026 22:39:34 -0700 Subject: [PATCH 01/48] feat(sim): add Python task routing evaluation Signed-off-by: ayushag --- Cargo.lock | 2 + README.md | 1 + crates/libsy-llm-client/src/lib.rs | 2 +- crates/libsy-llm-client/src/observation.rs | 4 +- crates/libsy-llm-client/src/run.rs | 127 +++++++- crates/switchyard-py/Cargo.toml | 2 + crates/switchyard-py/src/lib.rs | 2 + crates/switchyard-py/src/libsy_bindings.rs | 62 ++-- crates/switchyard-py/src/runner_bindings.rs | 293 +++++++++++++++++ crates/switchyard-runner/src/config.rs | 12 +- crates/switchyard-runner/src/failure.rs | 62 ++++ crates/switchyard-runner/src/route.rs | 22 +- docs/simulation.md | 183 +++++++++++ mkdocs.yml | 1 + switchyard/runner.py | 8 + switchyard/sim/__init__.py | 24 ++ switchyard/sim/__main__.py | 148 +++++++++ switchyard/sim/dataset.py | 135 ++++++++ switchyard/sim/evaluate.py | 292 +++++++++++++++++ switchyard/sim/harbor.py | 271 ++++++++++++++++ switchyard/sim/models.py | 135 ++++++++ switchyard/sim/report.py | 277 ++++++++++++++++ switchyard_rust/runner.py | 116 +++++++ tests/sim/test_dataset.py | 168 ++++++++++ tests/sim/test_evaluate.py | 296 +++++++++++++++++ tests/sim/test_harbor.py | 305 ++++++++++++++++++ tests/sim/test_integration.py | 148 +++++++++ tests/sim/test_report.py | 262 ++++++++++++++++ tests/test_runner_bindings.py | 331 ++++++++++++++++++++ 29 files changed, 3644 insertions(+), 47 deletions(-) create mode 100644 crates/switchyard-py/src/runner_bindings.rs create mode 100644 docs/simulation.md create mode 100644 switchyard/runner.py create mode 100644 switchyard/sim/__init__.py create mode 100644 switchyard/sim/__main__.py create mode 100644 switchyard/sim/dataset.py create mode 100644 switchyard/sim/evaluate.py create mode 100644 switchyard/sim/harbor.py create mode 100644 switchyard/sim/models.py create mode 100644 switchyard/sim/report.py create mode 100644 switchyard_rust/runner.py create mode 100644 tests/sim/test_dataset.py create mode 100644 tests/sim/test_evaluate.py create mode 100644 tests/sim/test_harbor.py create mode 100644 tests/sim/test_integration.py create mode 100644 tests/sim/test_report.py create mode 100644 tests/test_runner_bindings.py diff --git a/Cargo.lock b/Cargo.lock index b1cbce1be..17eef38bf 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2413,7 +2413,9 @@ dependencies = [ "serde", "serde_json", "switchyard-libsy", + "switchyard-llm-client", "switchyard-protocol", + "switchyard-runner", "switchyard-server", "tokio", ] diff --git a/README.md b/README.md index 460ddb6a3..dc2b96350 100644 --- a/README.md +++ b/README.md @@ -63,6 +63,7 @@ The [routing overview](docs/routing_algorithms/overview.md) retains the full cat ![Task completion versus cost for Switchyard classification, stage, and escalation routing, compared with Opus 4.8 and GLM 5.2 single-model baselines.](assets/switchyard-cost-accuracy.png) Results depend on the benchmark, model pool, serving stack, and routing configuration. +Use the Python [task routing evaluator](docs/simulation.md) to compare routing decisions against recorded Harbor Claude and Codex runs. For latency and routing overhead testing, see [Soak Testing](docs/operations/soak_test.md). ### Further reading diff --git a/crates/libsy-llm-client/src/lib.rs b/crates/libsy-llm-client/src/lib.rs index d5f579c7d..f62cd6acb 100644 --- a/crates/libsy-llm-client/src/lib.rs +++ b/crates/libsy-llm-client/src/lib.rs @@ -30,7 +30,7 @@ pub use client::{AuxiliaryOperation, ModelConfig, TranslatingLlmClient}; pub use error::{LlmClientError, Result}; pub use observation::{LlmCallObservation, RunObservation, RunObserver}; pub use raw::RawResponse; -pub use run::{ClientRouter, decide, run}; +pub use run::{ClientRouter, decide, decide_with_observer, run}; pub use switchyard_translation::RawEventStream; /// Registers process-wide compatibility gauges with the global meter provider. diff --git a/crates/libsy-llm-client/src/observation.rs b/crates/libsy-llm-client/src/observation.rs index 308ad4b28..459a47f1e 100644 --- a/crates/libsy-llm-client/src/observation.rs +++ b/crates/libsy-llm-client/src/observation.rs @@ -22,7 +22,7 @@ pub struct LlmCallObservation { pub usage: Option, } -/// Events emitted inline while [`crate::run`] serves a routing request. +/// Events emitted inline by [`crate::run`] and [`crate::decide_with_observer`]. #[derive(Clone, Debug)] pub enum RunObservation { /// Metadata attached to the completed routing outcome. @@ -31,7 +31,7 @@ pub enum RunObservation { LlmCall(LlmCallObservation), /// A completed terminal model call made from the routing outcome. AnswerCall(LlmCallObservation), - /// Routing time recorded by the `switchyard.routing_overhead_ms` metric. + /// Elapsed routing time, including model calls required by the algorithm. RoutingOverhead(Duration), } diff --git a/crates/libsy-llm-client/src/run.rs b/crates/libsy-llm-client/src/run.rs index 691e169cf..d0f39a34a 100644 --- a/crates/libsy-llm-client/src/run.rs +++ b/crates/libsy-llm-client/src/run.rs @@ -131,23 +131,57 @@ pub async fn decide( request: Request, models: Arc, ) -> Result { + decide_with_observer(algorithm, clients, request, models, None).await +} + +/// Resolve a decision and report its completed calls and elapsed routing time. +/// +/// Observations belong to this invocation, including calls completed before a failure. +/// A routing-time response is reported as an answer call when it becomes the outcome. +pub async fn decide_with_observer( + algorithm: Arc, + clients: ClientRouter, + request: Request, + models: Arc, + observer: Option, +) -> Result { + let started = Instant::now(); let routing_clients = clients.clone(); - let mut outcome = match clients.stored_state_owner(&request) { - Some(owner) => continue_on(owner, algorithm.name(), request), + let observations = observer.as_ref().map(|_| Arc::new(Mutex::new(Vec::new()))); + let outcome = match clients.stored_state_owner(&request) { + Some(owner) => Ok(continue_on(owner, algorithm.name(), request)), None => { - drive(algorithm, request, models, move |call| { - serve(routing_clients.clone(), call, None) + drive(algorithm, request, models, { + let observations = observations.clone(); + move |call| serve(routing_clients.clone(), call, observations.clone()) }) - .await? + .await } }; - let selected_model_id = outcome.selected_model_id()?.clone(); - outcome.request = clients.prepare_completion_request(outcome.request, &selected_model_id); - outcome.response = outcome - .response - .map(|response| clients.remember_state_owner(&outcome.request, response)) - .transpose()?; - Ok(outcome) + let answered_model = outcome + .as_ref() + .ok() + .and_then(|outcome| outcome.response.as_ref()) + .and_then(Response::served_model); + emit_routing_observations(&observer, &observations, answered_model); + let outcome = outcome.and_then(|mut outcome| { + let selected_model_id = outcome.selected_model_id()?.clone(); + outcome.request = clients.prepare_completion_request(outcome.request, &selected_model_id); + outcome.response = outcome + .response + .map(|response| clients.remember_state_owner(&outcome.request, response)) + .transpose()?; + Ok(outcome) + }); + if let Some(observer) = observer { + if let Ok(outcome) = &outcome + && let Some(metadata) = &outcome.metadata + { + observer(RunObservation::Outcome(metadata.clone())); + } + observer(RunObservation::RoutingOverhead(started.elapsed())); + } + outcome } /// Emits completed routing calls after the outcome reveals whether one response became the answer. @@ -566,6 +600,11 @@ enum Routing { } impl ClientRouter { + /// Configured target that may produce an answer during routing. + pub fn routing_answer_target(&self) -> Option<&ModelId> { + self.inner.routing_answer_target.as_ref() + } + /// Build a router over `model name -> client`, for targets spread across providers. pub fn new(by_model: HashMap>) -> Self { Self::new_with_target_prompts(by_model, HashMap::new(), None) @@ -1817,6 +1856,70 @@ mod tests { Ok(()) } + #[tokio::test] + async fn observed_decision_does_not_call_a_selected_completion() -> Result<()> { + let client = Arc::new(CandidateClient { + calls: Mutex::new(Vec::new()), + requests: Mutex::new(Vec::new()), + first: FirstOutcome::StreamSuccess, + }); + let observations = Arc::new(Mutex::new(Vec::new())); + let captured = Arc::clone(&observations); + let observer: RunObserver = Arc::new(move |event| captured.lock().push(event)); + + let outcome = decide_with_observer( + Arc::new(CandidateAlgorithm {}), + ClientRouter::single(client.clone()), + request(), + to_category_map(&["weak", "strong"]), + Some(observer), + ) + .await?; + + assert_eq!(outcome.selected_model_id()?, "weak"); + assert!(outcome.response.is_none()); + assert!(client.calls.lock().is_empty()); + assert!(matches!( + &observations.lock()[..], + [ + RunObservation::Outcome(_), + RunObservation::RoutingOverhead(_) + ] + )); + Ok(()) + } + + #[tokio::test] + async fn observed_decision_reports_completed_calls_on_failure() { + let client = Arc::new(CandidateClient { + calls: Mutex::new(Vec::new()), + requests: Mutex::new(Vec::new()), + first: FirstOutcome::Unauthorized, + }); + let observations = Arc::new(Mutex::new(Vec::new())); + let captured = Arc::clone(&observations); + let observer: RunObserver = Arc::new(move |event| captured.lock().push(event)); + + let result = decide_with_observer( + Arc::new(AnsweredAlgorithm { + model: "weak".into(), + }), + ClientRouter::single(client.clone()), + request(), + to_category_map(&["weak", "strong"]), + Some(observer), + ) + .await; + + assert!(result.is_err()); + assert_eq!(&*client.calls.lock(), &[ModelId::from("weak")]); + assert!(matches!( + &observations.lock()[..], + [RunObservation::LlmCall(call), RunObservation::RoutingOverhead(_)] + if call.selected_model == "weak" && !call.is_success && call.usage.is_none() + )); + } + #[tokio::test] async fn decision_prompts_a_routing_response_target() -> Result<()> { let client = Arc::new(CandidateClient { diff --git a/crates/switchyard-py/Cargo.toml b/crates/switchyard-py/Cargo.toml index aec8d6797..0e1eb9c8e 100644 --- a/crates/switchyard-py/Cargo.toml +++ b/crates/switchyard-py/Cargo.toml @@ -19,11 +19,13 @@ crate-type = ["cdylib", "rlib"] futures.workspace = true http.workspace = true switchyard-libsy.workspace = true +switchyard-llm-client.workspace = true pyo3 = { version = "0.28.3", features = ["abi3-py310"] } pyo3-async-runtimes = { version = "0.28", features = ["tokio-runtime"] } pythonize = "0.28.0" serde.workspace = true serde_json.workspace = true switchyard-protocol.workspace = true +switchyard-runner.workspace = true switchyard-server.workspace = true tokio.workspace = true diff --git a/crates/switchyard-py/src/lib.rs b/crates/switchyard-py/src/lib.rs index 1749c9ce4..29fa2e883 100644 --- a/crates/switchyard-py/src/lib.rs +++ b/crates/switchyard-py/src/lib.rs @@ -6,12 +6,14 @@ use pyo3::prelude::*; mod errors; mod libsy_bindings; mod py_serde; +mod runner_bindings; mod server_bindings; #[pymodule] fn _switchyard_rust(module: &Bound<'_, PyModule>) -> PyResult<()> { errors::register(module)?; libsy_bindings::register(module)?; + runner_bindings::register(module)?; server_bindings::register(module)?; Ok(()) } diff --git a/crates/switchyard-py/src/libsy_bindings.rs b/crates/switchyard-py/src/libsy_bindings.rs index 23c90d7ad..42103c433 100644 --- a/crates/switchyard-py/src/libsy_bindings.rs +++ b/crates/switchyard-py/src/libsy_bindings.rs @@ -61,7 +61,9 @@ fn category_models_from_python( } /// Convert Python-owned headers into the request metadata expected by libsy. -fn header_map_from_python(headers: &HashMap) -> PyResult { +pub(crate) fn header_map_from_python( + headers: &HashMap, +) -> PyResult { let mut result = http::HeaderMap::new(); for (name, value) in headers { let name = HeaderName::from_bytes(name.as_bytes()) @@ -591,7 +593,7 @@ impl PyOutcomeMetadata { /// The terminal routing selection, rewritten request, optional response, and metadata. #[pyclass(name = "RoutingOutcome", module = "switchyard.libsy", frozen)] -struct PyRoutingOutcome { +pub(crate) struct PyRoutingOutcome { selected_model_ids: Vec, request: Py, response: Option>, @@ -768,37 +770,39 @@ fn step_to_python(step: RustStep) -> PyResult { call: Py::new(py, PyModelCall::new(py, *call)?)?, }) }), - RustStep::Done(outcome) => { - let RoutingOutcome { - selected_model_ids, - request, - response, - metadata, - } = *outcome; - Python::attach(|py| { - Ok(PyStep::Done { - outcome: Py::new( - py, - PyRoutingOutcome { - metadata: metadata - .map(|inner| Py::new(py, PyOutcomeMetadata { inner })) - .transpose()?, - selected_model_ids: selected_model_ids - .iter() - .map(ToString::to_string) - .collect(), - request: to_python(py, &request.llm_request)?, - response: response - .map(|response| response_to_python(py, response.llm_response)) - .transpose()?, - }, - )?, - }) + RustStep::Done(outcome) => Python::attach(|py| { + Ok(PyStep::Done { + outcome: outcome_to_python(py, *outcome)?, }) - } + }), } } +pub(crate) fn outcome_to_python( + py: Python<'_>, + outcome: RoutingOutcome, +) -> PyResult> { + Py::new( + py, + PyRoutingOutcome { + metadata: outcome + .metadata + .map(|inner| Py::new(py, PyOutcomeMetadata { inner })) + .transpose()?, + selected_model_ids: outcome + .selected_model_ids + .iter() + .map(ToString::to_string) + .collect(), + request: to_python(py, &outcome.request.llm_request)?, + response: outcome + .response + .map(|response| response_to_python(py, response.llm_response)) + .transpose()?, + }, + ) +} + /// Construct the no-op reference algorithm. #[pyfunction(name = "noop")] fn noop_algorithm() -> PyAlgorithm { diff --git a/crates/switchyard-py/src/runner_bindings.rs b/crates/switchyard-py/src/runner_bindings.rs new file mode 100644 index 000000000..1de632405 --- /dev/null +++ b/crates/switchyard-py/src/runner_bindings.rs @@ -0,0 +1,293 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +//! Native configured routing decisions for Python hosts. + +use std::collections::HashMap; +use std::path::PathBuf; +use std::sync::{Arc, Mutex}; + +use pyo3::create_exception; +use pyo3::exceptions::{PyRuntimeError, PyValueError}; +use pyo3::prelude::*; +use switchyard_llm_client::{LlmCallObservation, RunObservation, RunObserver}; +use switchyard_protocol::{Metadata, ModelId, Request}; +use switchyard_runner::{DecisionTarget, Route, Runner, RunnerError}; + +use crate::libsy_bindings::{PyRoutingOutcome, header_map_from_python, outcome_to_python}; +use crate::py_serde::{from_python, to_python}; + +create_exception!(_switchyard_rust, DecisionError, PyRuntimeError); + +/// Configured identity of one selected target. +#[pyclass(name = "DecisionTarget", module = "switchyard.runner", frozen)] +struct PyDecisionTarget { + inner: DecisionTarget, +} + +#[pymethods] +impl PyDecisionTarget { + #[getter] + fn target(&self) -> &str { + &self.inner.target + } + + #[getter] + fn model(&self) -> &str { + self.inner.model.as_str() + } +} + +/// A completed logical model call, including backend retries in its duration. +#[pyclass(name = "RoutingCall", module = "switchyard.runner", frozen)] +struct PyRoutingCall { + inner: LlmCallObservation, +} + +#[pymethods] +impl PyRoutingCall { + #[getter] + fn model(&self) -> &str { + self.inner.selected_model.as_str() + } + + #[getter] + fn is_success(&self) -> bool { + self.inner.is_success + } + + #[getter] + fn duration_seconds(&self) -> f64 { + self.inner.duration.as_secs_f64() + } + + /// Native normalized usage; missing usage and token fields remain unknown. + #[getter] + fn usage(&self, py: Python<'_>) -> PyResult>> { + self.inner + .usage + .as_ref() + .map(|usage| to_python(py, usage)) + .transpose() + } +} + +/// A native outcome, its configured target identities, and request-scoped costs. +#[pyclass(name = "Decision", module = "switchyard.runner", frozen)] +struct PyDecision { + #[pyo3(get)] + selected: Py, + #[pyo3(get)] + fallbacks: Vec>, + #[pyo3(get)] + outcome: Py, + #[pyo3(get)] + calls: Vec>, + #[pyo3(get)] + duration_seconds: f64, +} + +#[derive(Default)] +struct Observations { + calls: Vec, + duration_seconds: f64, +} + +/// Shared native route configuration and algorithm state. +#[pyclass(name = "Runner", module = "switchyard.runner", frozen)] +struct PyRunner { + inner: Arc, +} + +#[pymethods] +impl PyRunner { + /// Load and validate a native deployment TOML file without starting a server. + #[staticmethod] + fn load(py: Python<'_>, path: PathBuf) -> PyResult { + py.detach(move || { + let _guard = pyo3_async_runtimes::tokio::get_runtime().enter(); + Runner::load(path) + .map(Self::from_runner) + .map_err(config_error) + }) + } + + /// Load and validate a native deployment TOML document. + #[staticmethod] + fn from_toml(py: Python<'_>, source: String) -> PyResult { + py.detach(move || { + let _guard = pyo3_async_runtimes::tokio::get_runtime().enter(); + Runner::from_toml(&source) + .map(Self::from_runner) + .map_err(config_error) + }) + } + + /// Check a route before evaluation can issue any provider calls. + #[pyo3(signature = (model, *, allow_response=false))] + fn validate_decision_route( + &self, + py: Python<'_>, + model: &str, + allow_response: bool, + ) -> PyResult>> { + decision_route(&self.inner, model, allow_response)? + .decision_targets() + .iter() + .cloned() + .map(|inner| Py::new(py, PyDecisionTarget { inner })) + .collect() + } + + /// Resolve one normalized IR request using native configuration and transport. + /// + /// Response-based algorithms require `allow_response=True` because they can call + /// an answer model while deciding. Separate runs may execute concurrently. The + /// caller must give independent tasks distinct session identities and serialize + /// requests whose shared session state must remain ordered. + #[pyo3(signature = (request, *, headers=None, allow_response=false))] + fn decide<'py>( + &self, + py: Python<'py>, + request: &Bound<'_, PyAny>, + headers: Option>, + allow_response: bool, + ) -> PyResult> { + let headers = headers.as_ref().map(header_map_from_python).transpose()?; + let request = Request { + llm_request: from_python(request)?, + raw_request: None, + metadata: headers.map(|headers| Metadata::from_headers(&headers)), + }; + let model = request + .llm_request + .model + .as_deref() + .filter(|model| !model.trim().is_empty()) + .ok_or_else(|| PyValueError::new_err("request must include a non-empty model"))?; + decision_route(&self.inner, model, allow_response)?; + let model = ModelId::from(model); + let runner = Arc::clone(&self.inner); + pyo3_async_runtimes::tokio::future_into_py(py, async move { + let observations = Arc::new(Mutex::new(Observations::default())); + let captured = Arc::clone(&observations); + let observer: RunObserver = Arc::new(move |event| { + let mut captured = captured.lock().expect("observation lock poisoned"); + match event { + RunObservation::LlmCall(call) | RunObservation::AnswerCall(call) => { + captured.calls.push(call); + } + RunObservation::RoutingOverhead(duration) => { + captured.duration_seconds = duration.as_secs_f64(); + } + RunObservation::Outcome(_) => {} + } + }); + let route = decision_route(&runner, model.as_str(), allow_response)?; + let result = route.decide_with_observer(request, Some(observer)).await; + let observations = + std::mem::take(&mut *observations.lock().expect("observation lock poisoned")); + Python::attach(|py| { + let calls = observations + .calls + .into_iter() + .map(|inner| Py::new(py, PyRoutingCall { inner })) + .collect::>>()?; + let outcome = result.map_err(|error| { + decision_error(py, error, &calls, observations.duration_seconds) + })?; + let description = runner.describe_decision(&model, &outcome).ok_or_else(|| { + decision_error( + py, + RunnerError::Algorithm(switchyard_libsy::LibsyError::AlgorithmError { + message: "routing outcome has no configured target".to_string(), + }), + &calls, + observations.duration_seconds, + ) + })?; + Py::new( + py, + PyDecision { + selected: Py::new( + py, + PyDecisionTarget { + inner: description.selected, + }, + )?, + fallbacks: description + .fallbacks + .into_iter() + .map(|inner| Py::new(py, PyDecisionTarget { inner })) + .collect::>>()?, + outcome: outcome_to_python(py, outcome)?, + calls, + duration_seconds: observations.duration_seconds, + }, + ) + }) + }) + } +} + +impl PyRunner { + fn from_runner(inner: Runner) -> Self { + Self { + inner: Arc::new(inner), + } + } +} + +fn decision_route<'a>( + runner: &'a Runner, + model: &str, + allow_response: bool, +) -> PyResult<&'a Route> { + let route = runner + .route(model) + .ok_or_else(|| PyValueError::new_err(format!("unknown route model {model:?}")))?; + if !allow_response && route.routing_answer_target().is_some() { + return Err(PyValueError::new_err( + "route can call an answer model while deciding; set allow_response=True to permit it", + )); + } + Ok(route) +} + +fn config_error(error: RunnerError) -> PyErr { + PyValueError::new_err(error.configuration_diagnostic()) +} + +fn decision_error( + py: Python<'_>, + error: RunnerError, + calls: &[Py], + duration_seconds: f64, +) -> PyErr { + let summary = error.execution_error_summary(); + // Provider error bodies may echo credentials or request text. The native summary + // preserves the actionable class and HTTP status without those untrusted details. + let error = DecisionError::new_err(format!("routing failed: {}", summary.kind.as_str())); + let value = error.value(py); + let set_fields = || -> PyResult<()> { + value.setattr("kind", summary.kind.as_str())?; + value.setattr("upstream_status", summary.upstream_status)?; + value.setattr("target", summary.target.as_ref().map(ModelId::as_str))?; + value.setattr("calls", calls)?; + value.setattr("duration_seconds", duration_seconds) + }; + if let Err(error) = set_fields() { + return error; + } + error +} + +pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { + let runner_module = PyModule::new(module.py(), "runner")?; + runner_module.add("DecisionError", module.py().get_type::())?; + runner_module.add_class::()?; + runner_module.add_class::()?; + runner_module.add_class::()?; + runner_module.add_class::()?; + module.add_submodule(&runner_module) +} diff --git a/crates/switchyard-runner/src/config.rs b/crates/switchyard-runner/src/config.rs index f37b4cda9..c2de21a5b 100644 --- a/crates/switchyard-runner/src/config.rs +++ b/crates/switchyard-runner/src/config.rs @@ -448,6 +448,8 @@ impl DeploymentConfig { let response = self.targets.get(response_name).ok_or_else(|| { RunnerError::configuration(format!("route references unknown target {response_name}")) })?; + // Keep execution metadata even when this target has no prompt override. + policy.routing_answer_target = Some(response.id.clone()); if !policy.prompts.contains_key(&response.id) { return Ok(policy); } @@ -460,7 +462,6 @@ impl DeploymentConfig { response.id, ))); } - policy.routing_answer_target = Some(response.id.clone()); Ok(policy) } @@ -1164,7 +1165,14 @@ new = ["send_message"] "base_threshold = 0.5", "base_threshold = 0.5\nescalation = { confirmations = 2 }", ); - runner_from_toml(&escalating)?; + let runner = runner_from_toml(&escalating)?; + assert_eq!( + runner + .route("switchyard/classifier") + .and_then(Route::routing_answer_target) + .map(ModelId::as_str), + Some("weak/model") + ); let reversible = VALID_CONFIG.replace( "base_threshold = 0.5", diff --git a/crates/switchyard-runner/src/failure.rs b/crates/switchyard-runner/src/failure.rs index 8113ef25b..2ceb7ea65 100644 --- a/crates/switchyard-runner/src/failure.rs +++ b/crates/switchyard-runner/src/failure.rs @@ -83,6 +83,35 @@ pub struct RouteErrorSummary { } impl RunnerError { + /// Configuration diagnostics without TOML source snippets or offending values. + /// + /// Native validation messages identify fields and constraints. Deserialization + /// errors can include credentials from a source line or literal, so those return + /// only the error category and a one-based byte position when available. + pub fn configuration_diagnostic(&self) -> String { + let Self::Configuration { message, source } = self else { + return "invalid Switchyard configuration".to_string(); + }; + if let Some(source) = source { + if let Some(error) = source.downcast_ref::() { + return error.configuration_diagnostic(); + } + if let Some(error) = source.downcast_ref::() { + return match error.span() { + Some(span) => format!( + "invalid deployment TOML syntax or field value at byte {}", + span.start + 1 + ), + None => "invalid deployment TOML syntax or field value".to_string(), + }; + } + if let Some(error) = source.downcast_ref::() { + return format!("cannot read deployment configuration: {:?}", error.kind()); + } + } + message.clone() + } + /// Returns a safe telemetry summary for a failure before response delivery. pub fn execution_error_summary(&self) -> RouteErrorSummary { match self { @@ -178,6 +207,39 @@ mod tests { const SECRET: &str = "patient name is Jane Doe"; + #[test] + fn configuration_diagnostic_preserves_validation_context() { + let error = RunnerError::configuration("route references unknown target missing"); + let wrapped = RunnerError::configuration_source("invalid file configuration", error); + assert_eq!( + wrapped.configuration_diagnostic(), + "route references unknown target missing" + ); + let error = RunnerError::configuration_source( + "failed to read config", + std::io::Error::new(std::io::ErrorKind::NotFound, SECRET), + ); + assert_eq!( + error.configuration_diagnostic(), + "cannot read deployment configuration: NotFound" + ); + } + + #[test] + fn configuration_diagnostic_omits_toml_source_and_literal() { + let source = "count = 'provider-secret'"; + let error = toml::from_str::>(source) + .expect_err("string cannot be a token count"); + assert!(error.to_string().contains("provider-secret")); + let error = RunnerError::configuration_source(error.to_string(), error); + let wrapped = RunnerError::configuration_source(error.to_string(), error); + let diagnostic = wrapped.configuration_diagnostic(); + assert!(diagnostic.contains("TOML")); + assert!(diagnostic.contains("byte")); + assert!(!diagnostic.contains("provider-secret")); + assert!(!diagnostic.contains("count =")); + } + #[test] fn execution_error_summary_keeps_http_status_and_target_without_body() { let error = RunnerError::Algorithm(LibsyError::ClientCall { diff --git a/crates/switchyard-runner/src/route.rs b/crates/switchyard-runner/src/route.rs index 9d177c768..d510e0526 100644 --- a/crates/switchyard-runner/src/route.rs +++ b/crates/switchyard-runner/src/route.rs @@ -156,6 +156,16 @@ impl Route { self.algorithm.name() } + /// Configured target that may produce an answer during routing. + pub fn routing_answer_target(&self) -> Option<&ModelId> { + self.clients.routing_answer_target() + } + + /// Configured completion targets available to routing decisions. + pub fn decision_targets(&self) -> &[DecisionTarget] { + &self.decision_targets + } + /// Returns model-list capability metadata. pub fn capabilities(&self) -> ModelCapabilities { self.capabilities @@ -211,11 +221,21 @@ impl Route { /// Completes routing-time calls without serving a post-routing completion. pub async fn decide(&self, request: Request) -> Result { - switchyard_llm_client::decide( + self.decide_with_observer(request, None).await + } + + /// Completes routing-time calls with request-scoped observations. + pub async fn decide_with_observer( + &self, + request: Request, + observer: Option, + ) -> Result { + switchyard_llm_client::decide_with_observer( Arc::clone(&self.algorithm), self.clients.clone(), request, Arc::clone(&self.models), + observer, ) .await .map_err(Into::into) diff --git a/docs/simulation.md b/docs/simulation.md new file mode 100644 index 000000000..6a030c57e --- /dev/null +++ b/docs/simulation.md @@ -0,0 +1,183 @@ +# Evaluate task routing + +`switchyard.sim` evaluates a routing policy against completed Harbor runs. It +loads the input available before an agent starts, asks Switchyard to select a +target, and scores that choice using the target's recorded task outcomes. + +The simulator is a Python library included in `nemo-switchyard`. It uses the +public `switchyard.runner` Python interface. Switchyard's native bindings load +the existing TOML configuration and execute routing calls. No local server or +Harbor installation is needed. + +## Start with paired recordings + +Each target names a complete recorded model and agent configuration. Its name +must match a target in your Switchyard deployment. Use comparable task versions, +agent settings, reasoning budgets, and verifier settings across runs. A matching +task name or model name alone does not establish that the experiments are comparable. + +```python +import asyncio + +from switchyard.runner import Runner +from switchyard.sim import Dataset, evaluate, load_harbor + +fast = load_harbor("jobs/fast-baseline", target="fast") +strong = load_harbor("jobs/strong-baseline", target="strong") +dataset = Dataset.from_runs( + {"fast": fast, "strong": strong}, + input_target="fast", +) + +async def main(): + report = await evaluate( + dataset, + Runner.load("routes.toml"), + route="auto", + concurrency=8, + timeout=60, + on_result=lambda result: print(result.task_id, result.target, result.error), + ) + print(report.format_text()) + return report.to_dict() + +summary = asyncio.run(main()) +``` + +Create a fresh `Runner` for each independent experiment. It owns routing state, +including affinity and random-number generators. The evaluator creates a distinct +session ID for each task. Use `concurrency=1` with a seeded random route when task +order must be reproducible. Concurrent scheduling can change which task receives +each random draw. + +The configured route may select any recorded target, including a fixed-target +subset. Known recorded model IDs must match the configured model IDs. Use the +explicit `model_aliases={"recorded/provider/model": "configured/model"}` argument +to `evaluate` or `score` when the two systems name the same model differently. Aliases are a caller assertion; +the library does not guess equivalence by trimming model names. Missing model +metadata remains visible as unverified coverage. + +## Import rules and coverage + +`load_harbor` accepts a trial directory, a job directory, or a downloaded run's +`jobs/` layout. It reads per-trial `result.json` and `agent/trajectory.json`. +ATIF 1.5 and 1.7 Claude/Codex recordings are covered by tests. The importer reads +the fields it needs using the Python standard library. + +Routing input contains every initial system and user message before the first +agent step. This matters for Codex, whose first user message can describe the +environment and whose second contains the task. Completed agent messages, +verifier rewards, model names, and trial IDs are not added to classifier input. +The recorded initial input itself may contain harness or environment details. +`input_target` explicitly chooses which run supplies that input. + +Task names are matched within the supplied runs. Use `dataset="benchmark-v1"` +to namespace them. Conflicting task checksums are rejected. When a checksum is +missing, all initial user messages must match. Full system scaffolding may differ +between agents. Repeated trials retain their identities; each target's reward, +cost, and duration are averaged across all repeats of a task. Tasks then have +equal weight regardless of their number of repeats. + +Missing rewards, cost, duration, and token counts stay unknown. A task measurement +is unknown if any of its repeats lacks that measurement. Agent errors remain on +the imported trials. An observed verifier reward of zero is retained as zero. + +Imports and pairing are strict by default. To inspect an imperfect recording: + +```python +fast = load_harbor("jobs/fast", target="fast", on_error="record") +strong = load_harbor("jobs/strong", target="strong", on_error="record") +dataset = Dataset.from_runs( + {"fast": fast, "strong": strong}, + input_target="fast", + intersection=True, +) +print(dataset.coverage) +``` + +The intersection excludes a task from every target if any target is missing or +has an invalid trial for it. Issues and excluded task IDs remain in the report. +An invalid result with no recoverable task identity must be repaired before +pairing; otherwise a failed repeat could silently disappear from a task's mean. +For a missing trajectory, callers can supply `task_inputs={task_id: instruction}`. +Malformed existing trajectories still fail validation. Raw agent logs and +multimodal ATIF inputs are outside this initial importer. + +## Interpret the report + +The report includes processed, scored, unscored, and error counts; target +selection counts; recorded reward, task cost, and agent execution duration; +routing latency, call counts, usage, and routing cost coverage. Unknown totals +and means are `null`; `observed_total` retains the known portion. `complete` +means every expected task has a scored reward and no evaluation error. Inspect +each cost and usage field's coverage separately. + +Reward comparisons use one common cohort where the routed outcome and every +fixed-target reward are observed. The empirical recorded-outcome oracle selects +the largest target mean reward for each task. It is an upper bound on these +recordings, not a validated policy or evidence about future tasks. Cost comparisons +use their own common cohort with all fixed-target costs observed. Their task +counts can differ from reward comparisons. + +Recorded cost prefers `result.agent_result.cost_usd`, falling back to ATIF's +`final_metrics.total_cost_usd`. Choose `cost_source="trajectory"` to reverse that +preference. These producers can use different prices. Imported trials retain the +chosen source, and coverage includes source counts. Costs are not repriced to a +shared catalog. Duration covers only agent execution, excluding environment setup +and verification. + +Routing calls are live classifier work. They are distinct from the selected +recorded task cost. Native usage is returned unchanged; cache fields are separate +from non-cached input, and reasoning detail must not blindly be added to output. +No current price catalog is embedded. For USD estimates, supply +`price_call(call) -> float | None` to `evaluate` or `score`. The callback receives +the actual model and native usage. Return `None` for unknown rates or usage. +Without it, zero-call routes cost zero and other routing costs remain unknown. +Backend retries and failed calls may incur unreported spend. + +## Save progressive results + +```sh +python -m switchyard.sim \ + --config routes.toml --route auto \ + --run fast=jobs/fast --run strong=jobs/strong \ + --input-target fast --output evaluation-output +``` + +The output directory must be new. It contains flushed `results.jsonl` rows, a +`report.json` summary, and a `manifest.json` with configuration hash, inputs, +versions, and coverage. The deployment source and credentials are not copied. +Use `--skip-invalid --intersection` to retain import issues and explicitly +evaluate the common valid subset. Exit status is 0 for complete reward coverage, +1 for an incomplete evaluation, and 2 for invalid inputs or configuration. +Completed rows survive interruption; automatic resume is not implemented. +Use `--model-alias RECORDED=CONFIGURED` for an explicit model-ID equivalence. + +The Python `on_result` callback runs after each completed task. It can update a +dashboard or write an application-owned result stream. Keep it short; a callback +failure stops evaluation and drains pending work. Each decision has a deadline. +Cancellation stops local workers, but cannot revoke requests already received +by a provider. Task concurrency bounds simultaneous decisions; an algorithm may +make multiple provider calls inside a decision. + +## Library boundaries + +`Trial` and `HarborRun` represent imported evidence. `Dataset.from_runs` validates +and pairs it. `score(task, decision)` is a pure scorer for a saved native decision. +`evaluate` owns bounded scheduling and calls that scorer. `Report.add` accumulates +results without retaining full trajectories or result rows. Import holds only +one full trajectory at a time; paired task inputs and small outcome records stay +in memory. + +Native `Runner.load` and `Runner.from_toml` use Switchyard's configuration parser. +`await runner.decide(normalized_request, headers=...)` returns configured selected +and fallback targets, the existing `RoutingOutcome`, completed-call observations, +and elapsed routing time. Provider failures raise `DecisionError` with completed +observations and safe error metadata. Algorithms that generate an answer while +routing require explicit `allow_response=True`; task simulation rejects them. +The task scorer also rejects answers and material conversation rewrites, because +recorded outcomes cannot score those changes. + +Future trajectory replay can send successive normalized histories through the +same public Runner interface and supply its own scorer. Task simulation makes no +claim about mid-trajectory switching, replay fidelity, or changed agent behavior. diff --git a/mkdocs.yml b/mkdocs.yml index 57cf32d62..69029eb60 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -40,6 +40,7 @@ nav: - Escalation-Router Routing: routing_algorithms/escalation_router_routing.md - Advisor-Gate Routing: routing_algorithms/advisor_gate_routing.md - Operations: + - Task routing evaluation: simulation.md - Context-Window Handling: operations/context_window.md - Soak Testing: operations/soak_test.md - Reference: diff --git a/switchyard/runner.py b/switchyard/runner.py new file mode 100644 index 000000000..d07238ac2 --- /dev/null +++ b/switchyard/runner.py @@ -0,0 +1,8 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Load native route configuration and resolve routing decisions.""" + +from switchyard_rust.runner import Decision, DecisionError, DecisionTarget, RoutingCall, Runner + +__all__ = ["Decision", "DecisionError", "DecisionTarget", "RoutingCall", "Runner"] diff --git a/switchyard/sim/__init__.py b/switchyard/sim/__init__.py new file mode 100644 index 000000000..35b99e6c1 --- /dev/null +++ b/switchyard/sim/__init__.py @@ -0,0 +1,24 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Evaluate Switchyard task routing against recorded Harbor outcomes.""" + +from .dataset import Dataset +from .evaluate import evaluate, score +from .harbor import load_harbor +from .models import HarborRun, LoadIssue, Outcome, Result, Task, Trial +from .report import Report + +__all__ = [ + "Dataset", + "HarborRun", + "LoadIssue", + "Outcome", + "Report", + "Result", + "Task", + "Trial", + "evaluate", + "load_harbor", + "score", +] diff --git a/switchyard/sim/__main__.py b/switchyard/sim/__main__.py new file mode 100644 index 000000000..9c676a557 --- /dev/null +++ b/switchyard/sim/__main__.py @@ -0,0 +1,148 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""A thin command-line interface to the task evaluation library.""" + +from __future__ import annotations + +import argparse +import asyncio +import hashlib +import json +import sys +from dataclasses import asdict +from pathlib import Path + +from switchyard import __version__ +from switchyard.runner import Runner + +from . import Dataset, Result, evaluate, load_harbor + + +def main(argv: list[str] | None = None) -> int: + parser = argparse.ArgumentParser( + description="Evaluate Switchyard task routing on recorded Harbor runs." + ) + parser.add_argument( + "--config", required=True, type=Path, help="Native Switchyard deployment TOML" + ) + parser.add_argument("--route", required=True, help="Configured route model ID") + parser.add_argument( + "--run", + action="append", + required=True, + metavar="TARGET=PATH", + help="Recorded Harbor run for a configured target; repeat for each target", + ) + parser.add_argument( + "--input-target", required=True, help="Target whose initial agent input the router sees" + ) + parser.add_argument( + "--output", required=True, type=Path, help="New output directory (must not already exist)" + ) + parser.add_argument("--dataset", help="Optional task namespace") + parser.add_argument("--reward-key", default="reward") + parser.add_argument( + "--model-alias", + action="append", + default=[], + metavar="RECORDED=CONFIGURED", + help="Explicit equivalence between recorded and configured model IDs", + ) + parser.add_argument("--cost-source", choices=("result", "trajectory"), default="result") + parser.add_argument( + "--intersection", + action="store_true", + help="Explicitly evaluate only tasks valid for every target", + ) + parser.add_argument( + "--skip-invalid", + action="store_true", + help="Record rejected inputs; requires --intersection to evaluate remaining tasks", + ) + parser.add_argument("--concurrency", type=int, default=8) + parser.add_argument( + "--timeout", type=float, default=60, help="Whole-decision deadline in seconds" + ) + args = parser.parse_args(argv) + paths = {} + for value in args.run: + target, separator, path = value.partition("=") + if not separator or not target or not path or target in paths: + parser.error("each --run must be a unique TARGET=PATH") + paths[target] = Path(path).resolve() + aliases = {} + for value in args.model_alias: + recorded, separator, configured = value.partition("=") + if not separator or not recorded or not configured or recorded in aliases: + parser.error("each --model-alias must be a unique RECORDED=CONFIGURED") + aliases[recorded] = configured + try: + runs = { + target: load_harbor( + path, + target=target, + dataset=args.dataset, + reward_key=args.reward_key, + cost_source=args.cost_source, + on_error="record" if args.skip_invalid else "raise", + ) + for target, path in paths.items() + } + dataset = Dataset.from_runs( + runs, input_target=args.input_target, intersection=args.intersection + ) + runner = Runner.load(args.config) + args.output.mkdir(parents=True, exist_ok=False) + manifest = { + "schema_version": 1, + "switchyard_version": __version__, + "config_sha256": hashlib.sha256(args.config.read_bytes()).hexdigest(), + "route": args.route, + "runs": {target: str(path) for target, path in paths.items()}, + "input_target": args.input_target, + "reward_key": args.reward_key, + "cost_source": args.cost_source, + "model_aliases": aliases, + "concurrency": args.concurrency, + "timeout_seconds": args.timeout, + "coverage": dict(dataset.coverage), + } + (args.output / "manifest.json").write_text( + json.dumps(manifest, indent=2, allow_nan=False) + "\n" + ) + with (args.output / "results.jsonl").open("x", encoding="utf-8") as stream: + + def save(result: Result) -> None: + stream.write(json.dumps(asdict(result), allow_nan=False) + "\n") + stream.flush() + + report = asyncio.run( + evaluate( + dataset, + runner, + route=args.route, + concurrency=args.concurrency, + timeout=args.timeout, + on_result=save, + model_aliases=aliases, + ) + ) + (args.output / "report.json").write_text( + json.dumps(report.to_dict(), indent=2, allow_nan=False) + "\n" + ) + except (OSError, ValueError) as error: + print(f"switchyard.sim: {error}", file=sys.stderr) + return 2 + except KeyboardInterrupt: + print( + "switchyard.sim: interrupted; completed result rows remain in the output directory", + file=sys.stderr, + ) + return 130 + print(report.format_text()) + return 0 if report.complete else 1 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/switchyard/sim/dataset.py b/switchyard/sim/dataset.py new file mode 100644 index 000000000..d61d587ab --- /dev/null +++ b/switchyard/sim/dataset.py @@ -0,0 +1,135 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Match recorded targets without choosing favorable trials or hiding exclusions.""" + +from __future__ import annotations + +import json +from collections import Counter, defaultdict +from collections.abc import Mapping +from dataclasses import dataclass + +from .models import HarborRun, Outcome, Task, Trial + + +@dataclass(frozen=True) +class Dataset: + """A validated cohort with one decision per task and equal weight per task.""" + + tasks: tuple[Task, ...] + targets: tuple[str, ...] + coverage: Mapping[str, object] + + @classmethod + def from_runs( + cls, + runs: Mapping[str, HarborRun], + *, + input_target: str, + intersection: bool = False, + ) -> Dataset: + """Pair targets by task ID and checksum, averaging all recorded repeats. + + ``input_target`` supplies the routing input because agent scaffolding may + differ between baseline runs. ``intersection`` explicitly excludes tasks + missing or invalid in any run; exclusions remain in ``coverage``. + """ + if not runs or input_target not in runs: + raise ValueError("input_target must name one of the supplied runs") + grouped: dict[str, dict[str, list[Trial]]] = {} + invalid: set[str] = set() + issues = [] + for target, run in runs.items(): + groups: dict[str, list[Trial]] = defaultdict(list) + seen: set[tuple[str, str]] = set() + for trial in run.trials: + if trial.target != target: + raise ValueError(f"trial target {trial.target!r} does not match run {target!r}") + key = trial.task_id, trial.trial_id + if key in seen: + raise ValueError(f"duplicate trial {trial.trial_id!r} for {trial.task_id!r}") + seen.add(key) + groups[trial.task_id].append(trial) + grouped[target] = groups + for issue in run.issues: + if issue.task_id is None: + raise ValueError( + f"input issue has no task identity: {issue.source}; " + "repair or identify the rejected trial before pairing" + ) + issues.append( + { + "target": target, + "source": issue.source, + "task_id": issue.task_id, + "message": issue.message, + } + ) + if issue.task_id is not None: + invalid.add(issue.task_id) + task_sets = [set(groups) for groups in grouped.values()] + union = set.union(*task_sets) | invalid + common = set.intersection(*task_sets) - invalid + excluded = sorted(union - common) + if not intersection and (excluded or issues): + raise ValueError( + f"incomplete cohort: {len(excluded)} tasks missing or invalid, " + f"{len(issues)} input issues; use intersection=True to explicitly exclude them" + ) + if not common: + raise ValueError("no tasks have valid trials for every target") + tasks = [] + for task_id in sorted(common): + trials = { + target: tuple(sorted(groups[task_id], key=lambda trial: trial.trial_id)) + for target, groups in grouped.items() + } + all_trials = [trial for values in trials.values() for trial in values] + checksums = {trial.task_checksum for trial in all_trials if trial.task_checksum} + if len(checksums) > 1: + raise ValueError(f"conflicting task checksums for {task_id!r}") + if not all(trial.task_checksum for trial in all_trials): + instructions = {_instruction(trial) for trial in all_trials} + if len(instructions) != 1: + raise ValueError( + f"task {task_id!r} has conflicting inputs without complete checksums" + ) + messages = trials[input_target][0].messages + if not messages or not any(message.get("role") == "user" for message in messages): + raise ValueError(f"task {task_id!r} has no user input") + tasks.append( + Task( + task_id, + messages, + {target: Outcome.from_trials(values) for target, values in trials.items()}, + trials, + ) + ) + return cls( + tuple(tasks), + tuple(runs), + { + "input_target": input_target, + "intersection": intersection, + "tasks_seen": len(union), + "tasks_included": len(tasks), + "tasks_by_target": {target: len(groups) for target, groups in grouped.items()}, + "excluded_task_ids": excluded, + "input_issues": issues, + "cost_sources_by_target": { + target: dict(Counter(trial.cost_source or "unknown" for trial in run.trials)) + for target, run in runs.items() + }, + }, + ) + + +def _instruction(trial: Trial) -> str: + """Require the same user input when a Harbor checksum is unavailable.""" + contents = [ + message.get("content") for message in trial.messages if message.get("role") == "user" + ] + if not contents: + raise ValueError(f"task {trial.task_id!r} has no user input") + return json.dumps(contents, sort_keys=True, allow_nan=False) diff --git a/switchyard/sim/evaluate.py b/switchyard/sim/evaluate.py new file mode 100644 index 000000000..0e89b0157 --- /dev/null +++ b/switchyard/sim/evaluate.py @@ -0,0 +1,292 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Route and score completed tasks through the public Switchyard Python API.""" + +from __future__ import annotations + +import asyncio +import copy +import math +import time +import uuid +from collections.abc import Callable, Iterable, Mapping, Sequence +from typing import TYPE_CHECKING + +from .dataset import Dataset +from .models import Result, Task, Trial +from .report import Report + +if TYPE_CHECKING: + from switchyard.runner import Decision, RoutingCall, Runner + +PriceCall = Callable[["RoutingCall"], float | None] +_TOKEN_FIELDS = ( + "input_tokens", + "cached_input_tokens", + "cache_creation_input_tokens", + "output_tokens", + "reasoning_tokens", + "total_tokens", +) + + +def score( + task: Task, + decision: Decision, + *, + price_call: PriceCall | None = None, + model_aliases: Mapping[str, str] | None = None, +) -> Result: + """Score a native decision against recorded outcomes, without making calls. + + Saved outcomes support target selection only. They cannot score an answer + produced during routing or changes to the original task conversation. + ``price_call`` may estimate a routing call's USD cost using caller-owned + pricing; return ``None`` when usage or rates are unavailable. + Known recorded model IDs must match the decision, optionally using explicit + ``model_aliases``. Missing recorded IDs cannot be verified; ``evaluate`` + reports their coverage separately. + """ + _validate_selection(task, decision, _aliases(model_aliases)) + return _result(task, decision, price_call=price_call) + + +def _validate_selection(task: Task, decision: Decision, aliases: Mapping[str, str]) -> None: + target = decision.selected.target + if target not in task.outcomes: + raise ValueError(f"selected target {target!r} has no recorded outcome") + _validate_models(task.trials.get(target, ()), target, decision.selected.model, aliases) + outcome = decision.outcome + if outcome.response is not None: + raise ValueError("recorded task outcomes cannot score a response produced during routing") + request = outcome.request + if ( + request.get("messages") != list(task.messages) + or request.get("instructions") + or request.get("tools") + ): + raise ValueError("recorded task outcomes cannot score a rewritten task request") + + +def _aliases(model_aliases: Mapping[str, str] | None) -> dict[str, str]: + aliases = dict(model_aliases or {}) + if any( + not isinstance(name, str) or not name.strip() for pair in aliases.items() for name in pair + ): + raise ValueError("model_aliases must map non-empty recorded IDs to configured IDs") + return aliases + + +def _validate_models( + trials: Iterable[Trial], target: str, model: str, aliases: Mapping[str, str] +) -> dict[str, object]: + recorded_models: set[str] = set() + known = unknown = 0 + for trial in trials: + if trial.model is None: + unknown += 1 + elif not isinstance(trial.model, str) or not trial.model.strip(): + raise ValueError(f"invalid recorded model for target {target!r}") + else: + known += 1 + recorded_models.add(trial.model) + resolved_models = {aliases.get(recorded, recorded) for recorded in recorded_models} + if len(resolved_models) > 1: + raise ValueError( + f"conflicting recorded models for target {target!r}: {sorted(resolved_models)!r}" + ) + if resolved_models and resolved_models != {model}: + raise ValueError( + f"recorded models {sorted(recorded_models)!r} do not match configured model " + f"{model!r} for target {target!r}; provide model_aliases only for equivalent IDs" + ) + return { + "configured_model": model, + "recorded_models": sorted(recorded_models), + "known_trials": known, + "unknown_trials": unknown, + "verified": known > 0 and unknown == 0, + } + + +def _result( + task: Task, + decision: Decision, + *, + price_call: PriceCall | None, + error: str | None = None, +) -> Result: + metadata = decision.outcome.metadata + calls = decision.calls + return Result( + task_id=task.task_id, + baselines=task.outcomes, + target=decision.selected.target, + model=decision.selected.model, + outcome=task.outcomes.get(decision.selected.target) if error is None else None, + decision_id=metadata.outcome_id if metadata else None, + algorithm=metadata.algorithm if metadata else None, + evidence=metadata.evidence if metadata else None, + fallbacks=tuple(target.target for target in decision.fallbacks), + routing_seconds=decision.duration_seconds, + routing_calls=len(calls), + routing_failed_calls=sum(not call.is_success for call in calls), + routing_cost_usd=_cost(calls, price_call), + routing_usage=_usage(calls), + error=error, + ) + + +def _cost(calls: Sequence[RoutingCall], price_call: PriceCall | None) -> float | None: + if not calls: + return 0.0 + if price_call is None: + return None + values = [price_call(call) for call in calls] + for value in values: + if value is not None and ( + isinstance(value, bool) + or not isinstance(value, (int, float)) + or not math.isfinite(value) + or value < 0 + ): + raise ValueError("price_call must return a finite non-negative cost or None") + return ( + math.fsum(value for value in values if value is not None) + if all(value is not None for value in values) + else None + ) + + +def _usage(calls: Sequence[RoutingCall]) -> dict[str, int | None]: + result: dict[str, int | None] = {} + usages = [call.usage for call in calls] + for field in _TOKEN_FIELDS: + values = [usage.get(field) if usage is not None else None for usage in usages] + result[field] = ( + sum(value for value in values if value is not None) + if all(value is not None for value in values) + else None + ) + return result + + +async def evaluate( + dataset: Dataset, + runner: Runner, + *, + route: str, + concurrency: int = 8, + timeout: float = 60.0, + on_result: Callable[[Result], None] | None = None, + price_call: PriceCall | None = None, + model_aliases: Mapping[str, str] | None = None, +) -> Report: + """Evaluate a validated cohort with bounded work and progressive results. + + Use a fresh Runner for each independent experiment. The runner owns native + algorithm state and retries. Each task gets a distinct session ID. Use + ``concurrency=1`` for reproducible task ordering with a seeded random router. + Known recorded model IDs must match configured targets. ``model_aliases`` + maps recorded IDs to configured IDs explicitly; provider prefixes are never + guessed. Unknown model IDs remain visible in the report's input coverage. + + ``timeout`` covers the complete routing decision. Errors become result rows; + callback errors or cancellation stop and drain pending work. Keep callbacks + short. Cancelling local work cannot revoke provider requests already sent. + """ + if isinstance(concurrency, bool) or not isinstance(concurrency, int) or concurrency < 1: + raise ValueError("concurrency must be a positive integer") + if isinstance(timeout, bool) or not math.isfinite(timeout) or timeout <= 0: + raise ValueError("timeout must be finite and positive") + if not dataset.tasks or len({task.task_id for task in dataset.tasks}) != len(dataset.tasks): + raise ValueError("dataset must contain non-empty, unique tasks") + targets = runner.validate_decision_route(route, allow_response=False) + configured = {target.target for target in targets} + if not configured or not configured.issubset(dataset.targets): + raise ValueError( + f"configured targets {sorted(configured)!r} do not match available recorded targets {sorted(dataset.targets)!r}" + ) + aliases = _aliases(model_aliases) + model_validation = { + target.target: _validate_models( + (trial for task in dataset.tasks for trial in task.trials[target.target]), + target.target, + target.model, + aliases, + ) + for target in targets + } + run_id = uuid.uuid4().hex + report = Report( + len(dataset.tasks), + dataset.targets, + run_id=run_id, + concurrency=concurrency, + coverage={**dataset.coverage, "model_validation": model_validation}, + ) + + async def route_task(index: int, task: Task) -> Result: + started = time.monotonic() + try: + decision = await asyncio.wait_for( + runner.decide( + {"model": route, "messages": copy.deepcopy(list(task.messages))}, + headers={"x-switchyard-session-id": f"sim-{run_id}-{index}"}, + allow_response=False, + ), + timeout=timeout, + ) + except Exception as error: + # Native decision errors expose completed observations without provider + # bodies. Other errors (including timeouts) cannot prove call counts. + calls = getattr(error, "calls", None) + return Result( + task.task_id, + task.outcomes, + routing_seconds=time.monotonic() - started, + routing_calls=len(calls) if calls is not None else None, + routing_failed_calls=sum(not call.is_success for call in calls) + if calls is not None + else None, + routing_cost_usd=_cost(calls, price_call) if calls is not None else None, + routing_usage=_usage(calls) if calls is not None else {}, + error=f"routing failed: {type(error).__name__}", + ) + score_error = None + try: + _validate_selection(task, decision, aliases) + except ValueError as error: + score_error = str(error) + return _result(task, decision, price_call=price_call, error=score_error) + + pending: set[asyncio.Task[Result]] = set() + done: set[asyncio.Task[Result]] = set() + remaining = iter(enumerate(dataset.tasks)) + + def fill() -> None: + while len(pending) < concurrency: + item = next(remaining, None) + if item is None: + break + pending.add(asyncio.create_task(route_task(*item))) + + try: + fill() + while pending: + done, pending = await asyncio.wait(pending, return_when=asyncio.FIRST_COMPLETED) + # Consume every completed task before refilling. All callbacks run on + # this coordinator, so sinks need no locks. + for future in done: + result = future.result() + report.add(result) + if on_result is not None: + on_result(result) + fill() + finally: + for future in pending: + future.cancel() + if pending or done: + await asyncio.gather(*pending, *done, return_exceptions=True) + return report diff --git a/switchyard/sim/harbor.py b/switchyard/sim/harbor.py new file mode 100644 index 000000000..8d934cc8b --- /dev/null +++ b/switchyard/sim/harbor.py @@ -0,0 +1,271 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Read Harbor outcomes and the ATIF input visible before the first agent step.""" + +from __future__ import annotations + +import json +from collections.abc import Mapping +from datetime import datetime +from pathlib import Path +from typing import Any, Literal + +from .models import HarborRun, LoadIssue, Trial + + +def load_harbor( + path: str | Path, + *, + target: str, + dataset: str | None = None, + reward_key: str = "reward", + on_error: Literal["raise", "record"] = "raise", + task_inputs: Mapping[str, str] | None = None, + cost_source: Literal["result", "trajectory"] = "result", +) -> HarborRun: + """Load a trial, a Harbor job, or a downloaded run containing ``jobs/``. + + ``target`` names the recorded model/agent configuration. An explicit dataset + namespaces task names as ``dataset/task_name``; temporary task paths are never + used as identities. Repeats remain separate trials. + + Only system/user messages before the first ATIF agent step become routing + input. Missing trajectories or inputs require an explicit ``task_inputs`` + fallback keyed by task ID or task name. Raw session logs are not parsed. + + Reported cost comes from the preferred ``cost_source``, falling back to the + other source when absent. These values can differ because the producers use + different prices. No token pricing is inferred. Duration measures only the + agent execution phase. Missing measurements remain unknown. + + By default any invalid trial raises. ``on_error='record'`` keeps rejected + trial paths and reasons in ``issues`` so callers can account for exclusions. + Only one full trajectory is held in memory at a time. + """ + if not isinstance(target, str) or not target.strip(): + raise ValueError("target must be a non-empty string") + if dataset is not None and (not isinstance(dataset, str) or not dataset.strip()): + raise ValueError("dataset must be a non-empty string") + if not isinstance(reward_key, str) or not reward_key: + raise ValueError("reward_key must be a non-empty string") + if on_error not in ("raise", "record"): + raise ValueError("on_error must be 'raise' or 'record'") + if cost_source not in ("result", "trajectory"): + raise ValueError("cost_source must be 'result' or 'trajectory'") + + root = Path(path) + if not root.is_dir(): + raise FileNotFoundError(f"Harbor directory does not exist: {root}") + paths = _trial_paths(root) + if not paths: + raise ValueError(f"No Harbor trials found under {root}") + + trials: list[Trial] = [] + issues: list[LoadIssue] = [] + trial_ids: set[str] = set() + for result_path in paths: + task_id = None + try: + result = _read_object(result_path) + name = _string(result.get("task_name"), "task_name") + task_id = f"{dataset}/{name}" if dataset else name + trial = _trial( + result_path, result, task_id, target, reward_key, task_inputs, cost_source + ) + if trial.trial_id in trial_ids: + raise ValueError("duplicate trial ID") + trial_ids.add(trial.trial_id) + trials.append(trial) + except (OSError, ValueError) as error: + message = str(error) if isinstance(error, ValueError) else "cannot read trial artifact" + if on_error == "raise": + raise ValueError(f"{result_path}: {message}") from None + issues.append(LoadIssue(source=str(result_path), message=message, task_id=task_id)) + return HarborRun(tuple(trials), tuple(issues)) + + +def _trial_paths(root: Path) -> list[Path]: + """Inspect known Harbor layouts, excluding nested copies of trial metadata.""" + result_path = root / "result.json" + if (root / "agent").is_dir(): + return [result_path] + if result_path.is_file(): + try: + if "task_name" in _read_object(result_path): + return [result_path] + except ValueError: + return [result_path] + jobs = root / "jobs" + roots = sorted(p for p in jobs.iterdir() if p.is_dir()) if jobs.is_dir() else [root] + return [ + trial / "result.json" + for job in roots + for trial in sorted(job.iterdir()) + if trial.is_dir() + and ( + (trial / "result.json").is_file() + or (trial / "agent").is_dir() + or (trial / "config.json").is_file() + ) + ] + + +def _read_object(path: Path) -> dict[str, Any]: + try: + with path.open(encoding="utf-8") as stream: + value = json.load(stream) + except (json.JSONDecodeError, UnicodeError): + raise ValueError(f"invalid JSON in {path.name}") from None + return _object(value, path.name) + + +def _object(value: Any, field: str) -> dict[str, Any]: + if not isinstance(value, dict): + raise ValueError(f"{field} must be an object") + return value + + +def _optional_object(value: Any, field: str) -> dict[str, Any]: + return {} if value is None else _object(value, field) + + +def _string(value: Any, field: str) -> str: + if not isinstance(value, str) or not value.strip(): + raise ValueError(f"{field} must be a non-empty string") + return value + + +def _trial( + path: Path, + result: dict[str, Any], + task_id: str, + target: str, + reward_key: str, + task_inputs: Mapping[str, str] | None, + cost_source: str, +) -> Trial: + trajectory_path = path.parent / "agent" / "trajectory.json" + trajectory = _read_object(trajectory_path) if trajectory_path.is_file() else {} + fallback = None + if task_inputs is not None: + fallback = task_inputs.get(task_id, task_inputs.get(result["task_name"])) + messages = _messages(trajectory, fallback) + agent_result = _optional_object(result.get("agent_result"), "agent_result") + metrics = _optional_object(trajectory.get("final_metrics"), "final_metrics") + verifier = _optional_object(result.get("verifier_result"), "verifier_result") + rewards = _optional_object(verifier.get("rewards"), "rewards") + agent = _optional_object(result.get("agent_info"), "agent_info") + model = _optional_object(agent.get("model_info"), "model_info") + error = _optional_object(result.get("exception_info"), "exception_info") + checksum = result.get("task_checksum") + if checksum is not None: + checksum = _string(checksum, "task_checksum") + + costs = { + "result": agent_result.get("cost_usd"), + "trajectory": metrics.get("total_cost_usd"), + } + cost = costs[cost_source] + if cost is None: + cost_source = "trajectory" if cost_source == "result" else "result" + cost = costs[cost_source] + + usage: dict[str, int | None] = {} + for source, keys in ( + (agent_result, ("n_input_tokens", "n_output_tokens", "n_cache_tokens")), + (metrics, ("total_prompt_tokens", "total_completion_tokens", "total_cached_tokens")), + ): + for key in keys: + if key not in source: + continue + value = source[key] + if value is not None and ( + isinstance(value, bool) or not isinstance(value, int) or value < 0 + ): + raise ValueError(f"{key} must be a non-negative integer or null") + usage[key] = value + + name = model.get("name") + provider = model.get("provider") + if name is not None: + name = _string(name, "model_info.name") + if provider: + name = f"{_string(provider, 'model_info.provider')}/{name}" + return Trial( + task_id=task_id, + trial_id=_string( + result["id"] if result.get("id") is not None else path.parent.name, "trial ID" + ), + target=target, + messages=messages, + reward=rewards.get(reward_key), + cost_usd=cost, + cost_source=cost_source if cost is not None else None, + duration_seconds=_duration(result), + task_checksum=checksum, + model=name, + source=str(path), + usage=usage, + error=_string(error["exception_type"], "exception_type") + if error.get("exception_type") + else None, + ) + + +def _messages(trajectory: dict[str, Any], fallback: str | None) -> tuple[dict[str, object], ...]: + messages: list[dict[str, object]] = [] + has_user_input = False + if trajectory: + version = trajectory.get("schema_version") + if version not in {f"ATIF-v1.{minor}" for minor in range(8)}: + raise ValueError("unsupported or missing ATIF schema_version") + steps = trajectory.get("steps") + if not isinstance(steps, list): + raise ValueError("ATIF steps must be an array") + for raw_step in steps: + step = _object(raw_step, "ATIF step") + role = step.get("source") + if role == "agent": + break + if role not in ("system", "user"): + raise ValueError("unsupported ATIF input source") + content = step.get("message") + if isinstance(content, str): + content = [{"type": "text", "text": content}] + elif isinstance(content, list): + blocks = [] + for raw_block in content: + block = _object(raw_block, "ATIF message content") + if block.get("type") != "text" or not isinstance(block.get("text"), str): + raise ValueError("only text ATIF input is supported") + blocks.append({"type": "text", "text": block["text"]}) + content = blocks + else: + raise ValueError("ATIF input message must contain text") + if content: + messages.append({"role": role, "content": content}) + has_user_input |= role == "user" and any(block["text"].strip() for block in content) + if has_user_input: + return tuple(messages) + if fallback is not None: + text = _string(fallback, "task_inputs value") + return ({"role": "user", "content": [{"type": "text", "text": text}]},) + raise ValueError("missing initial task input; provide task_inputs explicitly") + + +def _duration(result: dict[str, Any]) -> float | None: + timing = _optional_object(result.get("agent_execution"), "agent_execution") + start, finish = timing.get("started_at"), timing.get("finished_at") + if start is None or finish is None: + return None + if not isinstance(start, str) or not isinstance(finish, str): + raise ValueError("agent_execution timestamps must be strings") + try: + # Python 3.10's fromisoformat does not accept the UTC Z suffix. + return ( + datetime.fromisoformat(finish.replace("Z", "+00:00")) + - datetime.fromisoformat(start.replace("Z", "+00:00")) + ).total_seconds() + except (ValueError, TypeError): + raise ValueError("invalid or incompatible agent_execution timestamps") from None diff --git a/switchyard/sim/models.py b/switchyard/sim/models.py new file mode 100644 index 000000000..b62f7945d --- /dev/null +++ b/switchyard/sim/models.py @@ -0,0 +1,135 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Recorded task outcomes and the results of evaluating routing decisions.""" + +from __future__ import annotations + +import math +from collections.abc import Mapping +from dataclasses import dataclass, field + + +@dataclass(frozen=True) +class Trial: + """One recorded attempt. Messages contain only the input before agent execution. + + A target identifies a complete model/agent configuration, not just a model name. + Unknown measurements stay ``None``. Usage preserves source counter names. + """ + + task_id: str + trial_id: str + target: str + messages: tuple[dict[str, object], ...] + reward: float | None = None + cost_usd: float | None = None + cost_source: str | None = None + duration_seconds: float | None = None + task_checksum: str | None = None + model: str | None = None + source: str | None = None + usage: Mapping[str, int | None] = field(default_factory=dict) + error: str | None = None + + def __post_init__(self) -> None: + for name in ("task_id", "trial_id", "target"): + if not isinstance(getattr(self, name), str) or not getattr(self, name).strip(): + raise ValueError(f"{name} must be a non-empty string") + for name in ("reward", "cost_usd", "duration_seconds"): + value = getattr(self, name) + try: + finite = value is None or math.isfinite(value) + except (TypeError, OverflowError): + finite = False + if value is not None and ( + isinstance(value, bool) + or not isinstance(value, (int, float)) + or not finite + or (name != "reward" and value < 0) + ): + raise ValueError( + f"{name} must be finite" + (" and non-negative" if name != "reward" else "") + ) + + +@dataclass(frozen=True) +class LoadIssue: + """An input problem retained alongside successfully loaded trials.""" + + source: str + message: str + task_id: str | None = None + + +@dataclass(frozen=True) +class HarborRun: + """Imported trials and every rejected input, without retaining full trajectories.""" + + trials: tuple[Trial, ...] + issues: tuple[LoadIssue, ...] = () + + +@dataclass(frozen=True) +class Outcome: + """Mean recorded measurements for one task and target. + + Each mean is unavailable unless every repeat reports that measurement. + Counts distinguish missing measurements from observed zero values. + """ + + trials: int + reward: float | None + cost_usd: float | None + duration_seconds: float | None + reward_trials: int + cost_trials: int + duration_trials: int + + @classmethod + def from_trials(cls, trials: tuple[Trial, ...]) -> Outcome: + if not trials: + raise ValueError("an outcome requires at least one trial") + + def measurement(name: str) -> tuple[float | None, int]: + values = [getattr(trial, name) for trial in trials if getattr(trial, name) is not None] + return ( + math.fsum(values) / len(trials) if len(values) == len(trials) else None, + len(values), + ) + + reward, reward_count = measurement("reward") + cost, cost_count = measurement("cost_usd") + duration, duration_count = measurement("duration_seconds") + return cls(len(trials), reward, cost, duration, reward_count, cost_count, duration_count) + + +@dataclass(frozen=True) +class Task: + """One routing input with recorded outcomes for each candidate target.""" + + task_id: str + messages: tuple[dict[str, object], ...] + outcomes: Mapping[str, Outcome] + trials: Mapping[str, tuple[Trial, ...]] + + +@dataclass(frozen=True) +class Result: + """One scored decision or visible evaluation error; safe to stream to a sink.""" + + task_id: str + baselines: Mapping[str, Outcome] + target: str | None = None + model: str | None = None + outcome: Outcome | None = None + decision_id: str | None = None + algorithm: str | None = None + evidence: object = None + fallbacks: tuple[str, ...] = () + routing_seconds: float | None = None + routing_calls: int | None = None + routing_failed_calls: int | None = None + routing_cost_usd: float | None = None + routing_usage: Mapping[str, int | None] = field(default_factory=dict) + error: str | None = None diff --git a/switchyard/sim/report.py b/switchyard/sim/report.py new file mode 100644 index 000000000..064479124 --- /dev/null +++ b/switchyard/sim/report.py @@ -0,0 +1,277 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Bounded, task-weighted summaries of recorded routing evaluations.""" + +from __future__ import annotations + +import math +from collections import Counter +from collections.abc import Mapping +from copy import deepcopy +from dataclasses import dataclass + +from .models import Result + + +@dataclass +class _Measurement: + known: int = 0 + total: float = 0.0 + + def add(self, value: float | None) -> None: + if value is not None: + self.known += 1 + self.total += value + + def summary(self, count: int) -> dict[str, int | float | None]: + complete = count > 0 and self.known == count + return { + "known_tasks": self.known, + "observed_total": self.total if self.known else None, + "total": self.total if complete else None, + "mean": self.total / count if complete else None, + } + + +def _validate_measurement(value: float | None, name: str, *, nonnegative: bool = True) -> None: + if value is not None and ( + isinstance(value, bool) + or not isinstance(value, (int, float)) + or not math.isfinite(value) + or (nonnegative and value < 0) + ): + raise ValueError(f"invalid {name}: expected a finite measurement or None") + + +class Report: + """Accumulate one result per task without keeping decisions or trajectories. + + Recorded measurements already average repeats within a task and target. Tasks + have equal weight here. Unknown measurements invalidate the corresponding + total and mean; ``observed_total`` retains the known part. Reward and cost + comparisons each use successful decisions with that measurement known for + the selected and every fixed target. The empirical reward oracle takes the + best target mean for each task, after averaging repeats. It is a retrospective + comparison, not a deployable routing policy. + """ + + def __init__( + self, + expected: int, + targets: tuple[str, ...], + *, + run_id: str, + concurrency: int, + coverage: Mapping[str, object] | None = None, + ) -> None: + if isinstance(expected, bool) or not isinstance(expected, int) or expected < 0: + raise ValueError("expected must be a non-negative integer") + if isinstance(concurrency, bool) or not isinstance(concurrency, int) or concurrency < 1: + raise ValueError("concurrency must be a positive integer") + if not targets or any(not isinstance(target, str) or not target for target in targets): + raise ValueError("targets must contain non-empty names") + if len(set(targets)) != len(targets): + raise ValueError("targets must be unique") + self.expected = expected + self.targets = targets + self.run_id = run_id + self.concurrency = concurrency + self._coverage = deepcopy(dict(coverage)) if coverage is not None else None + self.processed = 0 + self.routed = 0 + self.scored = 0 + self.errors = 0 + self._seen: set[str] = set() + self._selected: Counter[str] = Counter(dict.fromkeys(targets, 0)) + self._recorded = { + name: _Measurement() for name in ("reward", "cost_usd", "duration_seconds") + } + self._routing_seconds = _Measurement() + self._routing_cost = _Measurement() + self._routing_calls = _Measurement() + self._routing_failed_calls = _Measurement() + self._zero_call_tasks = 0 + self._routing_usage: dict[str, _Measurement] = {} + self._estimated_cost = _Measurement() + self._comparison_tasks = 0 + self._comparison_routed = 0.0 + self._comparison_targets = dict.fromkeys(targets, 0.0) + self._comparison_best = 0.0 + self._cost_comparison_tasks = 0 + self._cost_comparison_routed = 0.0 + self._cost_comparison_targets = dict.fromkeys(targets, 0.0) + self._cost_comparison_routing = _Measurement() + + @property + def complete(self) -> bool: + return ( + self.expected > 0 + and self.processed == self.expected + and self.scored == self.expected + and not self.errors + ) + + def add(self, result: Result) -> None: + """Add one result, rejecting duplicate IDs and invalid measurements.""" + if result.task_id in self._seen: + raise ValueError(f"duplicate result for task {result.task_id!r}") + if self.processed >= self.expected: + raise ValueError("more results than expected tasks") + if result.error is None and result.target is not None and result.target not in self.targets: + raise ValueError("successful result target must be one of the configured targets") + for name in ("routing_calls", "routing_failed_calls"): + count = getattr(result, name) + if count is not None and ( + isinstance(count, bool) or not isinstance(count, int) or count < 0 + ): + raise ValueError(f"{name} must be a non-negative integer or None") + if ( + result.routing_failed_calls is not None + and result.routing_calls is not None + and result.routing_failed_calls > result.routing_calls + ): + raise ValueError("routing_failed_calls cannot exceed routing_calls") + _validate_measurement(result.routing_seconds, "routing_seconds") + _validate_measurement(result.routing_cost_usd, "routing_cost_usd") + for name, value in result.routing_usage.items(): + _validate_measurement(value, name) + outcomes = [*result.baselines.values()] + if result.outcome is not None: + outcomes.append(result.outcome) + for recorded in outcomes: + _validate_measurement(recorded.reward, "reward", nonnegative=False) + _validate_measurement(recorded.cost_usd, "cost_usd") + _validate_measurement(recorded.duration_seconds, "duration_seconds") + + self._seen.add(result.task_id) + self.processed += 1 + self.errors += int(result.error is not None) + if result.target is not None: + self.routed += 1 + self._selected[result.target] += 1 + outcome = result.outcome if result.error is None and result.target is not None else None + self.scored += int(outcome is not None and outcome.reward is not None) + for name, measurement in self._recorded.items(): + measurement.add(getattr(outcome, name) if outcome is not None else None) + self._routing_seconds.add(result.routing_seconds) + self._routing_cost.add(result.routing_cost_usd) + self._routing_calls.add(result.routing_calls) + self._routing_failed_calls.add(result.routing_failed_calls) + for name in self._routing_usage.keys() | result.routing_usage.keys(): + if name not in self._routing_usage: + self._routing_usage[name] = _Measurement(known=self._zero_call_tasks) + value = result.routing_usage.get(name) + self._routing_usage[name].add( + 0 if value is None and result.routing_calls == 0 else value + ) + self._zero_call_tasks += int(result.routing_calls == 0) + if ( + outcome is not None + and outcome.cost_usd is not None + and result.routing_cost_usd is not None + ): + self._estimated_cost.add(outcome.cost_usd + result.routing_cost_usd) + + if outcome is not None and outcome.reward is not None: + rewards = [ + result.baselines[target].reward if target in result.baselines else None + for target in self.targets + ] + if all(reward is not None for reward in rewards): + known_rewards = [reward for reward in rewards if reward is not None] + self._comparison_tasks += 1 + self._comparison_routed += outcome.reward + for target, reward in zip(self.targets, known_rewards, strict=True): + self._comparison_targets[target] += reward + self._comparison_best += max(known_rewards) + + if outcome is not None and outcome.cost_usd is not None: + costs = [ + result.baselines[target].cost_usd if target in result.baselines else None + for target in self.targets + ] + if all(cost is not None for cost in costs): + self._cost_comparison_tasks += 1 + self._cost_comparison_routed += outcome.cost_usd + for target, cost in zip(self.targets, costs, strict=True): + assert cost is not None + self._cost_comparison_targets[target] += cost + self._cost_comparison_routing.add(result.routing_cost_usd) + + def to_dict(self) -> dict[str, object]: + """Return an independent JSON-compatible snapshot with explicit coverage.""" + count = self._comparison_tasks + cost_count = self._cost_comparison_tasks + routed_cost = self._cost_comparison_routed / cost_count if cost_count else None + routing_cost = self._cost_comparison_routing.summary(cost_count)["mean"] + estimated = self._estimated_cost.summary(self.processed) + if self.processed != self.expected: + estimated["total"] = None + return { + "schema_version": 1, + "run_id": self.run_id, + "concurrency": self.concurrency, + "coverage": deepcopy(self._coverage), + "complete": self.complete, + "counts": { + "total": self.expected, + "processed": self.processed, + "pending": self.expected - self.processed, + "routed": self.routed, + "scored": self.scored, + "errors": self.errors, + "unscored": self.processed - self.scored, + }, + "targets": dict(self._selected), + "recorded": { + name: value.summary(self.processed) for name, value in self._recorded.items() + }, + "routing": { + "seconds": self._routing_seconds.summary(self.processed), + "calls": self._routing_calls.summary(self.processed)["total"], + "known_call_tasks": self._routing_calls.known, + "observed_calls": self._routing_calls.total, + "failed_calls": self._routing_failed_calls.summary(self.processed)["total"], + "known_failed_call_tasks": self._routing_failed_calls.known, + "observed_failed_calls": self._routing_failed_calls.total, + "cost_usd": self._routing_cost.summary(self.processed), + "usage": { + name: value.summary(self.processed) + for name, value in sorted(self._routing_usage.items()) + }, + }, + "estimated_cost_usd": estimated, + "comparison": { + "tasks": count, + "routed_mean_reward": self._comparison_routed / count if count else None, + "targets": { + target: total / count if count else None + for target, total in self._comparison_targets.items() + }, + "empirical_oracle_mean_reward": self._comparison_best / count if count else None, + }, + "cost_comparison": { + "tasks": cost_count, + "routed_mean_cost_usd": routed_cost, + "targets": { + target: total / cost_count if cost_count else None + for target, total in self._cost_comparison_targets.items() + }, + "routing_mean_cost_usd": routing_cost, + "routed_mean_cost_with_routing_usd": routed_cost + routing_cost + if routed_cost is not None and routing_cost is not None + else None, + }, + } + + def format_text(self) -> str: + """Render coverage and the currently available headline measurements.""" + reward = self._recorded["reward"].summary(self.processed)["mean"] + cost = self._estimated_cost.summary(self.expected)["total"] + reward_text = f"{reward:.4f}" if reward is not None else "unknown" + cost_text = f"${cost:.4f}" if cost is not None else "unknown" + return ( + f"{self.processed}/{self.expected} tasks; {self.scored} scored; {self.errors} errors; " + f"mean reward {reward_text}; estimated cost {cost_text}" + ) diff --git a/switchyard_rust/runner.py b/switchyard_rust/runner.py new file mode 100644 index 000000000..38f1c12d2 --- /dev/null +++ b/switchyard_rust/runner.py @@ -0,0 +1,116 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Native configured routing decisions without a local server.""" + +from __future__ import annotations + +from collections.abc import Mapping +from os import PathLike +from typing import TYPE_CHECKING, Any, final + +from switchyard_rust._native import load_native + +_EXPORTS = frozenset({"Decision", "DecisionError", "DecisionTarget", "RoutingCall", "Runner"}) + +if TYPE_CHECKING: + from switchyard_rust.libsy import RoutingOutcome + + class DecisionError(RuntimeError): + """Routing failure with safe diagnostics and completed-call observations.""" + + kind: str + upstream_status: int | None + target: str | None + calls: list[RoutingCall] + duration_seconds: float + + @final + class DecisionTarget: + @property + def target(self) -> str: ... + + @property + def model(self) -> str: ... + + @final + class RoutingCall: + """A logical provider call; duration includes backend retries. + + Usage is the native normalized protocol mapping: ``input_tokens`` excludes + cache reads/writes. Output and reasoning counts retain the native codec's + semantics; reasoning can be a subset of output and must not be added blindly. + Missing usage or token fields are unknown, not zero. Failed attempts and + streamed routing calls may not report usage. + """ + + @property + def model(self) -> str: ... + + @property + def is_success(self) -> bool: ... + + @property + def duration_seconds(self) -> float: ... + + @property + def usage(self) -> dict[str, int | None] | None: ... + + @final + class Decision: + @property + def selected(self) -> DecisionTarget: ... + + @property + def fallbacks(self) -> list[DecisionTarget]: ... + + @property + def outcome(self) -> RoutingOutcome: ... + + @property + def calls(self) -> list[RoutingCall]: ... + + @property + def duration_seconds(self) -> float: ... + + @final + class Runner: + """Native route configuration, clients, and shared algorithm state. + + Give independent tasks distinct session headers. Process turns sharing + a session in order. Use a separate runner for independent experiments. + Cancelling a decision cancels local routing; a provider may still finish + or bill an already submitted request. + """ + + @staticmethod + def load(path: str | PathLike[str]) -> Runner: ... + + @staticmethod + def from_toml(source: str) -> Runner: ... + + def validate_decision_route( + self, model: str, *, allow_response: bool = False + ) -> list[DecisionTarget]: + """Validate without calls and return the configured completion targets.""" + ... + + async def decide( + self, + request: Mapping[str, object], + *, + headers: Mapping[str, str] | None = None, + allow_response: bool = False, + ) -> Decision: + """Route normalized IR; response-based algorithms need explicit opt-in.""" + ... + + +def __getattr__(name: str) -> object: + if name in _EXPORTS: + native: Any = load_native() + return getattr(native.runner, name) + raise AttributeError(f"module {__name__!r} has no attribute {name!r}") + + +__all__ = sorted(_EXPORTS) diff --git a/tests/sim/test_dataset.py b/tests/sim/test_dataset.py new file mode 100644 index 000000000..456eacb28 --- /dev/null +++ b/tests/sim/test_dataset.py @@ -0,0 +1,168 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Cohort and repeat validation for recorded routing evaluation.""" + +from dataclasses import replace + +import pytest + +from switchyard.sim.dataset import Dataset +from switchyard.sim.models import HarborRun, LoadIssue, Outcome, Trial + + +def trial(target="fast", task="task", attempt="1", **kwargs): + return Trial( + task, + attempt, + target, + ({"role": "user", "content": [{"type": "text", "text": task}]},), + **kwargs, + ) + + +def test_repeats_are_task_means_and_never_arbitrarily_paired(): + dataset = Dataset.from_runs( + { + "fast": HarborRun( + (trial(reward=1, cost_usd=2), trial(attempt="2", reward=0, cost_usd=4)) + ), + "strong": HarborRun( + (trial("strong", attempt="unrelated-id", reward=0.8, cost_usd=10),) + ), + }, + input_target="fast", + ) + task = dataset.tasks[0] + assert task.outcomes["fast"].reward == 0.5 + assert task.outcomes["fast"].cost_usd == 3 + assert task.outcomes["strong"].reward == 0.8 + assert task.outcomes["fast"].trials == 2 + + +def test_incomplete_repeat_measurements_stay_unknown(): + outcome = Outcome.from_trials((trial(reward=0, cost_usd=0), trial(attempt="2", reward=None))) + assert outcome.reward is None + assert outcome.cost_usd is None + assert outcome.reward_trials == outcome.cost_trials == 1 + assert outcome.trials == 2 + + +def test_missing_target_coverage_requires_explicit_intersection(): + runs = { + "fast": HarborRun((trial(), trial(task="extra"))), + "strong": HarborRun((trial("strong"),)), + } + with pytest.raises(ValueError, match="incomplete cohort"): + Dataset.from_runs(runs, input_target="fast") + dataset = Dataset.from_runs(runs, input_target="fast", intersection=True) + assert len(dataset.tasks) == 1 + assert dataset.coverage["tasks_seen"] == 2 + assert dataset.coverage["excluded_task_ids"] == ["extra"] + + +def test_invalid_repeat_excludes_whole_task_instead_of_improving_its_mean(): + runs = { + "fast": HarborRun( + (trial(), trial(task="safe")), (LoadIssue("broken", "bad input", "task"),) + ) + } + dataset = Dataset.from_runs(runs, input_target="fast", intersection=True) + assert [task.task_id for task in dataset.tasks] == ["safe"] + assert dataset.coverage["excluded_task_ids"] == ["task"] + assert len(dataset.coverage["input_issues"]) == 1 + + +def test_unidentified_rejected_repeat_cannot_leave_a_favorable_mean(): + run = HarborRun((trial(reward=1),), (LoadIssue("missing/result.json", "cannot read"),)) + with pytest.raises(ValueError, match="no task identity"): + Dataset.from_runs({"fast": run}, input_target="fast", intersection=True) + + +def test_cost_source_mixture_is_visible_in_coverage(): + run = HarborRun( + ( + trial(cost_usd=1, cost_source="result"), + trial(task="b", cost_usd=2, cost_source="trajectory"), + trial(task="c"), + ) + ) + cohort = Dataset.from_runs({"fast": run}, input_target="fast") + assert cohort.coverage["cost_sources_by_target"] == { + "fast": {"result": 1, "trajectory": 1, "unknown": 1} + } + + +def test_huge_json_numbers_are_rejected_as_invalid_measurements(): + with pytest.raises(ValueError, match="reward must be finite"): + trial(reward=10**400) + + +def test_checksums_reject_changed_tasks_even_when_names_match(): + with pytest.raises(ValueError, match="conflicting task checksums"): + Dataset.from_runs( + { + "a": HarborRun((trial("a", task_checksum="old"),)), + "b": HarborRun((trial("b", task_checksum="new"),)), + }, + input_target="a", + ) + + +def test_agent_wrappers_may_differ_and_input_target_is_explicit(): + fast = trial(task_checksum="same") + strong = replace( + trial("strong", task_checksum="same"), + messages=( + {"role": "system", "content": [{"type": "text", "text": "different agent"}]}, + *fast.messages, + ), + ) + dataset = Dataset.from_runs( + {"fast": HarborRun((fast,)), "strong": HarborRun((strong,))}, input_target="strong" + ) + assert dataset.tasks[0].messages == strong.messages + + +def test_missing_checksums_require_matching_task_instructions(): + other = replace(trial("strong"), messages=trial(task="different").messages) + with pytest.raises(ValueError, match="conflicting inputs"): + Dataset.from_runs( + {"fast": HarborRun((trial(),)), "strong": HarborRun((other,))}, input_target="fast" + ) + + +def test_missing_checksums_compare_all_initial_user_constraints(): + first = trial() + second = replace( + trial("strong"), + messages=( + {"role": "user", "content": [{"type": "text", "text": "additional constraint"}]}, + *first.messages, + ), + ) + with pytest.raises(ValueError, match="conflicting inputs"): + Dataset.from_runs( + {"fast": HarborRun((first,)), "strong": HarborRun((second,))}, input_target="fast" + ) + + +def test_duplicate_trial_and_incorrect_target_are_rejected(): + with pytest.raises(ValueError, match="duplicate trial"): + Dataset.from_runs({"fast": HarborRun((trial(), trial()))}, input_target="fast") + with pytest.raises(ValueError, match="does not match"): + Dataset.from_runs({"strong": HarborRun((trial(),))}, input_target="strong") + + +@pytest.mark.parametrize( + "kwargs", + [ + {"reward": float("nan")}, + {"cost_usd": -1}, + {"duration_seconds": float("inf")}, + {"reward": True}, + ], +) +def test_invalid_measurements_are_rejected(kwargs): + with pytest.raises(ValueError): + trial(**kwargs) diff --git a/tests/sim/test_evaluate.py b/tests/sim/test_evaluate.py new file mode 100644 index 000000000..55d88b9a8 --- /dev/null +++ b/tests/sim/test_evaluate.py @@ -0,0 +1,296 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Bounded scheduling, cleanup, and recorded-outcome validity.""" + +import asyncio +from dataclasses import replace +from types import SimpleNamespace + +import pytest + +from switchyard.sim.dataset import Dataset +from switchyard.sim.evaluate import evaluate, score +from switchyard.sim.models import HarborRun, Trial + + +def dataset(count=4): + trials = tuple( + Trial( + str(i), + str(i), + "fast", + ({"role": "user", "content": [{"type": "text", "text": f"input {i}"}]},), + reward=i / max(count, 1), + cost_usd=2, + ) + for i in range(count) + ) + return Dataset.from_runs({"fast": HarborRun(trials)}, input_target="fast") + + +class FakeRunner: + def __init__(self, delay=0.001): + self.active = self.peak = self.calls = self.cancelled = 0 + self.delay = delay + self.headers = [] + self.targets = [SimpleNamespace(target="fast", model="model-fast")] + + def validate_decision_route(self, route, *, allow_response): + assert route == "auto" and allow_response is False + return self.targets + + async def decide(self, request, *, headers, allow_response): + assert allow_response is False + self.headers.append(headers) + self.calls += 1 + self.active += 1 + self.peak = max(self.peak, self.active) + try: + await asyncio.sleep(self.delay) + return decision(request) + except asyncio.CancelledError: + self.cancelled += 1 + raise + finally: + self.active -= 1 + + +def decision(request): + return SimpleNamespace( + selected=SimpleNamespace(target="fast", model="model-fast"), + outcome=SimpleNamespace(request=request, response=None, metadata=None), + calls=[], + fallbacks=[], + duration_seconds=0.01, + ) + + +async def test_bounded_concurrency_and_progressive_results(): + runner = FakeRunner() + rows = [] + report = await evaluate(dataset(11), runner, route="auto", concurrency=3, on_result=rows.append) + assert runner.peak == 3 and runner.active == 0 + assert len(rows) == 11 + assert report.to_dict()["counts"]["scored"] == 11 + assert all(row.routing_cost_usd == 0 for row in rows) + assert len({headers["x-switchyard-session-id"] for headers in runner.headers}) == 11 + + +async def test_sessions_are_isolated_between_evaluations(): + runner = FakeRunner() + await evaluate(dataset(1), runner, route="auto") + await evaluate(dataset(1), runner, route="auto") + assert runner.headers[0] != runner.headers[1] + + +async def test_target_mapping_is_validated_before_any_calls(): + runner = FakeRunner() + runner.targets.append(SimpleNamespace(target="unknown")) + with pytest.raises(ValueError, match="do not match"): + await evaluate(dataset(), runner, route="auto") + assert runner.calls == 0 + + +async def test_timeout_is_visible_and_does_not_claim_zero_cost(): + runner = FakeRunner(delay=1) + rows = [] + report = await evaluate(dataset(2), runner, route="auto", timeout=0.01, on_result=rows.append) + assert report.to_dict()["counts"]["errors"] == 2 + assert runner.active == 0 and runner.cancelled == 2 + assert all(row.routing_cost_usd is None and row.routing_calls is None for row in rows) + + +async def test_callback_failure_cancels_pending_work(): + class SlowRunner(FakeRunner): + async def decide(self, request, **kwargs): + self.delay = 0 if request["messages"][0]["content"][0]["text"] == "input 0" else 10 + return await super().decide(request, **kwargs) + + runner = SlowRunner() + + def fail(_): + raise RuntimeError("sink failed") + + with pytest.raises(RuntimeError, match="sink failed"): + await evaluate(dataset(10), runner, route="auto", concurrency=3, on_result=fail) + assert runner.calls == 3 and runner.cancelled == 2 and runner.active == 0 + + +async def test_caller_cancellation_drains_all_workers(): + runner = FakeRunner(delay=10) + work = asyncio.create_task(evaluate(dataset(10), runner, route="auto", concurrency=2)) + while runner.active < 2: + await asyncio.sleep(0) + work.cancel() + with pytest.raises(asyncio.CancelledError): + await work + assert runner.cancelled == 2 and runner.active == 0 and runner.calls == 2 + + +@pytest.mark.parametrize("mutation", ["answer", "messages", "instructions", "tools", "target"]) +def test_scorer_rejects_unsupported_counterfactuals(mutation): + task = dataset(1).tasks[0] + native = decision({"messages": list(task.messages)}) + if mutation == "answer": + native.outcome.response = "new answer" + elif mutation == "target": + native.selected.target = "missing" + else: + native.outcome.request[mutation] = [{"text": "changed task"}] + with pytest.raises(ValueError): + score(task, native) + + +def test_routing_measurements_and_pricing_are_separate_from_task_cost(): + task = dataset(1).tasks[0] + native = decision({"messages": list(task.messages)}) + native.calls = [ + SimpleNamespace( + model="judge", is_success=True, usage={"input_tokens": 20, "output_tokens": 5} + ) + ] + row = score(task, native) + assert row.routing_calls == 1 and row.routing_usage["input_tokens"] == 20 + assert row.routing_usage["cached_input_tokens"] is None + assert row.routing_cost_usd is None and row.outcome.cost_usd == 2 + assert score(task, native, price_call=lambda call: 0.1).routing_cost_usd == 0.1 + + +async def test_failure_is_an_error_without_exposing_provider_body(): + class BrokenRunner(FakeRunner): + async def decide(self, *args, **kwargs): + raise RuntimeError("provider response containing a secret") + + rows = [] + report = await evaluate(dataset(1), BrokenRunner(), route="auto", on_result=rows.append) + assert report.to_dict()["counts"]["errors"] == 1 + assert "secret" not in rows[0].error + + +async def test_recorded_model_mismatch_fails_before_calls(): + data = dataset(1) + recorded = replace(data.tasks[0].trials["fast"][0], model="different-model") + data = Dataset.from_runs({"fast": HarborRun((recorded,))}, input_target="fast") + runner = FakeRunner() + with pytest.raises(ValueError, match="do not match configured model"): + await evaluate(data, runner, route="auto") + assert runner.calls == 0 + + +async def test_conflicting_recorded_models_require_explicit_equivalent_aliases(): + data = dataset(1) + first = replace(data.tasks[0].trials["fast"][0], model="model-fast") + second = replace(first, trial_id="repeat", model="provider/model-fast") + data = Dataset.from_runs({"fast": HarborRun((first, second))}, input_target="fast") + runner = FakeRunner() + with pytest.raises(ValueError, match="conflicting recorded models"): + await evaluate(data, runner, route="auto") + assert runner.calls == 0 + report = await evaluate( + data, runner, route="auto", model_aliases={"provider/model-fast": "model-fast"} + ) + validation = report.to_dict()["coverage"]["model_validation"]["fast"] + assert validation == { + "configured_model": "model-fast", + "recorded_models": ["model-fast", "provider/model-fast"], + "known_trials": 2, + "unknown_trials": 0, + "verified": True, + } + + +async def test_unknown_models_are_visible_and_fixed_route_accepts_extra_baselines(): + fast = dataset(1).tasks[0].trials["fast"][0] + strong = replace(fast, target="strong", model="other-model") + data = Dataset.from_runs( + {"fast": HarborRun((fast,)), "strong": HarborRun((strong,))}, input_target="fast" + ) + report = await evaluate(data, FakeRunner(), route="auto") + assert report.complete + validation = report.to_dict()["coverage"]["model_validation"] + assert validation["fast"]["unknown_trials"] == 1 + assert validation["fast"]["verified"] is False + assert report.to_dict()["comparison"]["targets"].keys() == {"fast", "strong"} + + +@pytest.mark.parametrize("mode", ["raises", "nan", "negative", "failed-routing"]) +async def test_pricing_errors_stop_and_drain_workers(mode): + call = SimpleNamespace(model="judge", is_success=True, usage={"input_tokens": 1}) + + class PaidRunner(FakeRunner): + async def decide(self, request, **kwargs): + self.delay = 0 if request["messages"][0]["content"][0]["text"] == "input 0" else 10 + native = await super().decide(request, **kwargs) + if mode == "failed-routing": + error = RuntimeError("provider failure") + error.calls = [call] + raise error + native.calls = [call] + return native + + def price(_): + if mode in {"raises", "failed-routing"}: + raise ValueError("pricing unavailable") + return float("nan") if mode == "nan" else -1 + + runner = PaidRunner() + with pytest.raises(ValueError, match="pricing unavailable|price_call must return"): + await evaluate(dataset(10), runner, route="auto", concurrency=3, price_call=price) + assert runner.calls == 3 and runner.cancelled == 2 and runner.active == 0 + + +async def test_unsupported_selection_retains_known_routing_cost(): + class RewritingRunner(FakeRunner): + async def decide(self, request, **kwargs): + native = await super().decide(request, **kwargs) + native.outcome.request["instructions"] = [{"content": "changed task"}] + native.calls = [SimpleNamespace(model="judge", is_success=True, usage={})] + return native + + rows = [] + report = await evaluate( + dataset(1), + RewritingRunner(), + route="auto", + price_call=lambda _: 0.25, + on_result=rows.append, + ) + assert report.errors == 1 + assert rows[0].outcome is None + assert rows[0].routing_cost_usd == 0.25 + + +def test_public_scorer_checks_recorded_models_and_accepts_explicit_aliases(): + recorded = replace(dataset(1).tasks[0].trials["fast"][0], model="provider/model-fast") + task = Dataset.from_runs({"fast": HarborRun((recorded,))}, input_target="fast").tasks[0] + native = decision({"messages": list(task.messages)}) + with pytest.raises(ValueError, match="do not match configured model"): + score(task, native) + row = score(task, native, model_aliases={"provider/model-fast": "model-fast"}) + assert row.error is None and row.outcome == task.outcomes["fast"] + native.selected.model = "unrelated-model" + with pytest.raises(ValueError, match="do not match configured model"): + score(task, native, model_aliases={"provider/model-fast": "model-fast"}) + + +def test_usage_crosses_native_boundary_once_per_call(): + class CountingCall: + model = "judge" + is_success = True + reads = 0 + + @property + def usage(self): + self.reads += 1 + return {"input_tokens": 3, "output_tokens": 1} + + task = dataset(1).tasks[0] + native = decision({"messages": list(task.messages)}) + calls = [CountingCall(), CountingCall()] + native.calls = calls + row = score(task, native) + assert row.routing_usage["input_tokens"] == 6 + assert row.routing_usage["output_tokens"] == 2 + assert row.routing_usage["cached_input_tokens"] is None + assert [call.reads for call in calls] == [1, 1] diff --git a/tests/sim/test_harbor.py b/tests/sim/test_harbor.py new file mode 100644 index 000000000..fc9a72427 --- /dev/null +++ b/tests/sim/test_harbor.py @@ -0,0 +1,305 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Sanitized Harbor fixtures covering native Claude and Codex ATIF layouts.""" + +import json +from pathlib import Path + +import pytest + +from switchyard.sim.harbor import load_harbor + + +def write_trial( + root: Path, + name: str = "trial-a", + *, + task: str = "benchmark/task-a", + version: str = "ATIF-v1.7", + codex: bool = False, +) -> Path: + trial = root / name + (trial / "agent").mkdir(parents=True) + result = { + "id": name, + "task_name": task, + "task_checksum": "checksum-a", + "task_id": {"path": "/tmp/ephemeral-task"}, + "agent_info": { + "name": "codex" if codex else "claude-code", + "version": "0.154.0" if codex else "2.1.216", + "model_info": {"name": "model-a", "provider": "provider"}, + }, + "agent_result": { + "n_input_tokens": 100, + "n_output_tokens": 20, + "n_cache_tokens": 80, + "cost_usd": 0.25, + }, + "verifier_result": {"rewards": {"reward": 0.75, "custom": 0.5}}, + "started_at": "2026-09-01T01:00:00Z", + "finished_at": "2026-09-01T02:00:00Z", + "agent_execution": { + "started_at": "2026-09-01T01:10:00Z", + "finished_at": "2026-09-01T01:10:12.500Z", + }, + } + steps = [] + if codex: + steps.extend( + [ + {"source": "system", "message": "Agent instructions."}, + {"source": "user", "message": "Working directory: /workspace"}, + ] + ) + steps.extend( + [ + {"source": "user", "message": "Implement the requested feature."}, + {"source": "agent", "message": "FUTURE_ASSISTANT_LEAK"}, + {"source": "user", "message": "FUTURE_USER_LEAK"}, + ] + ) + trajectory = { + "schema_version": version, + "steps": steps, + "final_metrics": { + "total_prompt_tokens": 100, + "total_completion_tokens": 20, + "total_cached_tokens": 80, + "total_cost_usd": 0.1, + }, + } + (trial / "result.json").write_text(json.dumps(result)) + (trial / "agent" / "trajectory.json").write_text(json.dumps(trajectory)) + return trial + + +def update(path: Path, **values: object) -> None: + row = json.loads(path.read_text()) + row.update(values) + path.write_text(json.dumps(row)) + + +@pytest.mark.parametrize( + "version,codex", [("ATIF-v1.7", False), ("ATIF-v1.7", True), ("ATIF-v1.5", True)] +) +def test_initial_messages_include_codex_context_without_future_leakage( + tmp_path: Path, version: str, codex: bool +) -> None: + write_trial(tmp_path, version=version, codex=codex) + run = load_harbor(tmp_path, target="baseline", dataset="suite") + trial = run.trials[0] + assert run.issues == () + assert trial.task_id == "suite/benchmark/task-a" + assert trial.task_checksum == "checksum-a" + assert [message["role"] for message in trial.messages] == ( + ["system", "user", "user"] if codex else ["user"] + ) + assert trial.messages[-1]["content"] == [ + {"type": "text", "text": "Implement the requested feature."} + ] + assert "FUTURE" not in json.dumps(trial.messages) + assert trial.reward == 0.75 + assert trial.cost_usd == 0.25 + assert trial.cost_source == "result" + assert trial.duration_seconds == 12.5 + assert trial.model == "provider/model-a" + assert trial.usage["total_prompt_tokens"] == 100 + assert trial.usage["n_cache_tokens"] == 80 + + +def test_discovery_ignores_nested_metadata_and_accepts_download_layout(tmp_path: Path) -> None: + job = tmp_path / "jobs" / "run-a" + trial = write_trial(job) + (trial / "_scaled_evals").mkdir() + (trial / "_scaled_evals" / "result.json").write_text((trial / "result.json").read_text()) + (job / "result.json").write_text(json.dumps({"n_total_trials": 1})) + for root in (tmp_path, job, trial): + assert len(load_harbor(root, target="baseline").trials) == 1 + + +def test_repeated_tasks_are_preserved_as_distinct_trials(tmp_path: Path) -> None: + write_trial(tmp_path, "repeat-1") + write_trial(tmp_path, "repeat-2") + trials = load_harbor(tmp_path, target="baseline").trials + assert len(trials) == 2 + assert trials[0].task_id == trials[1].task_id + assert trials[0].trial_id != trials[1].trial_id + + +def test_missing_measurements_remain_unknown_and_preserve_exception_type(tmp_path: Path) -> None: + trial = write_trial(tmp_path) + update( + trial / "result.json", + agent_result=None, + verifier_result=None, + agent_execution=None, + exception_info={ + "exception_type": "VerifierTimeoutError", + "exception_message": "PRIVATE_TRACE", + }, + ) + update(trial / "agent" / "trajectory.json", final_metrics=None) + row = load_harbor(tmp_path, target="baseline").trials[0] + assert row.reward is None + assert row.cost_usd is None + assert row.cost_source is None + assert row.duration_seconds is None + assert row.usage == {} + assert row.error == "VerifierTimeoutError" + assert "PRIVATE_TRACE" not in repr(row) + + +def test_explicit_zero_measurements_are_not_replaced_by_fallbacks(tmp_path: Path) -> None: + trial = write_trial(tmp_path) + update( + trial / "result.json", + agent_result={"cost_usd": 0}, + verifier_result={"rewards": {"reward": 0}}, + ) + row = load_harbor(tmp_path, target="baseline").trials[0] + assert row.reward == 0 + assert row.cost_usd == 0 + assert row.cost_source == "result" + + +def test_reward_key_and_cost_preference_are_explicit(tmp_path: Path) -> None: + write_trial(tmp_path) + row = load_harbor( + tmp_path, target="baseline", reward_key="custom", cost_source="trajectory" + ).trials[0] + assert row.reward == 0.5 + assert row.cost_usd == 0.1 + assert row.cost_source == "trajectory" + + +def test_absent_preferred_cost_falls_back_with_provenance(tmp_path: Path) -> None: + trial = write_trial(tmp_path) + update(trial / "result.json", agent_result={"cost_usd": None}) + row = load_harbor(tmp_path, target="baseline").trials[0] + assert row.cost_usd == 0.1 + assert row.cost_source == "trajectory" + + +@pytest.mark.parametrize("key", ["benchmark/task-a", "suite/benchmark/task-a"]) +def test_missing_trajectory_requires_explicit_task_input(tmp_path: Path, key: str) -> None: + trial = write_trial(tmp_path) + (trial / "agent" / "trajectory.json").unlink() + with pytest.raises(ValueError, match="provide task_inputs explicitly"): + load_harbor(tmp_path, target="baseline", dataset="suite") + row = load_harbor( + tmp_path, target="baseline", dataset="suite", task_inputs={key: "Canonical task."} + ).trials[0] + assert row.messages == ( + {"role": "user", "content": [{"type": "text", "text": "Canonical task."}]}, + ) + + +def test_malformed_trajectory_records_safe_error_and_task_identity(tmp_path: Path) -> None: + invalid = write_trial(tmp_path, "invalid") + write_trial(tmp_path, "valid", task="benchmark/task-b") + (invalid / "agent" / "trajectory.json").write_text('{"PRIVATE_TRACE":broken') + with pytest.raises(ValueError, match="invalid JSON in trajectory.json") as error: + load_harbor(tmp_path, target="baseline") + assert "PRIVATE_TRACE" not in str(error.value) + run = load_harbor(tmp_path, target="baseline", on_error="record") + assert len(run.trials) == 1 + assert len(run.issues) == 1 + assert run.issues[0].task_id == "benchmark/task-a" + assert run.issues[0].source.endswith("invalid/result.json") + assert "PRIVATE_TRACE" not in run.issues[0].message + + +def test_trial_without_result_is_an_issue_not_an_invisible_exclusion(tmp_path: Path) -> None: + trial = write_trial(tmp_path) + (trial / "result.json").unlink() + run = load_harbor(tmp_path, target="baseline", on_error="record") + assert not run.trials + assert len(run.issues) == 1 + assert run.issues[0].task_id is None + + +def test_duplicate_trial_ids_are_rejected(tmp_path: Path) -> None: + write_trial(tmp_path, "first") + second = write_trial(tmp_path, "second") + update(second / "result.json", id="first") + with pytest.raises(ValueError, match="duplicate trial ID"): + load_harbor(tmp_path, target="baseline") + + +@pytest.mark.parametrize("reward", [float("nan"), float("inf"), True, "1"]) +def test_invalid_reward_is_rejected(tmp_path: Path, reward: object) -> None: + trial = write_trial(tmp_path) + update(trial / "result.json", verifier_result={"rewards": {"reward": reward}}) + with pytest.raises(ValueError, match="reward must be finite"): + load_harbor(tmp_path, target="baseline") + + +@pytest.mark.parametrize( + "field,value", + [ + ("cost_usd", -1), + ("cost_usd", float("nan")), + ("n_input_tokens", -1), + ("n_cache_tokens", True), + ], +) +def test_invalid_accounting_is_rejected(tmp_path: Path, field: str, value: object) -> None: + trial = write_trial(tmp_path) + update(trial / "result.json", agent_result={field: value}) + with pytest.raises(ValueError, match="must be"): + load_harbor(tmp_path, target="baseline") + + +@pytest.mark.parametrize("finish", ["2026-09-01T00:00:00Z", "invalid", "2026-09-01T01:10:13"]) +def test_invalid_or_negative_agent_duration_is_rejected(tmp_path: Path, finish: str) -> None: + trial = write_trial(tmp_path) + update( + trial / "result.json", + agent_execution={"started_at": "2026-09-01T01:10:00Z", "finished_at": finish}, + ) + with pytest.raises(ValueError, match="duration_seconds|timestamps"): + load_harbor(tmp_path, target="baseline") + + +def test_text_content_blocks_are_normalized_without_private_metadata(tmp_path: Path) -> None: + trial = write_trial(tmp_path) + update( + trial / "agent" / "trajectory.json", + steps=[ + { + "source": "user", + "message": [{"type": "text", "text": "Task.", "private": "PRIVATE_TRACE"}], + } + ], + ) + row = load_harbor(tmp_path, target="baseline").trials[0] + assert row.messages == ({"role": "user", "content": [{"type": "text", "text": "Task."}]},) + + +@pytest.mark.parametrize( + "steps", + [ + [{"source": "user", "message": [{"type": "image", "source": "PRIVATE_IMAGE"}]}], + [{"source": "tool", "message": "Tool output."}], + [ + {"source": "agent", "message": "First answer."}, + {"source": "user", "message": "Late task."}, + ], + [{"source": "user", "message": " "}], + ], +) +def test_unsupported_or_absent_initial_input_is_rejected( + tmp_path: Path, steps: list[dict[str, object]] +) -> None: + trial = write_trial(tmp_path) + update(trial / "agent" / "trajectory.json", steps=steps) + with pytest.raises(ValueError): + load_harbor(tmp_path, target="baseline") + + +def test_unsupported_schema_is_rejected(tmp_path: Path) -> None: + write_trial(tmp_path, version="ATIF-v2.0") + with pytest.raises(ValueError, match="schema_version"): + load_harbor(tmp_path, target="baseline") diff --git a/tests/sim/test_integration.py b/tests/sim/test_integration.py new file mode 100644 index 000000000..e674510ca --- /dev/null +++ b/tests/sim/test_integration.py @@ -0,0 +1,148 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Harbor import through real native decisions and incremental report output.""" + +import json +from pathlib import Path + +from switchyard.runner import Runner +from switchyard.sim import Dataset, evaluate, load_harbor +from switchyard.sim.__main__ import main +from tests.test_runner_bindings import JudgeStub, deployment +from tests.test_runner_bindings import judge as judge + + +def write_trial( + root: Path, name: str, model: str, reward: float | None, *, prompt="easy task", codex=False +): + trial = root / f"{name}__attempt" + (trial / "agent").mkdir(parents=True) + (trial / "result.json").write_text( + json.dumps( + { + "task_name": name, + "id": name, + "task_checksum": f"checksum-{name}", + "agent_info": {"model_info": {"name": model}}, + "verifier_result": {"rewards": {"reward": reward}}, + "agent_result": {"cost_usd": 1 if model == "weak/model" else 3}, + } + ) + ) + steps = ( + [ + {"source": "system", "message": "agent instructions"}, + {"source": "user", "message": "workspace context"}, + ] + if codex + else [] + ) + steps += [ + {"source": "user", "message": prompt}, + {"source": "agent", "message": "FUTURE_ANSWER_MUST_NOT_LEAK"}, + ] + (trial / "agent" / "trajectory.json").write_text( + json.dumps({"schema_version": "ATIF-v1.7", "steps": steps}) + ) + + +async def test_harbor_to_classifier_scoring_and_matched_metrics(judge: JudgeStub, tmp_path: Path): + for name, prompt in ( + ("easy", "easy task"), + ("hard", "TASK_REQUIRES_STRONG"), + ("ungraded", "easy task"), + ): + write_trial( + tmp_path / "weak", + name, + "weak/model", + None if name == "ungraded" else float(name == "easy"), + prompt=prompt, + codex=True, + ) + write_trial(tmp_path / "strong", name, "strong/model", 1, prompt=prompt) + dataset = Dataset.from_runs( + {target: load_harbor(tmp_path / target, target=target) for target in ("weak", "strong")}, + input_target="weak", + ) + rows = [] + report = await evaluate( + dataset, + Runner.from_toml(deployment(judge.url)), + route="auto", + on_result=rows.append, + price_call=lambda call: 0.01, + ) + summary = report.to_dict() + assert summary["counts"]["scored"] == 2 + assert summary["counts"]["unscored"] == 1 + assert summary["comparison"]["tasks"] == 2 + assert summary["comparison"]["routed_mean_reward"] == 1 + assert summary["comparison"]["targets"] == {"weak": 0.5, "strong": 1} + assert summary["cost_comparison"]["tasks"] == 3 + assert summary["routing"]["calls"] == 3 + assert all(call["model"] == "judge/model" for call in judge.calls) + assert "FUTURE_ANSWER_MUST_NOT_LEAK" not in json.dumps(judge.calls) + assert len(judge.calls) == 3 + assert all(row.evidence["source"] == "llm-classifier" and row.decision_id for row in rows) + assert all(row.fallbacks for row in rows) + assert not report.complete + + +def test_cli_writes_flushed_rows_manifest_and_report_without_copying_config( + judge: JudgeStub, tmp_path: Path +): + write_trial(tmp_path / "recordings", "task", "weak/model", 1) + config = tmp_path / "routes.toml" + config.write_text(deployment(judge.url)) + output = tmp_path / "evaluation" + args = [ + "--config", + str(config), + "--route", + "fixed", + "--run", + f"weak={tmp_path / 'recordings'}", + "--input-target", + "weak", + "--output", + str(output), + ] + assert main(args) == 0 + assert judge.calls == [] + report = json.loads((output / "report.json").read_text()) + rows = [json.loads(line) for line in (output / "results.jsonl").read_text().splitlines()] + manifest = json.loads((output / "manifest.json").read_text()) + assert report["complete"] and len(rows) == 1 + assert rows[0]["outcome"]["reward"] == 1 + assert len(manifest["config_sha256"]) == 64 + assert "provider-secret" not in "".join(path.read_text() for path in output.iterdir()) + original = (output / "report.json").read_text() + assert main(args) == 2 + assert (output / "report.json").read_text() == original + + +def test_cli_exits_incomplete_for_missing_reward(judge: JudgeStub, tmp_path: Path): + write_trial(tmp_path / "recordings", "task", "weak/model", None) + config = tmp_path / "routes.toml" + config.write_text(deployment(judge.url)) + output = tmp_path / "evaluation" + assert ( + main( + [ + "--config", + str(config), + "--route", + "fixed", + "--run", + f"weak={tmp_path / 'recordings'}", + "--input-target", + "weak", + "--output", + str(output), + ] + ) + == 1 + ) + assert json.loads((output / "report.json").read_text())["counts"]["unscored"] == 1 diff --git a/tests/sim/test_report.py b/tests/sim/test_report.py new file mode 100644 index 000000000..484b96c58 --- /dev/null +++ b/tests/sim/test_report.py @@ -0,0 +1,262 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Coverage, weighting, and isolation of task evaluation summaries.""" + +import json +from dataclasses import replace + +import pytest + +from switchyard.sim.models import Outcome, Result, Trial +from switchyard.sim.report import Report + + +def outcome(rewards, *, cost=1.0, duration=3.0): + return Outcome.from_trials( + tuple( + Trial( + "task", + str(index), + "target", + (), + reward=reward, + cost_usd=cost, + duration_seconds=duration, + ) + for index, reward in enumerate(rewards) + ) + ) + + +def result(task_id, *, fast=None, strong=None, target="fast", **kwargs): + baselines = { + "fast": fast if fast is not None else outcome([0.0]), + "strong": strong if strong is not None else outcome([1.0], cost=2.0), + } + return Result( + task_id, + baselines, + target=target, + outcome=baselines[target] if target is not None else None, + routing_seconds=0.5, + routing_calls=1, + routing_failed_calls=0, + routing_cost_usd=0.1, + routing_usage={"input_tokens": 10, "output_tokens": 2}, + **kwargs, + ) + + +def report(expected): + return Report(expected, ("fast", "strong"), run_id="run", concurrency=2) + + +def test_repeats_are_averaged_within_task_and_comparisons_share_cohort(): + summary = report(3) + summary.add(result("one", fast=outcome([0, 1, 1, 0]))) + summary.add(result("two", fast=outcome([1]), strong=outcome([0]))) + summary.add(result("missing-baseline", fast=outcome([1]), strong=outcome([None]))) + data = summary.to_dict() + assert data["complete"] is True + assert data["counts"]["scored"] == 3 + assert data["recorded"]["reward"]["mean"] == pytest.approx(2.5 / 3) + assert data["comparison"] == { + "tasks": 2, + "routed_mean_reward": 0.75, + "targets": {"fast": 0.75, "strong": 0.5}, + "empirical_oracle_mean_reward": 1.0, + } + assert data["estimated_cost_usd"]["total"] == pytest.approx(3.3) + json.dumps(data, allow_nan=False) + + +def test_errors_and_unknowns_preserve_denominators_and_known_spend(): + summary = report(3) + summary.add(result("known")) + summary.add(replace(result("missing"), outcome=outcome([None], cost=None, duration=None))) + summary.add( + replace( + result("failed"), + target=None, + outcome=None, + error="timeout", + routing_failed_calls=1, + routing_cost_usd=None, + routing_usage={"input_tokens": None}, + ) + ) + data = summary.to_dict() + assert data["complete"] is False + assert data["counts"] == { + "total": 3, + "processed": 3, + "pending": 0, + "routed": 2, + "scored": 1, + "errors": 1, + "unscored": 2, + } + assert data["recorded"]["reward"] == { + "known_tasks": 1, + "observed_total": 0.0, + "total": None, + "mean": None, + } + assert data["routing"]["cost_usd"]["observed_total"] == pytest.approx(0.2) + assert data["routing"]["cost_usd"]["total"] is None + assert data["routing"]["usage"]["input_tokens"]["known_tasks"] == 2 + assert data["routing"]["usage"]["output_tokens"]["total"] is None + assert data["routing"]["failed_calls"] == 1 + assert data["estimated_cost_usd"]["total"] is None + assert data["comparison"]["tasks"] == 1 + + +def test_partial_run_snapshot_and_duplicate_rejection_are_atomic(): + summary = report(2) + summary.add(result("one")) + snapshot = summary.to_dict() + assert snapshot["estimated_cost_usd"]["total"] is None + with pytest.raises(ValueError, match="duplicate"): + summary.add(result("one")) + assert summary.to_dict() == snapshot + with pytest.raises(ValueError, match="routing_seconds"): + summary.add(replace(result("two"), routing_seconds=float("nan"))) + assert summary.to_dict() == snapshot + summary.add(result("two", target="strong")) + assert snapshot["counts"]["processed"] == 1 + assert snapshot["targets"] == {"fast": 1, "strong": 0} + assert summary.to_dict()["counts"]["processed"] == 2 + with pytest.raises(ValueError, match="more results"): + summary.add(result("three")) + + +def test_zero_call_usage_and_missing_call_usage_remain_distinct(): + summary = report(3) + summary.add(replace(result("no-call"), routing_calls=0, routing_usage={}, routing_cost_usd=0.0)) + summary.add(replace(result("unreported-call"), routing_usage={})) + summary.add(result("reported-call")) + usage = summary.to_dict()["routing"]["usage"] + assert usage["input_tokens"] == { + "known_tasks": 2, + "observed_total": 10.0, + "total": None, + "mean": None, + } + + +def test_selected_error_is_never_scored_and_empty_comparison_is_unknown(): + summary = report(1) + summary.add(result("unsupported", error="request was rewritten")) + data = summary.to_dict() + assert data["counts"]["routed"] == 1 + assert data["counts"]["scored"] == 0 + assert data["recorded"]["cost_usd"]["total"] is None + assert data["estimated_cost_usd"]["total"] is None + assert data["comparison"] == { + "tasks": 0, + "routed_mean_reward": None, + "targets": {"fast": None, "strong": None}, + "empirical_oracle_mean_reward": None, + } + assert "unknown" in summary.format_text() + + +def test_lost_observations_are_unknown_and_input_coverage_is_preserved(): + coverage = {"input_tasks": 4, "eligible_tasks": 2, "excluded": ["three", "four"]} + summary = Report(2, ("fast", "strong"), run_id="run", concurrency=2, coverage=coverage) + summary.add(result("one")) + summary.add( + replace( + result("two"), + error="canceled", + routing_calls=None, + routing_failed_calls=None, + routing_cost_usd=None, + routing_usage={}, + ) + ) + data = summary.to_dict() + assert data["routing"]["calls"] is None + assert data["routing"]["known_call_tasks"] == 1 + assert data["routing"]["observed_calls"] == 1 + assert data["routing"]["failed_calls"] is None + assert data["routing"]["usage"]["input_tokens"]["known_tasks"] == 1 + assert data["coverage"] == coverage + coverage["excluded"].append("five") + data["coverage"]["excluded"].clear() + assert summary.to_dict()["coverage"]["excluded"] == ["three", "four"] + + +def test_successful_unknown_target_is_rejected_without_changing_report(): + summary = report(1) + before = summary.to_dict() + with pytest.raises(ValueError, match="configured targets"): + summary.add(replace(result("invalid"), target="other")) + assert summary.to_dict() == before + + +def test_cost_comparisons_use_one_matched_task_cohort_and_equal_task_weights(): + summary = report(4) + summary.add(result("one", fast=outcome([0, 1, 1, 0], cost=2), strong=outcome([1], cost=8))) + summary.add( + result("two", fast=outcome([None], cost=4), strong=outcome([1], cost=6), target="strong") + ) + summary.add( + replace( + result("missing-baseline", fast=outcome([1], cost=100), strong=outcome([1], cost=None)), + routing_cost_usd=999, + ) + ) + summary.add( + replace( + result("failed", fast=outcome([1], cost=500), error="routing failed"), + routing_cost_usd=None, + ) + ) + data = summary.to_dict() + assert data["comparison"]["tasks"] == 2 + assert data["cost_comparison"] == { + "tasks": 2, + "routed_mean_cost_usd": 4.0, + "targets": {"fast": 3.0, "strong": 7.0}, + "routing_mean_cost_usd": pytest.approx(0.1), + "routed_mean_cost_with_routing_usd": pytest.approx(4.1), + } + + +def test_cost_comparison_routing_cost_requires_all_cohort_tasks(): + summary = report(3) + summary.add(result("known")) + summary.add(replace(result("unknown-routing"), routing_cost_usd=None)) + summary.add(result("outside-cohort", strong=outcome([1], cost=None))) + costs = summary.to_dict()["cost_comparison"] + assert costs == { + "tasks": 2, + "routed_mean_cost_usd": 1.0, + "targets": {"fast": 1.0, "strong": 2.0}, + "routing_mean_cost_usd": None, + "routed_mean_cost_with_routing_usd": None, + } + + +def test_empty_cost_comparison_is_unknown_and_zero_cost_remains_known(): + summary = report(2) + summary.add(result("missing", fast=outcome([1], cost=None))) + assert summary.to_dict()["cost_comparison"] == { + "tasks": 0, + "routed_mean_cost_usd": None, + "targets": {"fast": None, "strong": None}, + "routing_mean_cost_usd": None, + "routed_mean_cost_with_routing_usd": None, + } + summary.add( + replace( + result("free", fast=outcome([1], cost=0), strong=outcome([1], cost=0)), + routing_cost_usd=0, + ) + ) + costs = summary.to_dict()["cost_comparison"] + assert costs["tasks"] == 1 + assert costs["routed_mean_cost_usd"] == costs["routing_mean_cost_usd"] == 0 + assert costs["routed_mean_cost_with_routing_usd"] == 0 diff --git a/tests/test_runner_bindings.py b/tests/test_runner_bindings.py new file mode 100644 index 000000000..4b7bd1f57 --- /dev/null +++ b/tests/test_runner_bindings.py @@ -0,0 +1,331 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Configured native decisions call only the providers needed by routing.""" + +from __future__ import annotations + +import asyncio +import json +from collections.abc import Iterator +from dataclasses import dataclass, field +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer +from pathlib import Path +from threading import Event, Thread +from typing import Any + +import pytest + +from switchyard.libsy import RoutingOutcome +from switchyard.runner import DecisionError, Runner + + +@dataclass +class JudgeStub: + url: str = "" + calls: list[dict[str, Any]] = field(default_factory=list) + response_text: str | None = None + status: int = 200 + started: Event = field(default_factory=Event) + release: Event | None = None + completed: Event = field(default_factory=Event) + + +@pytest.fixture +def judge(monkeypatch: pytest.MonkeyPatch) -> Iterator[JudgeStub]: + monkeypatch.setenv("SWITCHYARD_RUNNER_TEST_KEY", "provider-secret") + stub = JudgeStub() + + class Handler(BaseHTTPRequestHandler): + def do_POST(self) -> None: + body = json.loads(self.rfile.read(int(self.headers["Content-Length"]))) + stub.calls.append(body) + stub.started.set() + if stub.release is not None: + stub.release.wait(timeout=5) + verdict = { + "crux": "bounded task", + "primary_rule": "SUP-1", + "capability_boundary": "supported", + "p_solve": 0.1 if "TASK_REQUIRES_STRONG" in json.dumps(body) else 0.9, + } + if stub.status == 200: + payload = { + "id": "judge-response", + "model": body["model"], + "choices": [ + { + "index": 0, + "message": { + "role": "assistant", + "content": stub.response_text or json.dumps(verdict), + }, + "finish_reason": "stop", + } + ], + "usage": { + "prompt_tokens": 17, + "completion_tokens": 8, + "total_tokens": 25, + "prompt_tokens_details": {"cached_tokens": 5}, + "completion_tokens_details": {"reasoning_tokens": 3}, + }, + } + else: + payload = {"error": {"message": "echoed provider-secret and private prompt"}} + encoded = json.dumps(payload).encode() + self.send_response(stub.status) + self.send_header("Content-Type", "application/json") + self.send_header("Content-Length", str(len(encoded))) + self.end_headers() + try: + self.wfile.write(encoded) + except (BrokenPipeError, ConnectionResetError): + pass + finally: + stub.completed.set() + + def log_message(self, format: str, *args: object) -> None: + pass + + server = ThreadingHTTPServer(("127.0.0.1", 0), Handler) + thread = Thread(target=server.serve_forever, daemon=True) + thread.start() + stub.url = f"http://127.0.0.1:{server.server_port}/v1" + try: + yield stub + finally: + if stub.release is not None: + stub.release.set() + server.shutdown() + server.server_close() + thread.join(timeout=5) + + +def deployment(url: str, *, escalation: bool = False) -> str: + classifier = ( + 'mode = "escalation"\nescalation = { confirmations = 2 }' + if escalation + else 'base_threshold = 0.5\nclassify_trigger = "new_session"' + ) + return f""" +schema_version = 1 +[llm_clients.provider] +format = "openai_chat" +base_url = {json.dumps(url)} +api_key_env = "SWITCHYARD_RUNNER_TEST_KEY" +max_retries = 0 +[targets.judge] +id = "judge/model" +llm_client = "provider" +[targets.strong] +id = "strong/model" +llm_client = "provider" +[targets.weak] +id = "weak/model" +llm_client = "provider" +[routes.classifier] +id = "auto" +type = "llm_classifier" +classifier_target = "judge" +strong_target = "strong" +weak_target = "weak" +{classifier} +[routes.fixed] +id = "fixed" +type = "passthrough" +target = "weak" +""" + + +def request(text: str = "easy task", *, model: str = "auto") -> dict[str, object]: + return { + "model": model, + "messages": [{"role": "user", "content": [{"type": "text", "text": text}]}], + } + + +def session(identity: str) -> dict[str, str]: + return {"x-switchyard-session-id": identity} + + +async def test_native_judge_only_decision_preserves_targets_and_usage( + judge: JudgeStub, tmp_path: Path +) -> None: + path = tmp_path / "routes.toml" + path.write_text(deployment(judge.url)) + runner = Runner.load(path) + + targets = runner.validate_decision_route("auto") + assert {(target.target, target.model) for target in targets} == { + ("strong", "strong/model"), + ("weak", "weak/model"), + } + assert judge.calls == [] + decision = await runner.decide(request(), headers=session("task-1")) + + assert (decision.selected.target, decision.selected.model) == ("weak", "weak/model") + assert [(target.target, target.model) for target in decision.fallbacks] == [ + ("strong", "strong/model") + ] + assert isinstance(decision.outcome, RoutingOutcome) + assert decision.outcome.selected_model_ids == ["weak/model", "strong/model"] + assert decision.outcome.request["model"] == "weak/model" + assert decision.outcome.response is None + assert decision.outcome.metadata.evidence["source"] == "llm-classifier" + assert [call["model"] for call in judge.calls] == ["judge/model"] + (call,) = decision.calls + assert call.model == "judge/model" + assert call.is_success + assert call.usage == { + "input_tokens": 12, + "cached_input_tokens": 5, + "cache_creation_input_tokens": None, + "output_tokens": 8, + "total_tokens": 25, + "reasoning_tokens": 3, + } + assert decision.duration_seconds >= call.duration_seconds >= 0 + + +async def test_passthrough_decision_never_calls_answer_model(judge: JudgeStub) -> None: + runner = Runner.from_toml(deployment(judge.url)) + decision = await runner.decide(request(model="fixed")) + assert decision.selected.target == "weak" + assert decision.calls == [] + assert decision.outcome.response is None + assert judge.calls == [] + + +async def test_affinity_and_observations_are_isolated_across_sessions_and_runners( + judge: JudgeStub, +) -> None: + runner = Runner.from_toml(deployment(judge.url)) + first = await runner.decide(request(), headers=session("shared")) + retained = await runner.decide(request("TASK_REQUIRES_STRONG"), headers=session("shared")) + weak, strong = await asyncio.gather( + runner.decide(request(), headers=session("independent-weak")), + runner.decide(request("TASK_REQUIRES_STRONG"), headers=session("independent-strong")), + ) + fresh = await Runner.from_toml(deployment(judge.url)).decide( + request("TASK_REQUIRES_STRONG"), headers=session("shared") + ) + assert first.selected.target == retained.selected.target == weak.selected.target == "weak" + assert strong.selected.target == fresh.selected.target == "strong" + assert retained.calls == [] + assert retained.outcome.metadata.evidence["source"] == "retained" + assert [len(decision.calls) for decision in (first, weak, strong, fresh)] == [1, 1, 1, 1] + assert len(judge.calls) == 4 + + +@pytest.mark.parametrize("advisor", [False, True]) +async def test_response_based_route_requires_opt_in_before_calls( + judge: JudgeStub, advisor: bool +) -> None: + source = deployment(judge.url, escalation=True) + if advisor: + source = ( + source.split("[routes.classifier]")[0] + + """ +[routes.advisor] +id = "auto" +type = "advisor" +executor_target = "weak" +advisor_target = "judge" +""" + ) + runner = Runner.from_toml(source) + with pytest.raises(ValueError, match="allow_response=True"): + runner.validate_decision_route("auto") + with pytest.raises(ValueError, match="allow_response=True"): + await runner.decide(request()) + assert runner.validate_decision_route("auto", allow_response=True) + assert judge.calls == [] + + +async def test_invalid_verdict_preserves_fail_open_evidence_and_call_cost(judge: JudgeStub) -> None: + judge.response_text = "not valid routing JSON" + runner = Runner.from_toml(deployment(judge.url)) + decision = await runner.decide(request(), headers=session("malformed")) + assert decision.selected.target == "strong" + assert decision.outcome.metadata.evidence == { + "source": "fail_open", + "reason_code": "parse_error", + } + assert len(decision.calls) == 1 + assert decision.calls[0].is_success + assert decision.calls[0].usage["input_tokens"] == 12 + assert [call["model"] for call in judge.calls] == ["judge/model"] + + +async def test_provider_failure_keeps_observations_and_safe_diagnostics(judge: JudgeStub) -> None: + judge.status = 503 + runner = Runner.from_toml(deployment(judge.url)) + with pytest.raises(DecisionError) as caught: + await runner.decide(request()) + error = caught.value + assert error.kind == "upstream_http" + assert error.upstream_status == 503 + assert error.target == "judge/model" + assert "provider-secret" not in str(error) + assert "private prompt" not in str(error) + (call,) = error.calls + assert call.model == "judge/model" + assert not call.is_success + assert call.usage is None + assert error.duration_seconds >= call.duration_seconds >= 0 + + +async def test_bad_request_and_configuration_fail_before_calls( + judge: JudgeStub, tmp_path: Path +) -> None: + runner = Runner.from_toml(deployment(judge.url)) + with pytest.raises(ValueError, match="unknown route"): + runner.validate_decision_route("missing") + with pytest.raises(ValueError, match="non-empty model"): + await runner.decide({"messages": []}) + with pytest.raises(ValueError): + await runner.decide(request(), headers={"invalid\nheader": "value"}) + with pytest.raises(ValueError) as caught: + Runner.from_toml('schema_version = "provider-secret"') + assert "provider-secret" not in str(caught.value) + assert "TOML" in str(caught.value) + assert "byte" in str(caught.value) + invalid_path = tmp_path / "invalid.toml" + invalid_path.write_text('schema_version = "provider-secret"') + with pytest.raises(ValueError) as caught: + Runner.load(invalid_path) + assert "provider-secret" not in str(caught.value) + assert "byte" in str(caught.value) + with pytest.raises(ValueError, match="NotFound"): + Runner.load(tmp_path / "absent.toml") + assert judge.calls == [] + + +def test_configuration_diagnostics_name_missing_target_and_environment( + judge: JudgeStub, monkeypatch: pytest.MonkeyPatch +) -> None: + unknown = deployment(judge.url).replace('strong_target = "strong"', 'strong_target = "missing"') + with pytest.raises(ValueError, match="unknown target missing"): + Runner.from_toml(unknown) + monkeypatch.delenv("SWITCHYARD_RUNNER_TEST_KEY") + with pytest.raises(ValueError, match="api_key_env SWITCHYARD_RUNNER_TEST_KEY"): + Runner.from_toml(deployment(judge.url)) + assert judge.calls == [] + + +async def test_cancellation_does_not_complete_or_retain_a_decision(judge: JudgeStub) -> None: + judge.release = Event() + runner = Runner.from_toml(deployment(judge.url)) + future = asyncio.ensure_future(runner.decide(request(), headers=session("cancelled"))) + assert await asyncio.to_thread(judge.started.wait, 5) + future.cancel() + with pytest.raises(asyncio.CancelledError): + await future + judge.release.set() + assert await asyncio.to_thread(judge.completed.wait, 5) + # A fresh call using the same identity must still consult the judge. + decision = await runner.decide(request("TASK_REQUIRES_STRONG"), headers=session("cancelled")) + assert decision.selected.target == "strong" + assert len(decision.calls) == 1 + assert [call["model"] for call in judge.calls] == ["judge/model", "judge/model"] From 9b936f10dc330c6a5d6fe1ec8961ad7a6f30da44 Mon Sep 17 00:00:00 2001 From: ayushag Date: Tue, 29 Sep 2026 22:47:50 -0700 Subject: [PATCH 02/48] fix(sim): preserve valid trials beside malformed artifacts Signed-off-by: ayushag --- switchyard/sim/harbor.py | 14 +++++++--- tests/sim/test_harbor.py | 58 ++++++++++++++++++++++++++++++++++++++++ 2 files changed, 68 insertions(+), 4 deletions(-) diff --git a/switchyard/sim/harbor.py b/switchyard/sim/harbor.py index 8d934cc8b..b05f4770e 100644 --- a/switchyard/sim/harbor.py +++ b/switchyard/sim/harbor.py @@ -90,15 +90,16 @@ def _trial_paths(root: Path) -> list[Path]: result_path = root / "result.json" if (root / "agent").is_dir(): return [result_path] + unreadable_result = False if result_path.is_file(): try: if "task_name" in _read_object(result_path): return [result_path] - except ValueError: - return [result_path] + except (OSError, ValueError): + unreadable_result = True jobs = root / "jobs" roots = sorted(p for p in jobs.iterdir() if p.is_dir()) if jobs.is_dir() else [root] - return [ + paths = [ trial / "result.json" for job in roots for trial in sorted(job.iterdir()) @@ -109,6 +110,9 @@ def _trial_paths(root: Path) -> list[Path]: or (trial / "config.json").is_file() ) ] + # A damaged job summary must not hide valid trial directories. Without + # children, keep the unreadable artifact visible as a standalone trial issue. + return paths or ([result_path] if unreadable_result else []) def _read_object(path: Path) -> dict[str, Any]: @@ -218,7 +222,9 @@ def _messages(trajectory: dict[str, Any], fallback: str | None) -> tuple[dict[st has_user_input = False if trajectory: version = trajectory.get("schema_version") - if version not in {f"ATIF-v1.{minor}" for minor in range(8)}: + if not isinstance(version, str) or version not in { + f"ATIF-v1.{minor}" for minor in range(8) + }: raise ValueError("unsupported or missing ATIF schema_version") steps = trajectory.get("steps") if not isinstance(steps, list): diff --git a/tests/sim/test_harbor.py b/tests/sim/test_harbor.py index fc9a72427..6b657a5ab 100644 --- a/tests/sim/test_harbor.py +++ b/tests/sim/test_harbor.py @@ -119,6 +119,46 @@ def test_discovery_ignores_nested_metadata_and_accepts_download_layout(tmp_path: assert len(load_harbor(root, target="baseline").trials) == 1 +@pytest.mark.parametrize("unreadable", [False, True]) +def test_invalid_job_summary_does_not_hide_valid_trials( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch, unreadable: bool +) -> None: + write_trial(tmp_path) + summary = tmp_path / "result.json" + summary.write_text("{") + if unreadable: + original_open = Path.open + + def open_artifact(path: Path, *args, **kwargs): + if path == summary: + raise PermissionError("cannot read summary") + return original_open(path, *args, **kwargs) + + monkeypatch.setattr(Path, "open", open_artifact) + + run = load_harbor(tmp_path, target="baseline", on_error="record") + assert len(run.trials) == 1 + assert run.issues == () + + +@pytest.mark.parametrize("has_agent_directory", [False, True]) +def test_standalone_malformed_result_is_still_a_trial_issue( + tmp_path: Path, has_agent_directory: bool +) -> None: + if has_agent_directory: + (tmp_path / "agent").mkdir() + result_path = tmp_path / "result.json" + result_path.write_text("{") + + with pytest.raises(ValueError, match="invalid JSON in result.json"): + load_harbor(tmp_path, target="baseline") + run = load_harbor(tmp_path, target="baseline", on_error="record") + assert run.trials == () + assert len(run.issues) == 1 + assert run.issues[0].source == str(result_path) + assert run.issues[0].task_id is None + + def test_repeated_tasks_are_preserved_as_distinct_trials(tmp_path: Path) -> None: write_trial(tmp_path, "repeat-1") write_trial(tmp_path, "repeat-2") @@ -303,3 +343,21 @@ def test_unsupported_schema_is_rejected(tmp_path: Path) -> None: write_trial(tmp_path, version="ATIF-v2.0") with pytest.raises(ValueError, match="schema_version"): load_harbor(tmp_path, target="baseline") + + +@pytest.mark.parametrize("version", [[], {}]) +def test_invalid_schema_type_records_issue_without_aborting_other_trials( + tmp_path: Path, version: object +) -> None: + invalid = write_trial(tmp_path, "invalid") + write_trial(tmp_path, "valid", task="benchmark/task-b") + update(invalid / "agent" / "trajectory.json", schema_version=version) + + with pytest.raises(ValueError, match="schema_version"): + load_harbor(tmp_path, target="baseline") + run = load_harbor(tmp_path, target="baseline", on_error="record") + assert len(run.trials) == 1 + assert run.trials[0].task_id == "benchmark/task-b" + assert len(run.issues) == 1 + assert run.issues[0].task_id == "benchmark/task-a" + assert "schema_version" in run.issues[0].message From 74ed6a610e8505061da4a9d09de38d2f6d71cd95 Mon Sep 17 00:00:00 2001 From: ayushag Date: Tue, 29 Sep 2026 22:50:00 -0700 Subject: [PATCH 03/48] fix(sim): retain routing diagnostics and validate CLI options early Signed-off-by: ayushag --- switchyard/sim/__main__.py | 2 + switchyard/sim/evaluate.py | 21 +++++++-- switchyard/sim/models.py | 3 ++ tests/sim/test_evaluate.py | 9 +++- tests/sim/test_integration.py | 80 +++++++++++++++++++++++++++++++++++ 5 files changed, 110 insertions(+), 5 deletions(-) diff --git a/switchyard/sim/__main__.py b/switchyard/sim/__main__.py index 9c676a557..5902e9dea 100644 --- a/switchyard/sim/__main__.py +++ b/switchyard/sim/__main__.py @@ -17,6 +17,7 @@ from switchyard.runner import Runner from . import Dataset, Result, evaluate, load_harbor +from .evaluate import _validate_options def main(argv: list[str] | None = None) -> int: @@ -78,6 +79,7 @@ def main(argv: list[str] | None = None) -> int: parser.error("each --model-alias must be a unique RECORDED=CONFIGURED") aliases[recorded] = configured try: + _validate_options(args.concurrency, args.timeout) runs = { target: load_harbor( path, diff --git a/switchyard/sim/evaluate.py b/switchyard/sim/evaluate.py index 0e89b0157..173ae1feb 100644 --- a/switchyard/sim/evaluate.py +++ b/switchyard/sim/evaluate.py @@ -172,6 +172,13 @@ def _usage(calls: Sequence[RoutingCall]) -> dict[str, int | None]: return result +def _validate_options(concurrency: int, timeout: float) -> None: + if isinstance(concurrency, bool) or not isinstance(concurrency, int) or concurrency < 1: + raise ValueError("concurrency must be a positive integer") + if isinstance(timeout, bool) or not math.isfinite(timeout) or timeout <= 0: + raise ValueError("timeout must be finite and positive") + + async def evaluate( dataset: Dataset, runner: Runner, @@ -196,10 +203,10 @@ async def evaluate( callback errors or cancellation stop and drain pending work. Keep callbacks short. Cancelling local work cannot revoke provider requests already sent. """ - if isinstance(concurrency, bool) or not isinstance(concurrency, int) or concurrency < 1: - raise ValueError("concurrency must be a positive integer") - if isinstance(timeout, bool) or not math.isfinite(timeout) or timeout <= 0: - raise ValueError("timeout must be finite and positive") + # Artifact imports and pure scoring do not need to load the native bindings. + from switchyard.runner import DecisionError + + _validate_options(concurrency, timeout) if not dataset.tasks or len({task.task_id for task in dataset.tasks}) != len(dataset.tasks): raise ValueError("dataset must contain non-empty, unique tasks") targets = runner.validate_decision_route(route, allow_response=False) @@ -242,6 +249,7 @@ async def route_task(index: int, task: Task) -> Result: # Native decision errors expose completed observations without provider # bodies. Other errors (including timeouts) cannot prove call counts. calls = getattr(error, "calls", None) + native_error = error if isinstance(error, DecisionError) else None return Result( task.task_id, task.outcomes, @@ -253,6 +261,11 @@ async def route_task(index: int, task: Task) -> Result: routing_cost_usd=_cost(calls, price_call) if calls is not None else None, routing_usage=_usage(calls) if calls is not None else {}, error=f"routing failed: {type(error).__name__}", + routing_error_kind=native_error.kind if native_error is not None else None, + routing_error_status=native_error.upstream_status + if native_error is not None + else None, + routing_error_target=native_error.target if native_error is not None else None, ) score_error = None try: diff --git a/switchyard/sim/models.py b/switchyard/sim/models.py index b62f7945d..b81b5d870 100644 --- a/switchyard/sim/models.py +++ b/switchyard/sim/models.py @@ -133,3 +133,6 @@ class Result: routing_cost_usd: float | None = None routing_usage: Mapping[str, int | None] = field(default_factory=dict) error: str | None = None + routing_error_kind: str | None = None + routing_error_status: int | None = None + routing_error_target: str | None = None diff --git a/tests/sim/test_evaluate.py b/tests/sim/test_evaluate.py index 55d88b9a8..90258a7ff 100644 --- a/tests/sim/test_evaluate.py +++ b/tests/sim/test_evaluate.py @@ -160,12 +160,19 @@ def test_routing_measurements_and_pricing_are_separate_from_task_cost(): async def test_failure_is_an_error_without_exposing_provider_body(): class BrokenRunner(FakeRunner): async def decide(self, *args, **kwargs): - raise RuntimeError("provider response containing a secret") + error = RuntimeError("provider response containing a secret") + error.kind = "secret" + error.upstream_status = 503 + error.target = "secret" + raise error rows = [] report = await evaluate(dataset(1), BrokenRunner(), route="auto", on_result=rows.append) assert report.to_dict()["counts"]["errors"] == 1 assert "secret" not in rows[0].error + assert rows[0].routing_error_kind is None + assert rows[0].routing_error_status is None + assert rows[0].routing_error_target is None async def test_recorded_model_mismatch_fails_before_calls(): diff --git a/tests/sim/test_integration.py b/tests/sim/test_integration.py index e674510ca..f930a1659 100644 --- a/tests/sim/test_integration.py +++ b/tests/sim/test_integration.py @@ -6,6 +6,8 @@ import json from pathlib import Path +import pytest + from switchyard.runner import Runner from switchyard.sim import Dataset, evaluate, load_harbor from switchyard.sim.__main__ import main @@ -146,3 +148,81 @@ def test_cli_exits_incomplete_for_missing_reward(judge: JudgeStub, tmp_path: Pat == 1 ) assert json.loads((output / "report.json").read_text())["counts"]["unscored"] == 1 + + +@pytest.mark.parametrize("status", [401, 503]) +def test_cli_retains_safe_native_failure_details(judge: JudgeStub, tmp_path: Path, status: int): + judge.status = status + for target in ("weak", "strong"): + write_trial(tmp_path / target, "task", f"{target}/model", 1) + config = tmp_path / "routes.toml" + config.write_text(deployment(judge.url)) + output = tmp_path / "evaluation" + assert ( + main( + [ + "--config", + str(config), + "--route", + "auto", + "--run", + f"weak={tmp_path / 'weak'}", + "--run", + f"strong={tmp_path / 'strong'}", + "--input-target", + "weak", + "--output", + str(output), + ] + ) + == 1 + ) + row = json.loads((output / "results.jsonl").read_text()) + assert row["routing_error_kind"] == "upstream_http" + assert row["routing_error_status"] == status + assert row["routing_error_target"] == "judge/model" + assert row["routing_calls"] == row["routing_failed_calls"] == 1 + assert row["routing_cost_usd"] is None + assert row["error"] == "routing failed: DecisionError" + assert "provider-secret" not in json.dumps(row) + assert "private prompt" not in json.dumps(row) + report = json.loads((output / "report.json").read_text()) + assert report["counts"]["errors"] == 1 + assert not report["complete"] + + +@pytest.mark.parametrize( + "option", + [ + "--concurrency=0", + "--concurrency=-1", + "--timeout=0", + "--timeout=-1", + "--timeout=nan", + "--timeout=inf", + ], +) +def test_cli_invalid_options_leave_output_path_available( + judge: JudgeStub, tmp_path: Path, option: str +): + write_trial(tmp_path / "recordings", "task", "weak/model", 1) + config = tmp_path / "routes.toml" + config.write_text(deployment(judge.url)) + output = tmp_path / "new-parent" / "evaluation" + args = [ + "--config", + str(config), + "--route", + "fixed", + "--run", + f"weak={tmp_path / 'recordings'}", + "--input-target", + "weak", + "--output", + str(output), + ] + assert main([*args, option]) == 2 + assert not output.parent.exists() + assert judge.calls == [] + assert main(args) == 0 + assert (output / "report.json").exists() From dd574d9a0f5c6075fd9aaf0083eb09ff689634cf Mon Sep 17 00:00:00 2001 From: ayushag Date: Tue, 29 Sep 2026 22:58:26 -0700 Subject: [PATCH 04/48] feat(sim): support ATIF trajectories and custom converters Signed-off-by: ayushag --- README.md | 2 +- docs/simulation.md | 150 +++++++++++++++++++++++--- switchyard/sim/__init__.py | 7 +- switchyard/sim/dataset.py | 6 +- switchyard/sim/harbor.py | 64 +++-------- switchyard/sim/models.py | 7 +- switchyard/sim/trajectory.py | 144 +++++++++++++++++++++++++ tests/sim/test_harbor.py | 4 +- tests/sim/test_trajectory.py | 198 +++++++++++++++++++++++++++++++++++ 9 files changed, 510 insertions(+), 72 deletions(-) create mode 100644 switchyard/sim/trajectory.py create mode 100644 tests/sim/test_trajectory.py diff --git a/README.md b/README.md index dc2b96350..2a2420a43 100644 --- a/README.md +++ b/README.md @@ -63,7 +63,7 @@ The [routing overview](docs/routing_algorithms/overview.md) retains the full cat ![Task completion versus cost for Switchyard classification, stage, and escalation routing, compared with Opus 4.8 and GLM 5.2 single-model baselines.](assets/switchyard-cost-accuracy.png) Results depend on the benchmark, model pool, serving stack, and routing configuration. -Use the Python [task routing evaluator](docs/simulation.md) to compare routing decisions against recorded Harbor Claude and Codex runs. +Use the Python [task routing evaluator](docs/simulation.md) to compare routing decisions against ATIF trajectories, custom recordings, or Harbor Claude and Codex runs. For latency and routing overhead testing, see [Soak Testing](docs/operations/soak_test.md). ### Further reading diff --git a/docs/simulation.md b/docs/simulation.md index 6a030c57e..8e06eab15 100644 --- a/docs/simulation.md +++ b/docs/simulation.md @@ -1,7 +1,8 @@ # Evaluate task routing -`switchyard.sim` evaluates a routing policy against completed Harbor runs. It -loads the input available before an agent starts, asks Switchyard to select a +`switchyard.sim` evaluates a routing policy against completed agent runs. It +accepts ATIF trajectories, custom recordings converted to ATIF, and Harbor runs. +It loads the input available before an agent starts, asks Switchyard to select a target, and scores that choice using the target's recorded task outcomes. The simulator is a Python library included in `nemo-switchyard`. It uses the @@ -16,6 +17,35 @@ must match a target in your Switchyard deployment. Use comparable task versions, agent settings, reasoning budgets, and verifier settings across runs. A matching task name or model name alone does not establish that the experiments are comparable. +Start with this `routes.toml` to check the pipeline without provider calls. Replace +`fast-model` and `strong-model` with the model IDs in your recordings: + +```toml +schema_version = 1 + +[llm_clients.unused] +format = "openai_chat" +base_url = "http://127.0.0.1:9/v1" + +[targets.fast] +id = "fast-model" +llm_client = "unused" + +[targets.strong] +id = "strong-model" +llm_client = "unused" + +[routes.fixed] +id = "auto" +type = "passthrough" +target = "fast" +``` + +This route always selects `fast`; the unused client is never called. `auto` is the +route ID passed to `evaluate` or `--route`. `fast` and `strong` are target keys used +by the dataset and `--run`; their `id` fields name the actual models. For a live +classifier, use the [Task routing configuration](routing_algorithms/llm_classifier_routing.md). + ```python import asyncio @@ -57,6 +87,88 @@ to `evaluate` or `score` when the two systems name the same model differently. A the library does not guess equivalence by trimming model names. Missing model metadata remains visible as unverified coverage. +## Use ATIF or custom recordings + +`Trajectory.from_dict(data)` takes an owned copy of an ATIF document. +`trajectory.to_dict()` returns an independent copy with all recorded steps, +metrics, and extension fields preserved. You can load JSON with +`Trajectory.from_dict(json.loads(path.read_text(encoding="utf-8")))`. +Construction checks the ATIF version and steps container. `initial_messages()` +validates the fields it reads to extract text input; neither method performs full +ATIF schema validation. + +A custom converter is a plain function returning `Trajectory`. For example, +suppose each line of `fast.jsonl` or `strong.jsonl` contains a harness record: + +```json +{"task_id": "sum-1", "trial_id": "attempt-1", "model": "fast-model", "prompt": "What is 2 + 2?", "agent_steps": ["4"], "reward": 1.0, "cost_usd": 0.02} +``` + +Convert each record, then project it into a trial with explicit outcome metadata: + +```python +import json + +from switchyard.sim import Dataset, Run, Trajectory + +def custom_to_atif(record): + steps = [ + {"source": "user", "message": record["prompt"]}, + *({"source": "agent", "message": text} for text in record["agent_steps"]), + ] + return Trajectory.from_dict({ + "schema_version": "ATIF-v1.7", + "session_id": record["trial_id"], + "agent": {"name": "custom-agent", "version": "1"}, + "steps": [dict(step_id=i, **step) for i, step in enumerate(steps, start=1)], + }) + +def trials_from_jsonl(path, target): + with open(path, encoding="utf-8") as stream: + for line in stream: + record = json.loads(line) + yield custom_to_atif(record).to_trial( + task_id=record["task_id"], + trial_id=record["trial_id"], + target=target, + reward=record.get("reward"), + model=record.get("model"), + cost_usd=record.get("cost_usd"), + cost_source="harness" if record.get("cost_usd") is not None else None, + ) + +runs = { + target: Run(tuple(trials_from_jsonl(f"{target}.jsonl", target))) + for target in ("fast", "strong") +} +dataset = Dataset.from_runs(runs, input_target="fast") +``` + +Use the same `evaluate` call shown above. Each file must contain its own recorded +outcomes and model identity for comparable tasks. The converter owns the source +format; `to_trial` owns the ATIF input projection. It does not infer task IDs, +rewards, costs, or the recorded model from ATIF. Per-step models can override an +ATIF agent default, so choose and verify that metadata explicitly. Omitted +measurements and model IDs remain unknown. You can also supply `task_checksum`, +`duration_seconds`, `usage`, `source`, and `error` to `to_trial`. +This example stops on invalid records. If your importer tolerates errors, retain +each rejection as a `LoadIssue(source, message, task_id)` in `Run.issues`; silently +dropping failed repeats can bias the reported score. + +`to_trial` keeps the initial input and supplied metadata without retaining the +full history. The generator above therefore releases each full trajectory before +reading the next record; `Run` retains only the trials. `HarborRun` is a +compatibility alias for `Run`. Neither path requires Harbor to be installed. + +Copied continuation context can contain earlier answers or progress summaries. +If an initial step has `is_copied_context=true`, projection rejects it unless you +provide the original task text with `to_trial(task_input=...)` or +`initial_messages(task_input=...)`. That text replaces the initial conversation. +Choose root task recordings and have converters supply the original input before +execution. The library cannot reliably recognize unmarked subagent logs or summaries. +The full ATIF document remains available through `to_dict()`. Storing full history +does not implement trajectory replay or validate outcomes after model switching. + ## Import rules and coverage `load_harbor` accepts a trial directory, a job directory, or a downloaded run's @@ -99,9 +211,11 @@ The intersection excludes a task from every target if any target is missing or has an invalid trial for it. Issues and excluded task IDs remain in the report. An invalid result with no recoverable task identity must be repaired before pairing; otherwise a failed repeat could silently disappear from a task's mean. -For a missing trajectory, callers can supply `task_inputs={task_id: instruction}`. -Malformed existing trajectories still fail validation. Raw agent logs and -multimodal ATIF inputs are outside this initial importer. +For a missing trajectory or copied continuation context, callers can supply +`task_inputs={task_id: original_instruction}` to `load_harbor`. +Invalid JSON, unsupported ATIF versions, and invalid fields read during input +projection still fail validation. Unused history fields are not fully validated. +Raw agent logs and multimodal ATIF inputs are outside this importer. ## Interpret the report @@ -112,6 +226,12 @@ and means are `null`; `observed_total` retains the known portion. `complete` means every expected task has a scored reward and no evaluation error. Inspect each cost and usage field's coverage separately. +Native routing failures retain `routing_error_kind`, `routing_error_status`, and +`routing_error_target` on each result row. The status is an upstream HTTP code +when available; the target identifies the failing model, such as the classifier. +These fields distinguish failures such as HTTP 401 and 503 without storing +provider response bodies. Other exception details remain suppressed. + Reward comparisons use one common cohort where the routed outcome and every fixed-target reward are observed. The empirical recorded-outcome oracle selects the largest target mean reward for each task. It is an upper bound on these @@ -144,13 +264,16 @@ python -m switchyard.sim \ --input-target fast --output evaluation-output ``` -The output directory must be new. It contains flushed `results.jsonl` rows, a -`report.json` summary, and a `manifest.json` with configuration hash, inputs, -versions, and coverage. The deployment source and credentials are not copied. +This CLI imports Harbor layouts. Use the Python API for custom formats. +The output directory must be new. The CLI writes a `manifest.json` with the +configuration hash, inputs, package version, and coverage, then flushes each +completed row to `results.jsonl`. It writes `report.json` after evaluation returns. +The deployment source and credentials are not copied. Use `--skip-invalid --intersection` to retain import issues and explicitly evaluate the common valid subset. Exit status is 0 for complete reward coverage, 1 for an incomplete evaluation, and 2 for invalid inputs or configuration. -Completed rows survive interruption; automatic resume is not implemented. +An interruption exits 130 and preserves the manifest and completed result rows +once those files exist; there may be no final report. Automatic resume is not implemented. Use `--model-alias RECORDED=CONFIGURED` for an explicit model-ID equivalence. The Python `on_result` callback runs after each completed task. It can update a @@ -162,11 +285,12 @@ make multiple provider calls inside a decision. ## Library boundaries -`Trial` and `HarborRun` represent imported evidence. `Dataset.from_runs` validates -and pairs it. `score(task, decision)` is a pure scorer for a saved native decision. +`Trajectory` preserves ATIF documents. `Trial` and `Run` represent projected +task evidence. `Dataset.from_runs` validates and pairs it. +`score(task, decision)` is a pure scorer for a saved native decision. `evaluate` owns bounded scheduling and calls that scorer. `Report.add` accumulates -results without retaining full trajectories or result rows. Import holds only -one full trajectory at a time; paired task inputs and small outcome records stay +results without retaining full trajectories or result rows. Import processes +one trajectory file at a time; paired task inputs and small outcome records stay in memory. Native `Runner.load` and `Runner.from_toml` use Switchyard's configuration parser. diff --git a/switchyard/sim/__init__.py b/switchyard/sim/__init__.py index 35b99e6c1..570761057 100644 --- a/switchyard/sim/__init__.py +++ b/switchyard/sim/__init__.py @@ -1,13 +1,14 @@ # SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 -"""Evaluate Switchyard task routing against recorded Harbor outcomes.""" +"""Evaluate Switchyard task routing against recorded outcomes.""" from .dataset import Dataset from .evaluate import evaluate, score from .harbor import load_harbor -from .models import HarborRun, LoadIssue, Outcome, Result, Task, Trial +from .models import HarborRun, LoadIssue, Outcome, Result, Run, Task, Trial from .report import Report +from .trajectory import Trajectory __all__ = [ "Dataset", @@ -16,8 +17,10 @@ "Outcome", "Report", "Result", + "Run", "Task", "Trial", + "Trajectory", "evaluate", "load_harbor", "score", diff --git a/switchyard/sim/dataset.py b/switchyard/sim/dataset.py index d61d587ab..4517f0a22 100644 --- a/switchyard/sim/dataset.py +++ b/switchyard/sim/dataset.py @@ -10,7 +10,7 @@ from collections.abc import Mapping from dataclasses import dataclass -from .models import HarborRun, Outcome, Task, Trial +from .models import Outcome, Run, Task, Trial @dataclass(frozen=True) @@ -24,7 +24,7 @@ class Dataset: @classmethod def from_runs( cls, - runs: Mapping[str, HarborRun], + runs: Mapping[str, Run], *, input_target: str, intersection: bool = False, @@ -126,7 +126,7 @@ def from_runs( def _instruction(trial: Trial) -> str: - """Require the same user input when a Harbor checksum is unavailable.""" + """Require the same user input when a task checksum is unavailable.""" contents = [ message.get("content") for message in trial.messages if message.get("role") == "user" ] diff --git a/switchyard/sim/harbor.py b/switchyard/sim/harbor.py index b05f4770e..206d23247 100644 --- a/switchyard/sim/harbor.py +++ b/switchyard/sim/harbor.py @@ -11,7 +11,8 @@ from pathlib import Path from typing import Any, Literal -from .models import HarborRun, LoadIssue, Trial +from .models import LoadIssue, Run, Trial +from .trajectory import _initial_messages, _task_messages def load_harbor( @@ -23,7 +24,7 @@ def load_harbor( on_error: Literal["raise", "record"] = "raise", task_inputs: Mapping[str, str] | None = None, cost_source: Literal["result", "trajectory"] = "result", -) -> HarborRun: +) -> Run: """Load a trial, a Harbor job, or a downloaded run containing ``jobs/``. ``target`` names the recorded model/agent configuration. An explicit dataset @@ -41,7 +42,7 @@ def load_harbor( By default any invalid trial raises. ``on_error='record'`` keeps rejected trial paths and reasons in ``issues`` so callers can account for exclusions. - Only one full trajectory is held in memory at a time. + Processes one trajectory file at a time. """ if not isinstance(target, str) or not target.strip(): raise ValueError("target must be a non-empty string") @@ -82,7 +83,7 @@ def load_harbor( if on_error == "raise": raise ValueError(f"{result_path}: {message}") from None issues.append(LoadIssue(source=str(result_path), message=message, task_id=task_id)) - return HarborRun(tuple(trials), tuple(issues)) + return Run(tuple(trials), tuple(issues)) def _trial_paths(root: Path) -> list[Path]: @@ -150,13 +151,19 @@ def _trial( cost_source: str, ) -> Trial: trajectory_path = path.parent / "agent" / "trajectory.json" - trajectory = _read_object(trajectory_path) if trajectory_path.is_file() else {} + trajectory = _read_object(trajectory_path) if trajectory_path.is_file() else None fallback = None if task_inputs is not None: fallback = task_inputs.get(task_id, task_inputs.get(result["task_name"])) - messages = _messages(trajectory, fallback) + messages = ( + _initial_messages(trajectory, fallback) + if trajectory is not None + else _task_messages(fallback) + ) agent_result = _optional_object(result.get("agent_result"), "agent_result") - metrics = _optional_object(trajectory.get("final_metrics"), "final_metrics") + metrics = _optional_object( + trajectory.get("final_metrics") if trajectory is not None else None, "final_metrics" + ) verifier = _optional_object(result.get("verifier_result"), "verifier_result") rewards = _optional_object(verifier.get("rewards"), "rewards") agent = _optional_object(result.get("agent_info"), "agent_info") @@ -217,49 +224,6 @@ def _trial( ) -def _messages(trajectory: dict[str, Any], fallback: str | None) -> tuple[dict[str, object], ...]: - messages: list[dict[str, object]] = [] - has_user_input = False - if trajectory: - version = trajectory.get("schema_version") - if not isinstance(version, str) or version not in { - f"ATIF-v1.{minor}" for minor in range(8) - }: - raise ValueError("unsupported or missing ATIF schema_version") - steps = trajectory.get("steps") - if not isinstance(steps, list): - raise ValueError("ATIF steps must be an array") - for raw_step in steps: - step = _object(raw_step, "ATIF step") - role = step.get("source") - if role == "agent": - break - if role not in ("system", "user"): - raise ValueError("unsupported ATIF input source") - content = step.get("message") - if isinstance(content, str): - content = [{"type": "text", "text": content}] - elif isinstance(content, list): - blocks = [] - for raw_block in content: - block = _object(raw_block, "ATIF message content") - if block.get("type") != "text" or not isinstance(block.get("text"), str): - raise ValueError("only text ATIF input is supported") - blocks.append({"type": "text", "text": block["text"]}) - content = blocks - else: - raise ValueError("ATIF input message must contain text") - if content: - messages.append({"role": role, "content": content}) - has_user_input |= role == "user" and any(block["text"].strip() for block in content) - if has_user_input: - return tuple(messages) - if fallback is not None: - text = _string(fallback, "task_inputs value") - return ({"role": "user", "content": [{"type": "text", "text": text}]},) - raise ValueError("missing initial task input; provide task_inputs explicitly") - - def _duration(result: dict[str, Any]) -> float | None: timing = _optional_object(result.get("agent_execution"), "agent_execution") start, finish = timing.get("started_at"), timing.get("finished_at") diff --git a/switchyard/sim/models.py b/switchyard/sim/models.py index b81b5d870..501a1dd64 100644 --- a/switchyard/sim/models.py +++ b/switchyard/sim/models.py @@ -63,13 +63,16 @@ class LoadIssue: @dataclass(frozen=True) -class HarborRun: - """Imported trials and every rejected input, without retaining full trajectories.""" +class Run: + """Imported evidence from any producer, including every rejected input.""" trials: tuple[Trial, ...] issues: tuple[LoadIssue, ...] = () +HarborRun = Run + + @dataclass(frozen=True) class Outcome: """Mean recorded measurements for one task and target. diff --git a/switchyard/sim/trajectory.py b/switchyard/sim/trajectory.py new file mode 100644 index 000000000..db69d1c29 --- /dev/null +++ b/switchyard/sim/trajectory.py @@ -0,0 +1,144 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""ATIF documents and their task-routing projection, independent of the producer.""" + +from __future__ import annotations + +from collections.abc import Mapping +from copy import deepcopy +from typing import Any + +from .models import Trial + +_SUPPORTED_VERSIONS = frozenset(f"ATIF-v1.{minor}" for minor in range(8)) + + +class Trajectory: + """An owned ATIF document, including fields unused by task simulation. + + Validates the version and steps container on construction. Input projection + validates the fields it consumes; this is not a full ATIF schema validator. + Converter functions can return this type without inheriting a base class. + """ + + def __init__(self, data: Mapping[str, Any]) -> None: + _steps(data) + self._data = deepcopy(dict(data)) + + @classmethod + def from_dict(cls, data: Mapping[str, Any]) -> Trajectory: + """Copy a JSON-compatible ATIF document without dropping extension fields.""" + return cls(data) + + def to_dict(self) -> dict[str, Any]: + """Return an independent copy, including the complete recorded history.""" + return deepcopy(self._data) + + def initial_messages(self, *, task_input: str | None = None) -> tuple[dict[str, object], ...]: + """Extract text input before the first agent step, without later evidence. + + ``task_input`` supplies original task text when the recording lacks it + or contains copied continuation context. Copied context may summarize + earlier execution, so it is never used as task-routing input. + """ + return _initial_messages(self._data, task_input) + + def to_trial( + self, + *, + task_id: str, + trial_id: str, + target: str, + reward: float | None = None, + cost_usd: float | None = None, + cost_source: str | None = None, + duration_seconds: float | None = None, + task_checksum: str | None = None, + model: str | None = None, + source: str | None = None, + usage: Mapping[str, int | None] | None = None, + error: str | None = None, + task_input: str | None = None, + ) -> Trial: + """Project into task evidence without retaining the full trajectory. + + Identity, outcomes, and accounting are explicit: ATIF alone does not + define verifier rewards or a cross-run task identity. Model identity is + also explicit because per-step models can override the ATIF agent default. + """ + return Trial( + task_id=task_id, + trial_id=trial_id, + target=target, + messages=self.initial_messages(task_input=task_input), + reward=reward, + cost_usd=cost_usd, + cost_source=cost_source, + duration_seconds=duration_seconds, + task_checksum=task_checksum, + model=model, + source=source, + usage=dict(usage) if usage is not None else {}, + error=error, + ) + + +def _steps(data: Mapping[str, Any]) -> list[Any]: + if not isinstance(data, Mapping): + raise ValueError("ATIF trajectory must be an object") + version = data.get("schema_version") + if not isinstance(version, str) or version not in _SUPPORTED_VERSIONS: + raise ValueError("unsupported or missing ATIF schema_version") + steps = data.get("steps") + if not isinstance(steps, list): + raise ValueError("ATIF steps must be an array") + return steps + + +def _initial_messages( + data: Mapping[str, Any], task_input: str | None +) -> tuple[dict[str, object], ...]: + """Share projection with file importers without copying unused history.""" + messages: list[dict[str, object]] = [] + has_user_input = False + for step in _steps(data): + if not isinstance(step, dict): + raise ValueError("ATIF step must be an object") + role = step.get("source") + if role == "agent": + break + if role not in ("system", "user"): + raise ValueError("unsupported ATIF input source") + copied = step.get("is_copied_context") + if copied is not None and not isinstance(copied, bool): + raise ValueError("ATIF is_copied_context must be a boolean or null") + if copied: + if task_input is None: + raise ValueError("copied ATIF context requires explicit task_input") + return _task_messages(task_input) + content = step.get("message") + blocks: list[dict[str, str]] = [] + if isinstance(content, str): + blocks.append({"type": "text", "text": content}) + elif isinstance(content, list): + for block in content: + if not isinstance(block, dict): + raise ValueError("ATIF message content must be an object") + if block.get("type") != "text" or not isinstance(block.get("text"), str): + raise ValueError("only text ATIF input is supported") + blocks.append({"type": "text", "text": block["text"]}) + else: + raise ValueError("ATIF input message must contain text") + if blocks: + messages.append({"role": role, "content": blocks}) + has_user_input |= role == "user" and any(block["text"].strip() for block in blocks) + return tuple(messages) if has_user_input else _task_messages(task_input) + + +def _task_messages(text: str | None) -> tuple[dict[str, object], ...]: + if text is None: + raise ValueError("missing initial task input; provide task_inputs explicitly") + if not isinstance(text, str) or not text.strip(): + raise ValueError("task input must be a non-empty string") + return ({"role": "user", "content": [{"type": "text", "text": text}]},) diff --git a/tests/sim/test_harbor.py b/tests/sim/test_harbor.py index 6b657a5ab..fe6220833 100644 --- a/tests/sim/test_harbor.py +++ b/tests/sim/test_harbor.py @@ -62,7 +62,9 @@ def write_trial( ) trajectory = { "schema_version": version, - "steps": steps, + "session_id": name, + "agent": {"name": "codex" if codex else "claude-code", "version": "fixture"}, + "steps": [dict(step_id=i, **step) for i, step in enumerate(steps, start=1)], "final_metrics": { "total_prompt_tokens": 100, "total_completion_tokens": 20, diff --git a/tests/sim/test_trajectory.py b/tests/sim/test_trajectory.py new file mode 100644 index 000000000..bbe3e311b --- /dev/null +++ b/tests/sim/test_trajectory.py @@ -0,0 +1,198 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Shared ATIF conversion preserves evidence while isolating task-routing input.""" + +import json +from pathlib import Path + +import pytest + +from switchyard.sim import Dataset, HarborRun, Run, Trajectory, load_harbor +from tests.sim.test_harbor import update, write_trial + + +def atif() -> dict: + return { + "schema_version": "ATIF-v1.7", + "session_id": "session-a", + "agent": {"name": "custom-agent", "version": "1", "model_name": "default-model"}, + "steps": [ + {"step_id": 1, "source": "system", "message": "Instructions."}, + {"step_id": 2, "source": "user", "message": "Original task."}, + { + "step_id": 3, + "source": "agent", + "model_name": "override-model", + "message": "LATER_ANSWER", + "tool_calls": [ + {"tool_call_id": "call-a", "function_name": "shell", "arguments": {"cmd": "ls"}} + ], + "observation": {"results": [{"source_call_id": "call-a", "content": "LATER_TOOL"}]}, + }, + { + "step_id": 4, + "source": "user", + "message": [ + {"type": "image", "source": {"media_type": "image/png", "path": "a.png"}} + ], + }, + ], + "final_metrics": {"total_prompt_tokens": 100, "total_cost_usd": 2.5}, + "continued_trajectory_ref": None, + "extra": {"extension": {"nullable": None, "scores": [0, 0.5]}}, + } + + +def test_full_atif_roundtrip_and_projection_have_independent_ownership() -> None: + source = atif() + expected = json.loads(json.dumps(source)) + trajectory = Trajectory.from_dict(source) + source["steps"][1]["message"] = "MUTATED_SOURCE" + exported = trajectory.to_dict() + assert exported == expected + exported["steps"][2]["tool_calls"][0]["arguments"]["cmd"] = "MUTATED_EXPORT" + exported["extra"]["extension"]["scores"].append(1) + assert trajectory.to_dict() == expected + + messages = trajectory.initial_messages() + assert messages == ( + {"role": "system", "content": [{"type": "text", "text": "Instructions."}]}, + {"role": "user", "content": [{"type": "text", "text": "Original task."}]}, + ) + assert "LATER_" not in json.dumps(messages) + messages[0]["content"][0]["text"] = "MUTATED_PROJECTION" + assert trajectory.to_dict() == expected + assert "LATER_" not in repr(trajectory) + + +def test_projection_requires_explicit_outcomes_and_whole_run_model_identity() -> None: + trajectory = Trajectory.from_dict(atif()) + unknown = trajectory.to_trial(task_id="task-a", trial_id="trial-a", target="fast") + assert unknown.reward is None + assert unknown.cost_usd is None + assert unknown.usage == {} + assert unknown.model is None + assert unknown.task_checksum is None + + usage = {"total_prompt_tokens": 100} + known = trajectory.to_trial( + task_id="task-a", + trial_id="trial-a", + target="fast", + reward=0, + cost_usd=0, + cost_source="custom", + duration_seconds=0, + model="recorded-model", + task_checksum="task-v1", + usage=usage, + ) + usage["total_prompt_tokens"] = 999 + assert known.reward == known.cost_usd == known.duration_seconds == 0 + assert known.cost_source == "custom" + assert known.model == "recorded-model" + assert known.usage == {"total_prompt_tokens": 100} + assert "LATER_" not in repr(known) + with pytest.raises(ValueError, match="reward must be finite"): + trajectory.to_trial( + task_id="task-a", trial_id="trial-a", target="fast", reward=float("nan") + ) + + +def test_custom_converter_feeds_the_same_dataset_and_repeat_accounting() -> None: + def convert(record: dict) -> Trajectory: + return Trajectory.from_dict( + { + "schema_version": "ATIF-v1.7", + "session_id": record["id"], + "agent": {"name": "my-agent", "version": "1"}, + "steps": [{"step_id": 1, "source": "user", "message": record["instruction"]}], + } + ) + + records = [ + {"id": "first", "instruction": "Task.", "reward": 0}, + {"id": "second", "instruction": "Task.", "reward": 1}, + ] + custom = Run( + tuple( + convert(record).to_trial( + task_id="suite/task-a", + trial_id=record["id"], + target="custom", + reward=record["reward"], + ) + for record in records + ) + ) + dataset = Dataset.from_runs({"custom": custom}, input_target="custom") + assert dataset.tasks[0].outcomes["custom"].reward == 0.5 + assert dataset.tasks[0].outcomes["custom"].trials == 2 + assert isinstance(custom, HarborRun) + assert HarborRun is Run + + +@pytest.mark.parametrize("version", [[], {}, None, "ATIF-v2.0"]) +def test_invalid_version_is_rejected_without_printing_recording(version: object) -> None: + with pytest.raises(ValueError, match="schema_version") as error: + Trajectory.from_dict({"schema_version": version, "steps": [], "private": "PRIVATE_TRACE"}) + assert "PRIVATE_TRACE" not in str(error.value) + + +@pytest.mark.parametrize("steps", [None, {}, "PRIVATE_TRACE"]) +def test_invalid_steps_container_is_rejected(steps: object) -> None: + with pytest.raises(ValueError, match="steps must be an array"): + Trajectory.from_dict({"schema_version": "ATIF-v1.7", "steps": steps}) + + +def test_multimodal_input_is_preserved_but_cannot_be_scored_as_text() -> None: + data = atif() + data["steps"] = [data["steps"][-1]] + trajectory = Trajectory.from_dict(data) + assert trajectory.to_dict() == data + with pytest.raises(ValueError, match="only text"): + trajectory.initial_messages() + + +def test_copied_continuation_context_requires_original_task_input() -> None: + data = atif() + data["steps"][1].update(is_copied_context=True, message="SUMMARY_OF_COMPLETED_WORK") + trajectory = Trajectory.from_dict(data) + with pytest.raises(ValueError, match="copied ATIF context"): + trajectory.initial_messages() + messages = trajectory.initial_messages(task_input="Canonical original task.") + assert messages == ( + {"role": "user", "content": [{"type": "text", "text": "Canonical original task."}]}, + ) + assert "SUMMARY" not in json.dumps(messages) + assert trajectory.to_dict() == data + + +@pytest.mark.parametrize("copied", ["false", 1, []]) +def test_malformed_context_flag_is_rejected(copied: object) -> None: + data = atif() + data["steps"][1]["is_copied_context"] = copied + with pytest.raises(ValueError, match="is_copied_context"): + Trajectory.from_dict(data).initial_messages() + + +def test_harbor_uses_the_shared_continuation_guard(tmp_path: Path) -> None: + trial = write_trial(tmp_path) + update( + trial / "agent" / "trajectory.json", + steps=[{"source": "user", "message": "LATER_SUMMARY", "is_copied_context": True}], + ) + with pytest.raises(ValueError, match="copied ATIF context"): + load_harbor(tmp_path, target="fast") + run = load_harbor( + tmp_path, target="fast", task_inputs={"benchmark/task-a": "Canonical original task."} + ) + assert run.trials[0].messages[0]["content"][0]["text"] == "Canonical original task." + + +def test_present_empty_atif_is_not_treated_as_a_missing_file(tmp_path: Path) -> None: + trial = write_trial(tmp_path) + (trial / "agent" / "trajectory.json").write_text("{}") + with pytest.raises(ValueError, match="schema_version"): + load_harbor(tmp_path, target="fast", task_inputs={"benchmark/task-a": "Task."}) From ba256f74c0b12e551b46060b1b458ab66e57f9f0 Mon Sep 17 00:00:00 2001 From: ayushag Date: Tue, 29 Sep 2026 22:59:55 -0700 Subject: [PATCH 05/48] fix(python): preserve decision headers and redact invalid credentials Signed-off-by: ayushag --- crates/switchyard-py/src/runner_bindings.rs | 6 +- crates/switchyard-runner/src/config.rs | 7 ++- tests/test_runner_bindings.py | 70 ++++++++++++++++++++- 3 files changed, 80 insertions(+), 3 deletions(-) diff --git a/crates/switchyard-py/src/runner_bindings.rs b/crates/switchyard-py/src/runner_bindings.rs index 1de632405..05f84349d 100644 --- a/crates/switchyard-py/src/runner_bindings.rs +++ b/crates/switchyard-py/src/runner_bindings.rs @@ -157,7 +157,11 @@ impl PyRunner { let request = Request { llm_request: from_python(request)?, raw_request: None, - metadata: headers.map(|headers| Metadata::from_headers(&headers)), + metadata: headers.map(|headers| { + let mut metadata = Metadata::from_headers(&headers); + metadata.http_headers = Some(headers); + metadata + }), }; let model = request .llm_request diff --git a/crates/switchyard-runner/src/config.rs b/crates/switchyard-runner/src/config.rs index c2de21a5b..231652837 100644 --- a/crates/switchyard-runner/src/config.rs +++ b/crates/switchyard-runner/src/config.rs @@ -4,6 +4,7 @@ //! Version-1 TOML deployment loading for the shared runner. use std::collections::{BTreeMap, BTreeSet, HashMap}; +use std::env::VarError; use std::fs; use std::path::Path; use std::sync::Arc; @@ -680,8 +681,12 @@ fn build_backend( ))); } let api_key = std::env::var(variable).map_err(|error| { + let reason = match error { + VarError::NotPresent => "environment variable not found", + VarError::NotUnicode(_) => "environment variable was not valid Unicode", + }; RunnerError::configuration(format!( - "llm client {client_name} could not read api_key_env {variable}: {error}" + "llm client {client_name} could not read api_key_env {variable}: {reason}" )) })?; if api_key.trim().is_empty() { diff --git a/tests/test_runner_bindings.py b/tests/test_runner_bindings.py index 4b7bd1f57..da8b6ecb5 100644 --- a/tests/test_runner_bindings.py +++ b/tests/test_runner_bindings.py @@ -7,6 +7,7 @@ import asyncio import json +import os from collections.abc import Iterator from dataclasses import dataclass, field from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer @@ -24,6 +25,7 @@ class JudgeStub: url: str = "" calls: list[dict[str, Any]] = field(default_factory=list) + request_headers: list[dict[str, str]] = field(default_factory=list) response_text: str | None = None status: int = 200 started: Event = field(default_factory=Event) @@ -40,6 +42,9 @@ class Handler(BaseHTTPRequestHandler): def do_POST(self) -> None: body = json.loads(self.rfile.read(int(self.headers["Content-Length"]))) stub.calls.append(body) + stub.request_headers.append( + {name.lower(): value for name, value in self.headers.items()} + ) stub.started.set() if stub.release is not None: stub.release.wait(timeout=5) @@ -72,7 +77,11 @@ def do_POST(self) -> None: }, } else: - payload = {"error": {"message": "echoed provider-secret and private prompt"}} + payload = { + "error": { + "message": f"echoed provider-secret and private prompt {self.headers.get('authorization', '')}" + } + } encoded = json.dumps(payload).encode() self.send_response(stub.status) self.send_header("Content-Type", "application/json") @@ -218,6 +227,47 @@ async def test_affinity_and_observations_are_isolated_across_sessions_and_runner assert len(judge.calls) == 4 +@pytest.mark.parametrize("forward_auth", [False, True]) +async def test_caller_headers_are_request_scoped_and_respect_configured_auth( + judge: JudgeStub, forward_auth: bool +) -> None: + source = deployment(judge.url) + if forward_auth: + source = source.replace('api_key_env = "SWITCHYARD_RUNNER_TEST_KEY"', "forward_auth = true") + runner = Runner.from_toml(source) + await asyncio.gather( + *( + runner.decide( + request(), + headers={ + **session(identity), + "authorization": f"Bearer caller-{identity}", + "x-request-id": identity, + }, + ) + for identity in ("first", "second") + ) + ) + assert { + headers.get("x-request-id"): headers.get("authorization") + for headers in judge.request_headers + } == { + identity: f"Bearer caller-{identity}" if forward_auth else "Bearer provider-secret" + for identity in ("first", "second") + } + + judge.status = 401 + with pytest.raises(DecisionError) as caught: + await runner.decide( + request(), + headers={**session("failed"), "authorization": "Bearer caller-failed"}, + ) + assert caught.value.upstream_status == 401 + assert "caller-failed" not in str(caught.value) + assert "provider-secret" not in str(caught.value) + assert "private prompt" not in str(caught.value) + + @pytest.mark.parametrize("advisor", [False, True]) async def test_response_based_route_requires_opt_in_before_calls( judge: JudgeStub, advisor: bool @@ -314,6 +364,24 @@ def test_configuration_diagnostics_name_missing_target_and_environment( assert judge.calls == [] +@pytest.mark.skipif( + not os.supports_bytes_environ, reason="requires byte-valued environment variables" +) +def test_configuration_diagnostics_hide_non_unicode_api_keys( + judge: JudgeStub, monkeypatch: pytest.MonkeyPatch, tmp_path: Path +) -> None: + monkeypatch.setitem(os.environb, b"SWITCHYARD_RUNNER_TEST_KEY", b"provider-secret\xff") + source = deployment(judge.url) + path = tmp_path / "routes.toml" + path.write_text(source) + for load in (lambda: Runner.from_toml(source), lambda: Runner.load(path)): + with pytest.raises(ValueError, match="api_key_env SWITCHYARD_RUNNER_TEST_KEY") as caught: + load() + assert "provider-secret" not in str(caught.value) + assert "not valid" in str(caught.value) + assert judge.calls == [] + + async def test_cancellation_does_not_complete_or_retain_a_decision(judge: JudgeStub) -> None: judge.release = Event() runner = Runner.from_toml(deployment(judge.url)) From dd36d1284a04e89b7ea921f2acd1b1325ff200f9 Mon Sep 17 00:00:00 2001 From: ayushag Date: Tue, 29 Sep 2026 23:11:58 -0700 Subject: [PATCH 06/48] fix(sim): validate and snapshot CLI configuration before output Signed-off-by: ayushag --- docs/simulation.md | 2 + switchyard/sim/__main__.py | 8 ++-- switchyard/sim/evaluate.py | 41 ++++++++++------- tests/sim/test_integration.py | 85 +++++++++++++++++++++++++++++++++++ 4 files changed, 116 insertions(+), 20 deletions(-) diff --git a/docs/simulation.md b/docs/simulation.md index 8e06eab15..fcfae007b 100644 --- a/docs/simulation.md +++ b/docs/simulation.md @@ -268,6 +268,8 @@ This CLI imports Harbor layouts. Use the Python API for custom formats. The output directory must be new. The CLI writes a `manifest.json` with the configuration hash, inputs, package version, and coverage, then flushes each completed row to `results.jsonl`. It writes `report.json` after evaluation returns. +The hash covers the exact TOML bytes loaded. Invalid routes and recorded-model +mismatches fail before output creation, leaving the path available for a corrected run. The deployment source and credentials are not copied. Use `--skip-invalid --intersection` to retain import issues and explicitly evaluate the common valid subset. Exit status is 0 for complete reward coverage, diff --git a/switchyard/sim/__main__.py b/switchyard/sim/__main__.py index 5902e9dea..22a447c05 100644 --- a/switchyard/sim/__main__.py +++ b/switchyard/sim/__main__.py @@ -17,7 +17,7 @@ from switchyard.runner import Runner from . import Dataset, Result, evaluate, load_harbor -from .evaluate import _validate_options +from .evaluate import _aliases, _validate_options, _validate_route def main(argv: list[str] | None = None) -> int: @@ -94,12 +94,14 @@ def main(argv: list[str] | None = None) -> int: dataset = Dataset.from_runs( runs, input_target=args.input_target, intersection=args.intersection ) - runner = Runner.load(args.config) + config_bytes = args.config.read_bytes() + runner = Runner.from_toml(config_bytes.decode("utf-8")) + _validate_route(dataset, runner, args.route, _aliases(aliases)) args.output.mkdir(parents=True, exist_ok=False) manifest = { "schema_version": 1, "switchyard_version": __version__, - "config_sha256": hashlib.sha256(args.config.read_bytes()).hexdigest(), + "config_sha256": hashlib.sha256(config_bytes).hexdigest(), "route": args.route, "runs": {target: str(path) for target, path in paths.items()}, "input_target": args.input_target, diff --git a/switchyard/sim/evaluate.py b/switchyard/sim/evaluate.py index 173ae1feb..c432836c2 100644 --- a/switchyard/sim/evaluate.py +++ b/switchyard/sim/evaluate.py @@ -179,6 +179,29 @@ def _validate_options(concurrency: int, timeout: float) -> None: raise ValueError("timeout must be finite and positive") +def _validate_route( + dataset: Dataset, runner: Runner, route: str, aliases: Mapping[str, str] +) -> dict[str, dict[str, object]]: + """Check route compatibility before workers or CLI artifacts are created.""" + if not dataset.tasks or len({task.task_id for task in dataset.tasks}) != len(dataset.tasks): + raise ValueError("dataset must contain non-empty, unique tasks") + targets = runner.validate_decision_route(route, allow_response=False) + configured = {target.target for target in targets} + if not configured or not configured.issubset(dataset.targets): + raise ValueError( + f"configured targets {sorted(configured)!r} do not match available recorded targets {sorted(dataset.targets)!r}" + ) + return { + target.target: _validate_models( + (trial for task in dataset.tasks for trial in task.trials[target.target]), + target.target, + target.model, + aliases, + ) + for target in targets + } + + async def evaluate( dataset: Dataset, runner: Runner, @@ -207,24 +230,8 @@ async def evaluate( from switchyard.runner import DecisionError _validate_options(concurrency, timeout) - if not dataset.tasks or len({task.task_id for task in dataset.tasks}) != len(dataset.tasks): - raise ValueError("dataset must contain non-empty, unique tasks") - targets = runner.validate_decision_route(route, allow_response=False) - configured = {target.target for target in targets} - if not configured or not configured.issubset(dataset.targets): - raise ValueError( - f"configured targets {sorted(configured)!r} do not match available recorded targets {sorted(dataset.targets)!r}" - ) aliases = _aliases(model_aliases) - model_validation = { - target.target: _validate_models( - (trial for task in dataset.tasks for trial in task.trials[target.target]), - target.target, - target.model, - aliases, - ) - for target in targets - } + model_validation = _validate_route(dataset, runner, route, aliases) run_id = uuid.uuid4().hex report = Report( len(dataset.tasks), diff --git a/tests/sim/test_integration.py b/tests/sim/test_integration.py index f930a1659..153dca662 100644 --- a/tests/sim/test_integration.py +++ b/tests/sim/test_integration.py @@ -3,6 +3,7 @@ """Harbor import through real native decisions and incremental report output.""" +import hashlib import json from pathlib import Path @@ -226,3 +227,87 @@ def test_cli_invalid_options_leave_output_path_available( assert judge.calls == [] assert main(args) == 0 assert (output / "report.json").exists() + + +@pytest.mark.parametrize( + "problem", ["unknown-route", "missing-target", "model-mismatch", "response-route"] +) +def test_cli_route_preflight_leaves_output_path_available( + judge: JudgeStub, tmp_path: Path, problem: str +): + model = "different/model" if problem == "model-mismatch" else "weak/model" + write_trial(tmp_path / "recordings", "task", model, 1) + config = tmp_path / "routes.toml" + config.write_text(deployment(judge.url, escalation=problem == "response-route")) + output = tmp_path / "new-parent" / "evaluation" + route = ( + "missing" + if problem == "unknown-route" + else ("auto" if problem in ("missing-target", "response-route") else "fixed") + ) + args = [ + "--config", + str(config), + "--route", + route, + "--run", + f"weak={tmp_path / 'recordings'}", + "--input-target", + "weak", + "--output", + str(output), + ] + assert main(args) == 2 + assert not output.parent.exists() + assert judge.calls == [] + + args[3] = "fixed" + if problem == "model-mismatch": + args.extend(["--model-alias", "different/model=weak/model"]) + assert main(args) == 0 + assert (output / "report.json").exists() + + +def test_manifest_hash_identifies_the_loaded_configuration_snapshot( + judge: JudgeStub, tmp_path: Path, monkeypatch: pytest.MonkeyPatch +): + write_trial(tmp_path / "recordings", "task", "weak/model", 1) + config = tmp_path / "routes.toml" + original = deployment(judge.url).replace("\n", "\r\n").encode("utf-8") + config.write_bytes(original) + + class ConcurrentEdit: + @staticmethod + def load(path): + runner = Runner.load(path) + config.write_bytes(original + b"\n# edited after loading\n") + return runner + + @staticmethod + def from_toml(source): + runner = Runner.from_toml(source) + config.write_bytes(original + b"\n# edited after loading\n") + return runner + + monkeypatch.setattr("switchyard.sim.__main__.Runner", ConcurrentEdit) + output = tmp_path / "evaluation" + assert ( + main( + [ + "--config", + str(config), + "--route", + "fixed", + "--run", + f"weak={tmp_path / 'recordings'}", + "--input-target", + "weak", + "--output", + str(output), + ] + ) + == 0 + ) + manifest = json.loads((output / "manifest.json").read_text()) + assert config.read_bytes() != original + assert manifest["config_sha256"] == hashlib.sha256(original).hexdigest() From a734f16e41ef24e3abdf5943e5a54ede4f829ac9 Mon Sep 17 00:00:00 2001 From: ayushag Date: Tue, 29 Sep 2026 23:18:57 -0700 Subject: [PATCH 07/48] fix(sim): release routing session state after task decisions Signed-off-by: ayushag --- crates/libsy/src/algorithms/fall_through.rs | 118 +++++++++++++++++--- docs/simulation.md | 3 +- switchyard/sim/evaluate.py | 5 +- tests/sim/test_evaluate.py | 1 + 4 files changed, 112 insertions(+), 15 deletions(-) diff --git a/crates/libsy/src/algorithms/fall_through.rs b/crates/libsy/src/algorithms/fall_through.rs index 8d6f5e71d..b3cf236d0 100644 --- a/crates/libsy/src/algorithms/fall_through.rs +++ b/crates/libsy/src/algorithms/fall_through.rs @@ -40,6 +40,18 @@ struct SessionState { type SessionStates = Mutex>>; +/// Releases completed-session state even when the routing future is cancelled. +struct FinalSession<'a, S> { + states: &'a SessionStates, + session: String, +} + +impl Drop for FinalSession<'_, S> { + fn drop(&mut self) { + self.states.lock().remove(&self.session); + } +} + /// Delete sessions that have been inactive this long. Catches sessions that did not terminate /// cleanly. /// A user resuming a deleted session is not fatal. Algorithms will be missing some context @@ -108,17 +120,20 @@ where /// Executes the processor and classifier sequence for wrappers and the trait entrypoint. pub(crate) async fn execute(&self, driver: Driver, request: Request) -> Result { self.start_cleanup_task(); - let session = session_id(&request); let session_final = request .metadata .as_ref() .and_then(|metadata| metadata.session_final) == Some(true); - let result = self.execute_session(driver, request).await; - if session_final && let Some(session) = session.as_deref() { - self.remove_session(session); - } - result + let _final_session = if session_final { + self.session_states + .as_deref() + .zip(session_id(&request)) + .map(|(states, session)| FinalSession { states, session }) + } else { + None + }; + self.execute_session(driver, request).await } /// Starts one cleanup task on the first request handled by a stateful router. @@ -173,13 +188,6 @@ where } } - /// Drops retained routing state once the host marks a session complete. - fn remove_session(&self, session: &str) { - if let Some(states) = &self.session_states { - states.lock().remove(session); - } - } - /// Returns this request's retained state without holding the registry lock. fn session_state(&self, request: &Request) -> Option>> { let states = self.session_states.as_ref()?; @@ -297,6 +305,9 @@ where #[cfg(test)] mod tests { use super::*; + use futures::StreamExt; + use tokio::sync::Notify; + use crate::Classification; use crate::algorithms::llm_class::DefaultCategoryClassifier; use crate::algorithms::util::prompts; @@ -738,6 +749,7 @@ mod tests { ..request() }; let (final_turn, _) = run_request(&router, final_request, echo()).await?; + assert!(router.session_states.as_ref().unwrap().lock().is_empty()); let (restarted_session, _) = run_turn(&router, echo()).await?; let (second_session, _) = run_request( &router, @@ -797,6 +809,86 @@ mod tests { assert!(!states.contains_key("session-1")); } + #[tokio::test] + async fn final_sessions_are_removed_when_model_calls_are_cancelled_or_fail() -> Result<()> { + struct DropSignal(Arc); + + impl Drop for DropSignal { + fn drop(&mut self) { + self.0.notify_one(); + } + } + + struct CallingClassifier(Arc); + + #[async_trait] + impl Classifier for CallingClassifier { + async fn score( + &self, + _state: &mut u32, + request: &mut Request, + driver: &Driver, + ) -> Result<(Classification, Option)> { + let _dropped = DropSignal(Arc::clone(&self.0)); + driver + .call_model(request.clone(), vec!["strong".into()]) + .await?; + Ok((Classification::Scores(vec![score("strong", 1.0)]), None)) + } + } + + for final_session in [false, true] { + for failed_call in [false, true] { + let stopped = Arc::new(Notify::new()); + let router = Arc::new( + FallThrough::::new_with_state() + .with_classifier(Arc::new(CallingClassifier(Arc::clone(&stopped)))), + ); + let models = Arc::new(crate::RuntimeModels::from(category_models( + Category::Any, + &["strong"], + ))); + for index in 0..8 { + let request = Request { + metadata: Some(Metadata { + session_id: Some(format!("session-{index}")), + session_final: Some(final_session), + ..Metadata::default() + }), + ..request() + }; + let outstanding_call = if failed_call { + let result = crate::drive( + router.clone(), + request, + Arc::clone(&models), + |call| async move { call.fail(test_error("provider failure")) }, + ) + .await; + assert!(result.is_err()); + None + } else { + let mut stream = router.clone().run_stream(request, Arc::clone(&models)); + let Some(Ok(crate::Step::CallModel(call))) = stream.next().await else { + panic!("classifier must request a model call"); + }; + // Keep the reply sender alive so only stream cancellation can + // interrupt the algorithm's wait for this response. + drop(stream); + Some(call) + }; + tokio::time::timeout(Duration::from_secs(1), stopped.notified()) + .await + .expect("algorithm task must release the cancelled classifier"); + drop(outstanding_call); + let retained = router.session_states.as_ref().unwrap().lock().len(); + assert_eq!(retained, if final_session { 0 } else { index + 1 }); + } + } + } + Ok(()) + } + #[test] fn cleanup_removes_only_inactive_idle_sessions() { let router = FallThrough::::new_with_state(); diff --git a/docs/simulation.md b/docs/simulation.md index fcfae007b..c1fe1bca9 100644 --- a/docs/simulation.md +++ b/docs/simulation.md @@ -76,7 +76,8 @@ summary = asyncio.run(main()) Create a fresh `Runner` for each independent experiment. It owns routing state, including affinity and random-number generators. The evaluator creates a distinct -session ID for each task. Use `concurrency=1` with a seeded random route when task +session ID for each task and marks its decision as the final turn so native +per-session state can be released. Use `concurrency=1` with a seeded random route when task order must be reproducible. Concurrent scheduling can change which task receives each random draw. diff --git a/switchyard/sim/evaluate.py b/switchyard/sim/evaluate.py index c432836c2..5e4fec23a 100644 --- a/switchyard/sim/evaluate.py +++ b/switchyard/sim/evaluate.py @@ -247,7 +247,10 @@ async def route_task(index: int, task: Task) -> Result: decision = await asyncio.wait_for( runner.decide( {"model": route, "messages": copy.deepcopy(list(task.messages))}, - headers={"x-switchyard-session-id": f"sim-{run_id}-{index}"}, + headers={ + "x-switchyard-session-id": f"sim-{run_id}-{index}", + "x-switchyard-session-final": "true", + }, allow_response=False, ), timeout=timeout, diff --git a/tests/sim/test_evaluate.py b/tests/sim/test_evaluate.py index 90258a7ff..45b461383 100644 --- a/tests/sim/test_evaluate.py +++ b/tests/sim/test_evaluate.py @@ -75,6 +75,7 @@ async def test_bounded_concurrency_and_progressive_results(): assert report.to_dict()["counts"]["scored"] == 11 assert all(row.routing_cost_usd == 0 for row in rows) assert len({headers["x-switchyard-session-id"] for headers in runner.headers}) == 11 + assert all(headers.get("x-switchyard-session-final") == "true" for headers in runner.headers) async def test_sessions_are_isolated_between_evaluations(): From 6394e144206b004a28179ba9b42c91e2a44809e2 Mon Sep 17 00:00:00 2001 From: ayushag Date: Tue, 29 Sep 2026 23:32:28 -0700 Subject: [PATCH 08/48] fix(runner): reject ambiguous completion target identities Signed-off-by: ayushag --- crates/switchyard-runner/src/config.rs | 88 +++++++++++++++++++++++- crates/switchyard-runner/src/route.rs | 93 ++++++++++++++++++++++++-- crates/switchyard-server/src/lib.rs | 6 +- docs/reference/toml_schema.md | 6 ++ docs/simulation.md | 6 ++ tests/test_runner_bindings.py | 51 ++++++++++++++ 6 files changed, 242 insertions(+), 8 deletions(-) diff --git a/crates/switchyard-runner/src/config.rs b/crates/switchyard-runner/src/config.rs index 231652837..652ac1ceb 100644 --- a/crates/switchyard-runner/src/config.rs +++ b/crates/switchyard-runner/src/config.rs @@ -239,11 +239,22 @@ impl DeploymentConfig { self.build_anthropic_auxiliary_target(config, &clients); let responses_auxiliary_target = self.build_responses_auxiliary_target(config, &clients); - let decision_targets = config + let decision_targets: Vec<_> = config .routing_target_names() .into_iter() .filter_map(|name| self.decision_target(name)) .collect(); + let mut names_by_model = HashMap::new(); + for target in &decision_targets { + if let Some(first_name) = names_by_model.insert(&target.model, &target.target) + && first_name != &target.target + { + return Err(RunnerError::configuration(format!( + "route {route_name} completion targets {first_name} and {} both use model {}; routing decisions identify models, so use distinct model ids within this route or put these targets in separate routes", + target.target, target.model + ))); + } + } let names = config .algorithm .runtime_model_names(route_name) @@ -1565,6 +1576,81 @@ target = "smart" Ok(()) } + #[test] + fn completion_target_aliases_in_one_route_are_rejected() { + const ALIASED_TARGETS: &str = r#" +schema_version = 1 +[llm_clients.primary] +format = "openai_chat" +base_url = "https://example.test/v1" +[targets.first] +id = "shared/model" +llm_client = "primary" +[targets.second] +id = "shared/model" +llm_client = "primary" +[targets.judge] +id = "judge/model" +llm_client = "primary" +"#; + for algorithm in [ + "type = \"random\"\ntargets = [\"first\", \"second\"]\nweights = [1, 99]", + "type = \"random\"\ntargets = [\"first\", \"second\"]\nweights = [0, 1]", + "type = \"llm_classifier\"\nclassifier_target = \"judge\"\nstrong_target = \"second\"\nweak_target = \"first\"\nbase_threshold = 0.5", + "type = \"passthrough\"\ntarget = \"first\"\n[routes.shared.subagents]\ntype = \"passthrough\"\ntarget = \"second\"", + ] { + let configured = format!( + "{ALIASED_TARGETS}\n[routes.shared]\nid = \"switchyard/shared\"\n{algorithm}" + ); + let message = error_message(&configured); + assert!( + message.contains("route shared") + && message.contains("completion targets first and second") + && message.contains("shared/model") + && message.contains("distinct model ids"), + "{algorithm}: {message}" + ); + } + } + + #[test] + fn completion_target_can_be_reused_by_parent_and_subagent() -> RunnerResult<()> { + let configured = format!( + "{VALID_CONFIG}\n[routes.passthrough.subagents]\ntype = \"passthrough\"\ntarget = \"weak\"" + ); + let runner = runner_from_toml(&configured)?; + let route = runner + .route("switchyard/passthrough") + .expect("passthrough route should exist"); + assert_eq!( + route + .decision_targets() + .iter() + .map(|target| target.target.as_str()) + .collect::>(), + ["weak", "weak"] + ); + Ok(()) + } + + #[test] + fn routing_judge_can_share_a_completion_model_id() -> RunnerResult<()> { + let configured = VALID_CONFIG.replace( + "id = \"classifier/model\"\nllm_client = \"primary\"", + "id = \"weak/model\"\nllm_client = \"anthropic\"", + ); + let runner = runner_from_toml(&configured)?; + let route = runner + .route("switchyard/classifier") + .expect("classifier route should exist"); + assert_eq!( + route.models().models_for(&Category::Judge), + route.models().models_for(&Category::Efficient) + ); + assert_eq!(route.decision_targets().len(), 2); + Ok(()) + } + #[test] fn accepts_same_model_id_on_different_llm_clients() -> RunnerResult<()> { // Separate routes may serve the same model through different clients. diff --git a/crates/switchyard-runner/src/route.rs b/crates/switchyard-runner/src/route.rs index d510e0526..0b1d2ca1c 100644 --- a/crates/switchyard-runner/src/route.rs +++ b/crates/switchyard-runner/src/route.rs @@ -176,12 +176,16 @@ impl Route { self.caller_auth } - /// Resolves a selected model to this route's non-secret target metadata. + /// Resolves a selected model when its configured target name is unambiguous. pub(crate) fn decision_target(&self, model: &ModelId) -> Option { - self.decision_targets + let mut matches = self + .decision_targets .iter() - .find(|target| target.model == *model) - .cloned() + .filter(|target| target.model == *model); + let target = matches.next()?; + matches + .all(|candidate| candidate.target == target.target) + .then(|| target.clone()) } /// Returns the models grouped for one algorithm execution. @@ -262,3 +266,84 @@ impl Route { .map_err(Into::into) } } + +#[cfg(test)] +mod tests { + use libsy::Passthrough; + use switchyard_protocol::Category; + + use super::*; + use crate::{DecisionDescription, Runner}; + + async fn describe_programmatic_decision( + targets: &[(&str, &str)], + ) -> Result, RunnerError> { + let route = Route::new( + Arc::new(Passthrough), + ClientRouter::new(Default::default()), + None, + ModelCapabilities::default(), + None, + None, + targets + .iter() + .map(|(target, model)| DecisionTarget { + target: (*target).to_string(), + model: (*model).into(), + format: WireFormat::OpenAiChat, + base_url: "http://localhost/v1".to_string(), + extra_body: Default::default(), + }) + .collect(), + RuntimeModels::new( + [( + Category::Any, + vec!["selected/model".into(), "fallback/model".into()], + )] + .into(), + ), + ); + let runner = Runner::new(vec![("auto".into(), route)]); + let outcome = runner + .route("auto") + .unwrap() + .decide(Request::default()) + .await?; + Ok(runner.describe_decision(&"auto".into(), &outcome)) + } + + #[tokio::test] + async fn programmatic_decision_rejects_ambiguous_selected_and_fallback_targets() + -> Result<(), RunnerError> { + for model in ["selected/model", "fallback/model"] { + let description = describe_programmatic_decision(&[ + ("selected", "selected/model"), + ("fallback", "fallback/model"), + ("alias", model), + ]) + .await?; + assert!( + description.is_none(), + "ambiguous model {model} must not resolve to its first target" + ); + } + Ok(()) + } + + #[tokio::test] + async fn programmatic_decision_accepts_repeated_references_to_one_target() + -> Result<(), RunnerError> { + let description = describe_programmatic_decision(&[ + ("unrelated", "other/model"), + ("selected", "selected/model"), + ("selected", "selected/model"), + ("fallback", "fallback/model"), + ]) + .await? + .expect("each selected model identifies one target"); + assert_eq!(description.selected.target, "selected"); + assert_eq!(description.fallbacks.len(), 1); + assert_eq!(description.fallbacks[0].target, "fallback"); + Ok(()) + } +} diff --git a/crates/switchyard-server/src/lib.rs b/crates/switchyard-server/src/lib.rs index fe5bdef83..ca674f523 100644 --- a/crates/switchyard-server/src/lib.rs +++ b/crates/switchyard-server/src/lib.rs @@ -760,9 +760,9 @@ async fn decision( }; match state.decision_response(&route_model, &outcome, response) { Some(response) => Json(response).into_response(), - None => { - server_error("routing outcome contains a model with no callable target configuration") - } + None => server_error( + "routing outcome contains a model with no unambiguous callable target configuration", + ), } } diff --git a/docs/reference/toml_schema.md b/docs/reference/toml_schema.md index 8bf382a08..0b5f905e9 100644 --- a/docs/reference/toml_schema.md +++ b/docs/reference/toml_schema.md @@ -112,6 +112,12 @@ This includes completion targets and targets used for judging or classification. same-model targets on different clients within a route because execution is keyed by model ID. Separate routes may use the same model ID on different clients. +Distinct completion target keys within one route must have distinct model IDs. +Routing decisions identify models, so aliases sharing an ID would make selected +and fallback target identity ambiguous. The same target key may be reused by the +parent and subagent routes. Judge-only targets may share a completion model ID +when the client and prompt rules below permit it. + Each selected or fallback target is prepared from the routed request independently. A prompt configured for one target is therefore not carried into another target's fallback request. Judge-only, classifier-only, and reviewer-only targets are not completion destinations and do not diff --git a/docs/simulation.md b/docs/simulation.md index c1fe1bca9..9d49a268d 100644 --- a/docs/simulation.md +++ b/docs/simulation.md @@ -88,6 +88,12 @@ to `evaluate` or `score` when the two systems name the same model differently. A the library does not guess equivalence by trimming model names. Missing model metadata remains visible as unverified coverage. +Distinct completion targets in one route must have distinct model IDs. Native +decisions identify models, so two target keys sharing an ID cannot be scored +separately; configuration loading rejects that ambiguity. Reuse one target key +for the same candidate, or configure distinct served model IDs. Separate routes +can still use different target keys for the same model. + ## Use ATIF or custom recordings `Trajectory.from_dict(data)` takes an owned copy of an ATIF document. diff --git a/tests/test_runner_bindings.py b/tests/test_runner_bindings.py index da8b6ecb5..606d7e8f8 100644 --- a/tests/test_runner_bindings.py +++ b/tests/test_runner_bindings.py @@ -268,6 +268,57 @@ async def test_caller_headers_are_request_scoped_and_respect_configured_auth( assert "private prompt" not in str(caught.value) +@pytest.mark.parametrize("kind", ["classifier", "random", "random-zero", "random-reversed"]) +def test_ambiguous_completion_target_aliases_fail_before_calls( + judge: JudgeStub, tmp_path: Path, kind: str +) -> None: + source = deployment(judge.url).replace('id = "strong/model"', 'id = "weak/model"') + if kind != "classifier": + targets = '["strong", "weak"]' if kind == "random-reversed" else '["weak", "strong"]' + weights = "[0, 1]" if kind == "random-zero" else "[1, 99]" + source = ( + source.split("[routes.classifier]")[0] + + f""" +[routes.random] +id = "auto" +type = "random" +targets = {targets} +weights = {weights} +seed = 1 +""" + ) + path = tmp_path / "routes.toml" + path.write_text(source) + for load in (lambda: Runner.from_toml(source), lambda: Runner.load(path)): + with pytest.raises(ValueError, match="completion targets.*model"): + load() + assert judge.calls == [] + + +async def test_same_model_aliases_in_separate_fixed_routes_keep_target_identity( + judge: JudgeStub, +) -> None: + source = deployment(judge.url).split("[routes.classifier]")[0] + source = source.replace('id = "strong/model"', 'id = "weak/model"') + for target in ("weak", "strong"): + source += f""" +[routes.{target}] +id = "route-{target}" +type = "passthrough" +target = "{target}" +""" + runner = Runner.from_toml(source) + for target in ("weak", "strong"): + selected = await runner.decide(request(model=f"route-{target}")) + assert selected.selected.target == target + assert selected.selected.model == "weak/model" + assert [entry.target for entry in runner.validate_decision_route(f"route-{target}")] == [ + target + ] + assert selected.calls == [] + assert judge.calls == [] + + @pytest.mark.parametrize("advisor", [False, True]) async def test_response_based_route_requires_opt_in_before_calls( judge: JudgeStub, advisor: bool From 01eb72b2b3eb98e29ae65adaafab09e2c539ed53 Mon Sep 17 00:00:00 2001 From: ayushag Date: Tue, 29 Sep 2026 23:43:29 -0700 Subject: [PATCH 09/48] docs(sim): clarify recovery and input ownership Signed-off-by: ayushag --- docs/simulation.md | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/docs/simulation.md b/docs/simulation.md index 9d49a268d..d71ae410e 100644 --- a/docs/simulation.md +++ b/docs/simulation.md @@ -280,9 +280,11 @@ mismatches fail before output creation, leaving the path available for a correct The deployment source and credentials are not copied. Use `--skip-invalid --intersection` to retain import issues and explicitly evaluate the common valid subset. Exit status is 0 for complete reward coverage, -1 for an incomplete evaluation, and 2 for invalid inputs or configuration. +1 for an incomplete evaluation, and 2 for invalid inputs, configuration, or file errors. An interruption exits 130 and preserves the manifest and completed result rows once those files exist; there may be no final report. Automatic resume is not implemented. +If an output write fails, the last JSONL line may be incomplete. Earlier complete +lines remain usable; check the exit status and final report before treating a run as complete. Use `--model-alias RECORDED=CONFIGURED` for an explicit model-ID equivalence. The Python `on_result` callback runs after each completed task. It can update a @@ -302,6 +304,10 @@ results without retaining full trajectories or result rows. Import processes one trajectory file at a time; paired task inputs and small outcome records stay in memory. +Treat projected `Trial`, `Task`, and `Dataset` records as read-only. Their frozen +dataclasses still contain mutable nested mappings. If you change recorded input, +create new trials and rebuild the dataset so pairing is validated again. + Native `Runner.load` and `Runner.from_toml` use Switchyard's configuration parser. `await runner.decide(normalized_request, headers=...)` returns configured selected and fallback targets, the existing `RoutingOutcome`, completed-call observations, From adb258b5fe917204e401dc0c7d8d5e735c5367a7 Mon Sep 17 00:00:00 2001 From: ayushag Date: Wed, 30 Sep 2026 00:11:08 -0700 Subject: [PATCH 10/48] fix(python): join native decision workers on completion Signed-off-by: ayushag --- crates/switchyard-py/src/runner_bindings.rs | 239 +++++++++++++++++- switchyard_rust/runner.py | 51 +++- tests/test_runner_bindings.py | 266 ++++++++++++++++++++ 3 files changed, 548 insertions(+), 8 deletions(-) diff --git a/crates/switchyard-py/src/runner_bindings.rs b/crates/switchyard-py/src/runner_bindings.rs index 05f84349d..2c09677b2 100644 --- a/crates/switchyard-py/src/runner_bindings.rs +++ b/crates/switchyard-py/src/runner_bindings.rs @@ -4,15 +4,21 @@ //! Native configured routing decisions for Python hosts. use std::collections::HashMap; +use std::future::Future; use std::path::PathBuf; -use std::sync::{Arc, Mutex}; +use std::pin::Pin; +use std::sync::{Arc, Condvar, Mutex}; +use std::task::{Context, Poll}; use pyo3::create_exception; use pyo3::exceptions::{PyRuntimeError, PyValueError}; use pyo3::prelude::*; +use pyo3_async_runtimes::TaskLocals; +use pyo3_async_runtimes::generic::{ContextExt, Runtime}; use switchyard_llm_client::{LlmCallObservation, RunObservation, RunObserver}; use switchyard_protocol::{Metadata, ModelId, Request}; use switchyard_runner::{DecisionTarget, Route, Runner, RunnerError}; +use tokio::sync::oneshot; use crate::libsy_bindings::{PyRoutingOutcome, header_map_from_python, outcome_to_python}; use crate::py_serde::{from_python, to_python}; @@ -99,6 +105,142 @@ struct PyRunner { inner: Arc, } +#[derive(Default)] +struct BridgeTasks { + pending: Mutex, + finished: Condvar, +} + +impl BridgeTasks { + fn track(self: &Arc) -> BridgeTask { + *self.pending.lock().expect("bridge task lock poisoned") += 1; + BridgeTask(Arc::clone(self)) + } + + fn wait(&self) { + let mut pending = self.pending.lock().expect("bridge task lock poisoned"); + while *pending != 0 { + pending = self + .finished + .wait(pending) + .expect("bridge task lock poisoned"); + } + } +} + +struct BridgeTask(Arc); + +impl Drop for BridgeTask { + fn drop(&mut self) { + let mut pending = self.0.pending.lock().expect("bridge task lock poisoned"); + *pending -= 1; + if *pending == 0 { + self.0.finished.notify_all(); + } + } +} + +// Field order matters: drop the future/closure and its Python references before +// announcing that this bridge task is finished, including during unwinding. +struct Tracked { + inner: T, + _task: BridgeTask, +} + +impl Future for Tracked>> { + type Output = F::Output; + + fn poll(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll { + self.inner.as_mut().poll(cx) + } +} + +impl Tracked { + fn run(self) { + (self.inner)(); + } +} + +tokio::task_local! { + static BRIDGE_TASKS: Arc; + static BRIDGE_LOCALS: TaskLocals; +} + +/// Use the existing PyO3 bridge while tracking its detached completion tasks. +struct BridgeRuntime; + +impl Runtime for BridgeRuntime { + type JoinError = tokio::task::JoinError; + type JoinHandle = tokio::task::JoinHandle<()>; + + fn spawn(future: F) -> Self::JoinHandle + where + F: Future + Send + 'static, + { + let tasks = BRIDGE_TASKS.with(Arc::clone); + let task = tasks.track(); + pyo3_async_runtimes::tokio::get_runtime().spawn(Tracked { + inner: Box::pin(BRIDGE_TASKS.scope(tasks, future)), + _task: task, + }) + } + + fn spawn_blocking(function: F) -> Self::JoinHandle + where + F: FnOnce() + Send + 'static, + { + let work = Tracked { + inner: function, + _task: BRIDGE_TASKS.with(|tasks| tasks.track()), + }; + pyo3_async_runtimes::tokio::get_runtime().spawn_blocking(move || work.run()) + } +} + +impl ContextExt for BridgeRuntime { + fn scope(locals: TaskLocals, future: F) -> Pin + Send>> + where + F: Future + Send + 'static, + { + Box::pin(BRIDGE_LOCALS.scope(locals, future)) + } + + fn get_task_locals() -> Option { + BRIDGE_LOCALS.try_with(Clone::clone).ok() + } +} + +/// Cancel routing without cancelling the Python completion bridge. +#[pyclass] +struct DecisionCancellation { + sender: Option>, + tasks: Arc, +} + +#[pymethods] +impl DecisionCancellation { + fn cancel(&mut self) { + if let Some(sender) = self.sender.take() { + let _ = sender.send(()); + } + } + + fn wait(&self, py: Python<'_>) { + // Called from a Python executor thread after result delivery. Release the + // GIL while the bridge finishes using Python and drops its references. + py.detach(|| self.tasks.wait()); + } + + fn is_finished(&self) -> bool { + *self + .tasks + .pending + .lock() + .expect("bridge task lock poisoned") + == 0 + } +} + #[pymethods] impl PyRunner { /// Load and validate a native deployment TOML file without starting a server. @@ -147,12 +289,27 @@ impl PyRunner { /// requests whose shared session state must remain ordered. #[pyo3(signature = (request, *, headers=None, allow_response=false))] fn decide<'py>( + slf: PyRef<'py, Self>, + py: Python<'py>, + request: &Bound<'py, PyAny>, + headers: Option>, + allow_response: bool, + ) -> PyResult> { + // Start routing only when the coroutine runs, so cancellation before its + // first step cannot leave an unowned native request running. + py.import("switchyard_rust.runner")? + .getattr("_decide")? + .call1((slf, request, headers, allow_response)) + } + + #[pyo3(signature = (request, *, headers=None, allow_response=false))] + fn _start_decision<'py>( &self, py: Python<'py>, request: &Bound<'_, PyAny>, headers: Option>, allow_response: bool, - ) -> PyResult> { + ) -> PyResult<(Bound<'py, PyAny>, DecisionCancellation)> { let headers = headers.as_ref().map(header_map_from_python).transpose()?; let request = Request { llm_request: from_python(request)?, @@ -172,7 +329,10 @@ impl PyRunner { decision_route(&self.inner, model, allow_response)?; let model = ModelId::from(model); let runner = Arc::clone(&self.inner); - pyo3_async_runtimes::tokio::future_into_py(py, async move { + let (sender, cancelled) = oneshot::channel(); + let tasks = Arc::new(BridgeTasks::default()); + let locals = pyo3_async_runtimes::tokio::get_current_locals(py)?; + let future = async move { let observations = Arc::new(Mutex::new(Observations::default())); let captured = Arc::clone(&observations); let observer: RunObserver = Arc::new(move |event| { @@ -188,7 +348,11 @@ impl PyRunner { } }); let route = decision_route(&runner, model.as_str(), allow_response)?; - let result = route.decide_with_observer(request, Some(observer)).await; + let result = tokio::select! { + biased; + _ = cancelled => return Ok(None), + result = route.decide_with_observer(request, Some(observer)) => result, + }; let observations = std::mem::take(&mut *observations.lock().expect("observation lock poisoned")); Python::attach(|py| { @@ -230,7 +394,20 @@ impl PyRunner { }, ) }) - }) + .map(Some) + }; + let future = BRIDGE_TASKS.sync_scope(Arc::clone(&tasks), || { + pyo3_async_runtimes::generic::future_into_py_with_locals::( + py, locals, future, + ) + })?; + Ok(( + future, + DecisionCancellation { + sender: Some(sender), + tasks, + }, + )) } } @@ -295,3 +472,55 @@ pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { runner_module.add_class::()?; module.add_submodule(&runner_module) } + +#[cfg(test)] +mod tests { + use std::sync::atomic::{AtomicBool, Ordering}; + + use super::*; + + struct Payload { + tasks: Arc, + dropped: Arc, + } + + impl Drop for Payload { + fn drop(&mut self) { + assert_eq!(*self.tasks.pending.lock().unwrap(), 1); + self.dropped.store(true, Ordering::SeqCst); + } + } + + #[test] + fn bridge_completion_follows_payload_drop_when_unpolled_or_panicking() { + for panic in [false, true] { + let tasks = Arc::new(BridgeTasks::default()); + let dropped = Arc::new(AtomicBool::new(false)); + let payload = Payload { + tasks: Arc::clone(&tasks), + dropped: Arc::clone(&dropped), + }; + if panic { + let work = Tracked { + inner: move || { + let _payload = payload; + panic!("bridge worker failed"); + }, + _task: tasks.track(), + }; + assert!(std::panic::catch_unwind(|| work.run()).is_err()); + } else { + let future = Tracked { + inner: Box::pin(async move { + let _payload = payload; + std::future::pending::<()>().await; + }), + _task: tasks.track(), + }; + drop(future); + } + assert!(dropped.load(Ordering::SeqCst)); + assert_eq!(*tasks.pending.lock().unwrap(), 0); + } + } +} diff --git a/switchyard_rust/runner.py b/switchyard_rust/runner.py index 38f1c12d2..09a7d3b89 100644 --- a/switchyard_rust/runner.py +++ b/switchyard_rust/runner.py @@ -5,9 +5,10 @@ from __future__ import annotations +import asyncio from collections.abc import Mapping from os import PathLike -from typing import TYPE_CHECKING, Any, final +from typing import TYPE_CHECKING, Any, cast, final from switchyard_rust._native import load_native @@ -79,8 +80,8 @@ class Runner: Give independent tasks distinct session headers. Process turns sharing a session in order. Use a separate runner for independent experiments. - Cancelling a decision cancels local routing; a provider may still finish - or bill an already submitted request. + Cancelling a decision waits for local routing and its Python bridge to + stop; a provider may still finish or bill an already submitted request. """ @staticmethod @@ -106,6 +107,50 @@ async def decide( ... +async def _decide( + runner: Any, + request: Mapping[str, object], + headers: Mapping[str, str] | None, + allow_response: bool, +) -> Decision: + future, cancellation = runner._start_decision( + request, headers=headers, allow_response=allow_response + ) + cancelled = False + try: + return cast("Decision", await asyncio.shield(future)) + except asyncio.CancelledError: + cancelled = True + cancellation.cancel() + # The PyO3 bridge attaches to Python after native routing finishes. Keep + # its future alive and uncancelled until that completion has been delivered, + # including when shutdown cancels this coroutine more than once. + await _drain(future) + if not future.cancelled(): + future.exception() + raise + finally: + if future.done() and not cancellation.is_finished(): + # Delivery can precede the bridge worker's exit: call_soon_threadsafe + # releases the GIL while waking the loop. Join that remaining cleanup. + join = asyncio.get_running_loop().run_in_executor(None, cancellation.wait) + interrupted = await _drain(join) + join.result() + if interrupted is not None and not cancelled: + raise interrupted + + +async def _drain(future: asyncio.Future[Any]) -> asyncio.CancelledError | None: + interrupted = None + while not future.done(): + try: + await asyncio.wait({future}) + except asyncio.CancelledError as error: + if interrupted is None: + interrupted = error + return interrupted + + def __getattr__(name: str) -> object: if name in _EXPORTS: native: Any = load_native() diff --git a/tests/test_runner_bindings.py b/tests/test_runner_bindings.py index 606d7e8f8..982ad0afa 100644 --- a/tests/test_runner_bindings.py +++ b/tests/test_runner_bindings.py @@ -8,6 +8,7 @@ import asyncio import json import os +import sys from collections.abc import Iterator from dataclasses import dataclass, field from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer @@ -19,6 +20,7 @@ from switchyard.libsy import RoutingOutcome from switchyard.runner import DecisionError, Runner +from switchyard_rust.runner import _decide @dataclass @@ -31,6 +33,8 @@ class JudgeStub: started: Event = field(default_factory=Event) release: Event | None = None completed: Event = field(default_factory=Event) + wait_for_disconnect: bool = False + disconnected: Event = field(default_factory=Event) @pytest.fixture @@ -46,6 +50,16 @@ def do_POST(self) -> None: {name.lower(): value for name, value in self.headers.items()} ) stub.started.set() + if stub.wait_for_disconnect: + self.connection.settimeout(5) + try: + if self.rfile.read(1) == b"": + stub.disconnected.set() + except ConnectionResetError: + stub.disconnected.set() + finally: + stub.completed.set() + return if stub.release is not None: stub.release.wait(timeout=5) verdict = { @@ -448,3 +462,255 @@ async def test_cancellation_does_not_complete_or_retain_a_decision(judge: JudgeS assert decision.selected.target == "strong" assert len(decision.calls) == 1 assert [call["model"] for call in judge.calls] == ["judge/model", "judge/model"] + + +async def test_cancellation_before_first_step_does_not_start_native_work(judge: JudgeStub) -> None: + runner = Runner.from_toml(deployment(judge.url)) + task = asyncio.create_task(runner.decide(request(), headers=session("never-started"))) + task.cancel() + with pytest.raises(asyncio.CancelledError): + await task + decision = await runner.decide(request(), headers=session("subsequent")) + assert decision.selected.target == "weak" + assert [call["model"] for call in judge.calls] == ["judge/model"] + + +@pytest.mark.parametrize("completion_error", [False, True]) +async def test_repeated_cancellation_drains_bridge_before_propagating( + completion_error: bool, +) -> None: + bridge: asyncio.Future[None] = asyncio.get_running_loop().create_future() + started = asyncio.Event() + cancelled = asyncio.Event() + + class PendingRunner: + def _start_decision(self, *args: object, **kwargs: object) -> tuple[object, object]: + started.set() + return bridge, self + + def cancel(self) -> None: + cancelled.set() + + def wait(self) -> None: + assert bridge.done() + + def is_finished(self) -> bool: + return False + + task = asyncio.create_task(_decide(PendingRunner(), request(), None, False)) + await started.wait() + task.cancel("original cancellation") + await cancelled.wait() + assert not task.done() + assert not bridge.cancelled() + task.cancel("repeated cancellation") + # Advance the event loop once so the repeated cancellation reaches the drain. + await asyncio.sleep(0) + assert not task.done() + assert not bridge.cancelled() + if completion_error: + bridge.set_exception(RuntimeError("native failure while cancellation was draining")) + else: + bridge.set_result(None) + with pytest.raises(asyncio.CancelledError) as caught: + await task + # Python 3.11 began propagating cancellation messages to task awaiters. + if sys.version_info >= (3, 11): + assert str(caught.value) == "original cancellation" + assert bridge.done() + assert not bridge.cancelled() + + +@pytest.mark.parametrize("outcome", ["success", "error", "cancel"]) +async def test_decision_waits_for_completion_worker_to_exit( + judge: JudgeStub, monkeypatch: pytest.MonkeyPatch, outcome: str +) -> None: + judge.release = Event() + if outcome == "error": + judge.status = 503 + runner = Runner.from_toml(deployment(judge.url)) + task = asyncio.create_task(runner.decide(request())) + assert await asyncio.to_thread(judge.started.wait, 5) + + loop = asyncio.get_running_loop() + if not hasattr(loop, "_write_to_self"): + task.cancel() + await asyncio.gather(task, return_exceptions=True) + pytest.skip("requires asyncio's socket wakeup implementation") + write_to_self = loop._write_to_self + entered = Event() + release = Event() + returned = Event() + finished_before_worker_exit: list[bool] = [] + + def gated_write() -> None: + entered.set() + write_to_self() + assert release.wait(5) + + def observe_completion() -> None: + if entered.wait(5): + # The future's completion callback can run while its worker is still + # inside call_soon_threadsafe. The public task must remain pending. + finished_before_worker_exit.append(returned.wait(0.1)) + # Finishing this custom wakeup tail requires event-loop progress. + loop.call_soon_threadsafe(release.set) + + observer = Thread(target=observe_completion) + task.add_done_callback(lambda _: returned.set()) + monkeypatch.setattr(loop, "_write_to_self", gated_write) + observer.start() + try: + if outcome == "cancel": + task.cancel() + with pytest.raises(asyncio.CancelledError): + await task + else: + judge.release.set() + if outcome == "error": + with pytest.raises(DecisionError): + await task + else: + assert (await task).selected.target == "weak" + finally: + release.set() + monkeypatch.setattr(loop, "_write_to_self", write_to_self) + observer.join(timeout=5) + assert finished_before_worker_exit == [False] + + +async def test_cancellation_first_arriving_during_bridge_join_is_drained() -> None: + bridge: asyncio.Future[None] = asyncio.get_running_loop().create_future() + bridge.set_result(None) + joining = Event() + release = Event() + + class CompletedRunner: + def _start_decision(self, *args: object, **kwargs: object) -> tuple[object, object]: + return bridge, self + + def cancel(self) -> None: + pytest.fail("routing was already complete when cancellation arrived") + + def wait(self) -> None: + joining.set() + assert release.wait(5) + + def is_finished(self) -> bool: + return False + + task = asyncio.create_task(_decide(CompletedRunner(), request(), None, False)) + try: + assert await asyncio.to_thread(joining.wait, 5) + task.cancel("cancel during join") + await asyncio.sleep(0) + task.cancel("repeated cancellation") + await asyncio.sleep(0) + assert not task.done() + finally: + release.set() + with pytest.raises(asyncio.CancelledError) as caught: + await task + if sys.version_info >= (3, 11): + assert str(caught.value) == "cancel during join" + + +def test_asyncio_shutdown_keeps_an_existing_bridge_join_alive() -> None: + joining = Event() + shutdown_started = Event() + release = Event() + returned = Event() + joined = Event() + finished_before_join: list[bool] = [] + + async def main() -> None: + bridge: asyncio.Future[None] = asyncio.get_running_loop().create_future() + bridge.set_result(None) + + class CompletedRunner: + def _start_decision(self, *args: object, **kwargs: object) -> tuple[object, object]: + return bridge, self + + def wait(self) -> None: + joining.set() + assert release.wait(5) + joined.set() + + def is_finished(self) -> bool: + return False + + task = asyncio.create_task(_decide(CompletedRunner(), request(), None, False)) + task.add_done_callback(lambda _: returned.set()) + assert await asyncio.to_thread(joining.wait, 5) + asyncio.get_running_loop().call_soon(shutdown_started.set) + + def observe_shutdown() -> None: + if shutdown_started.wait(5): + finished_before_join.append(returned.wait(0.1)) + release.set() + + observer = Thread(target=observe_shutdown) + observer.start() + try: + asyncio.run(main()) + finally: + release.set() + observer.join(timeout=5) + assert joined.is_set() + assert finished_before_join == [False] + + +@pytest.mark.parametrize("shutdown", [False, True], ids=["explicit-cancel", "asyncio-shutdown"]) +async def test_cancellation_finishes_bridge_before_interpreter_shutdown( + judge: JudgeStub, tmp_path: Path, shutdown: bool +) -> None: + judge.wait_for_disconnect = True + config = tmp_path / "routes.toml" + config.write_text(deployment(judge.url)) + script = """ +import asyncio +import sys +from switchyard.runner import Runner + +async def main(): + runner = Runner.load(sys.argv[1]) + task = asyncio.ensure_future(runner.decide({ + "model": "auto", + "messages": [{"role": "user", "content": [{"type": "text", "text": "task"}]}], + })) + await asyncio.to_thread(sys.stdin.readline) + if sys.argv[2] == "False": + task.cancel() + try: + await task + except asyncio.CancelledError: + pass + +asyncio.run(main()) +""" + # Repeat immediate process exit to exercise the completion thread scheduling + # race, including cancellation performed by asyncio.run itself. + for _ in range(5): + judge.started.clear() + judge.disconnected.clear() + process = await asyncio.create_subprocess_exec( + sys.executable, + "-c", + script, + str(config), + str(shutdown), + stdin=asyncio.subprocess.PIPE, + stdout=asyncio.subprocess.PIPE, + stderr=asyncio.subprocess.PIPE, + ) + try: + assert await asyncio.to_thread(judge.started.wait, 5) + stdout, stderr = await asyncio.wait_for(process.communicate(b"\n"), timeout=10) + assert process.returncode == 0, stderr.decode() + assert not stdout + assert not stderr, stderr.decode() + assert await asyncio.to_thread(judge.disconnected.wait, 5) + finally: + if process.returncode is None: + process.kill() + await process.wait() From 33c6b419ae95f4dd2f4f6bd34575c25455e12dde Mon Sep 17 00:00:00 2001 From: ayushag Date: Wed, 30 Sep 2026 00:14:41 -0700 Subject: [PATCH 11/48] docs(sim): show policy comparison and held-out evaluation Signed-off-by: ayushag --- docs/simulation.md | 46 ++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 46 insertions(+) diff --git a/docs/simulation.md b/docs/simulation.md index d71ae410e..d761c364e 100644 --- a/docs/simulation.md +++ b/docs/simulation.md @@ -262,6 +262,52 @@ the actual model and native usage. Return `None` for unknown rates or usage. Without it, zero-call routes cost zero and other routing costs remain unknown. Backend retries and failed calls may incur unreported spend. +## Compare and tune policies + +Reuse one paired dataset to compare configurations. Keep the target keys, recorded +model identities, and route ID the same. For example, keep `routes.toml` as the +fixed baseline above and put a candidate policy in `candidate.toml`: + +```python +import asyncio +import json + +from switchyard.runner import Runner +from switchyard.sim import evaluate + +async def compare(dataset): + summaries = {} + for name, path in {"fixed": "routes.toml", "candidate": "candidate.toml"}.items(): + report = await evaluate( + dataset, Runner.load(path), route="auto", concurrency=1, + ) + summaries[name] = report.to_dict() + return summaries + +summaries = asyncio.run(compare(dataset)) +for name, summary in summaries.items(): + print(json.dumps({ + "policy": name, + "complete": summary["complete"], + "counts": summary["counts"], + "reward_comparison": summary["comparison"], + "cost_comparison": summary["cost_comparison"], + }, indent=2)) +``` + +Each configuration gets a fresh runner. The reward comparison includes routed, +fixed-target, and empirical-oracle means. Before ranking policies, check that all +runs are complete and the reward comparison covers every task. For a cost ranking, +also require full cost-comparison coverage. Unknown routing cost stays `null`; +supply the same `price_call` function to each evaluation to include classifier +cost estimates. If coverage is partial, use `on_result` rows to compare the same +task IDs across policies; matching counts alone do not establish a common cohort. + +Split recordings by task ID before tuning, keeping every repeat of a task in the +same split. Use the development split to choose prompts, thresholds, or routing +rules. Freeze that choice before evaluating a separate held-out dataset. Reusing +held-out rewards to select the next policy makes them tuning data. + ## Save progressive results ```sh From 0d86f53f232dd65d608612a7868ac399ab1511fd Mon Sep 17 00:00:00 2001 From: ayushag Date: Wed, 30 Sep 2026 00:27:49 -0700 Subject: [PATCH 12/48] test(python): cover response-based decision outcomes Signed-off-by: ayushag --- tests/test_runner_bindings.py | 39 +++++++++++++++++++++++++++++++++++ 1 file changed, 39 insertions(+) diff --git a/tests/test_runner_bindings.py b/tests/test_runner_bindings.py index 982ad0afa..ad64dbe5d 100644 --- a/tests/test_runner_bindings.py +++ b/tests/test_runner_bindings.py @@ -358,6 +358,45 @@ async def test_response_based_route_requires_opt_in_before_calls( assert judge.calls == [] +@pytest.mark.parametrize("verdict", ["APPROVE", "REDO check the result"]) +async def test_response_opt_in_preserves_answer_or_revision_without_final_completion( + judge: JudgeStub, verdict: str +) -> None: + judge.response_text = verdict + source = deployment(judge.url).split("[routes.classifier]")[0] + """ +[routes.advisor] +id = "auto" +type = "advisor" +executor_target = "weak" +advisor_target = "judge" +redo_feedback_prefix = "Feedback: " +""" + original = request() + decision = await Runner.from_toml(source).decide(original, allow_response=True) + + assert original == request() + assert (decision.selected.target, decision.selected.model) == ("weak", "weak/model") + assert decision.fallbacks == [] + # Both approval and revision stop after the executor and advisor calls. + assert [call["model"] for call in judge.calls] == ["weak/model", "judge/model"] + assert [call.model for call in decision.calls] == ["weak/model", "judge/model"] + assert all(call.is_success and call.usage["input_tokens"] == 12 for call in decision.calls) + if verdict == "APPROVE": + assert decision.outcome.response.response["model"] == "weak/model" + assert decision.outcome.response.response["outputs"][0]["content"] == [ + {"type": "text", "text": verdict} + ] + assert decision.outcome.request["messages"] == original["messages"] + assert decision.outcome.metadata.evidence["verdict"] == "approve" + else: + assert decision.outcome.response is None + assert decision.outcome.request["messages"][-2:] == [ + {"role": "assistant", "content": [{"type": "text", "text": verdict}]}, + {"role": "user", "content": [{"type": "text", "text": "Feedback: check the result"}]}, + ] + assert decision.outcome.metadata.evidence["verdict"] == "redo" + + async def test_invalid_verdict_preserves_fail_open_evidence_and_call_cost(judge: JudgeStub) -> None: judge.response_text = "not valid routing JSON" runner = Runner.from_toml(deployment(judge.url)) From f0c84dd2fbe4cac47e29c6177d2225b46614f015 Mon Sep 17 00:00:00 2001 From: ayushag Date: Wed, 30 Sep 2026 00:54:52 -0700 Subject: [PATCH 13/48] fix(libsy): stop cleanup workers when routers are dropped Signed-off-by: ayushag --- crates/libsy/src/algorithms/fall_through.rs | 66 ++++++++++++++++++--- 1 file changed, 58 insertions(+), 8 deletions(-) diff --git a/crates/libsy/src/algorithms/fall_through.rs b/crates/libsy/src/algorithms/fall_through.rs index b3cf236d0..3cc5a0f0c 100644 --- a/crates/libsy/src/algorithms/fall_through.rs +++ b/crates/libsy/src/algorithms/fall_through.rs @@ -19,7 +19,7 @@ use std::{ collections::HashMap, - sync::{Arc, Once, Weak}, + sync::{Arc, OnceLock, Weak}, time::{Duration, Instant}, }; @@ -69,7 +69,15 @@ pub struct FallThrough { processors: Vec>>, classifiers: Vec>>, session_states: Option>>, - cleanup_started: Once, + cleanup_task: OnceLock>, +} + +impl Drop for FallThrough { + fn drop(&mut self) { + if let Some(task) = self.cleanup_task.get() { + task.abort(); + } + } } impl FallThrough<()> { @@ -80,7 +88,7 @@ impl FallThrough<()> { processors: Vec::new(), classifiers: Vec::new(), session_states: None, - cleanup_started: Once::new(), + cleanup_task: OnceLock::new(), } } } @@ -96,7 +104,7 @@ where processors: Vec::new(), classifiers: Vec::new(), session_states: Some(Arc::new(Mutex::new(HashMap::new()))), - cleanup_started: Once::new(), + cleanup_task: OnceLock::new(), } } @@ -141,10 +149,8 @@ where let Some(states) = &self.session_states else { return; }; - let states = Arc::downgrade(states); - self.cleanup_started.call_once(move || { - drop(tokio::spawn(cleanup_inactive_sessions(states))); - }); + self.cleanup_task + .get_or_init(|| tokio::spawn(cleanup_inactive_sessions(Arc::downgrade(states)))); } async fn execute_session(&self, driver: Driver, request: Request) -> Result { @@ -916,4 +922,48 @@ mod tests { assert!(states.contains_key("session-1")); assert!(!states.contains_key("session-2")); } + + #[tokio::test] + async fn dropping_a_router_stops_its_cleanup_task() -> Result<()> { + #[derive(Default)] + struct Payload(Arc<()>); + + let metrics = tokio::runtime::Handle::current().metrics(); + let baseline = metrics.num_alive_tasks(); + let stateless = FallThrough::new(); + stateless.start_cleanup_task(); + drop(stateless); + drop(FallThrough::::new_with_state()); + assert_eq!(metrics.num_alive_tasks(), baseline); + + let router = Arc::new( + FallThrough::::new_with_state() + .with_classifier(Arc::new(DefaultCategoryClassifier(Category::Any))), + ); + let state = router.session_state(&request()).unwrap(); + let payload = Arc::downgrade(&state.lock().await.0); + drop(state); + run_turn(&router, echo()).await?; + tokio::time::timeout(Duration::from_secs(1), async { + while Arc::strong_count(&router) != 1 { + tokio::task::yield_now().await; + } + }) + .await + .expect("completed routing task must release the router"); + assert_eq!(metrics.num_alive_tasks(), baseline + 1); + router.start_cleanup_task(); + assert_eq!(metrics.num_alive_tasks(), baseline + 1); + + drop(router); + assert!(payload.upgrade().is_none()); + tokio::time::timeout(Duration::from_secs(1), async { + while metrics.num_alive_tasks() != baseline { + tokio::task::yield_now().await; + } + }) + .await + .expect("router cleanup task must stop when its owner is dropped"); + Ok(()) + } } From 3e2b02dc7f68df69122c9f088d9ff04d8c3d0d32 Mon Sep 17 00:00:00 2001 From: ayushag Date: Wed, 30 Sep 2026 01:14:32 -0700 Subject: [PATCH 14/48] test(python): cover provider-specific decision usage Signed-off-by: ayushag --- tests/test_runner_bindings.py | 89 ++++++++++++++++++++++++++++++++++- 1 file changed, 87 insertions(+), 2 deletions(-) diff --git a/tests/test_runner_bindings.py b/tests/test_runner_bindings.py index ad64dbe5d..07c57e438 100644 --- a/tests/test_runner_bindings.py +++ b/tests/test_runner_bindings.py @@ -29,6 +29,7 @@ class JudgeStub: calls: list[dict[str, Any]] = field(default_factory=list) request_headers: list[dict[str, str]] = field(default_factory=list) response_text: str | None = None + response_payload: dict[str, Any] | None = None status: int = 200 started: Event = field(default_factory=Event) release: Event | None = None @@ -68,7 +69,9 @@ def do_POST(self) -> None: "capability_boundary": "supported", "p_solve": 0.1 if "TASK_REQUIRES_STRONG" in json.dumps(body) else 0.9, } - if stub.status == 200: + if stub.response_payload is not None: + payload = stub.response_payload + elif stub.status == 200: payload = { "id": "judge-response", "model": body["model"], @@ -211,6 +214,85 @@ async def test_native_judge_only_decision_preserves_targets_and_usage( assert decision.duration_seconds >= call.duration_seconds >= 0 +@pytest.mark.parametrize("wire_format", ["anthropic_messages", "openai_responses"]) +@pytest.mark.parametrize("has_usage", [True, False]) +async def test_provider_formats_preserve_decision_identity_and_usage( + judge: JudgeStub, wire_format: str, has_usage: bool +) -> None: + verdict = json.dumps( + { + "crux": "bounded task", + "primary_rule": "SUP-1", + "capability_boundary": "supported", + "p_solve": 0.9, + } + ) + payload: dict[str, Any] = {"id": "judge-response", "model": "provider/model-alias"} + if wire_format == "anthropic_messages": + payload.update( + type="message", + role="assistant", + content=[{"type": "text", "text": verdict}], + stop_reason="end_turn", + ) + usage = { + "input_tokens": 17, + "output_tokens": 8, + "cache_read_input_tokens": 5, + "cache_creation_input_tokens": 2, + "output_tokens_details": {"thinking_tokens": 3}, + } + else: + payload.update( + object="response", + status="completed", + output=[ + { + "type": "message", + "role": "assistant", + "content": [{"type": "output_text", "text": verdict}], + } + ], + ) + usage = { + "input_tokens": 17, + "output_tokens": 8, + "total_tokens": 25, + "input_tokens_details": {"cached_tokens": 5, "cache_write_tokens": 2}, + "output_tokens_details": {"reasoning_tokens": 3}, + } + if has_usage: + payload["usage"] = usage + judge.response_payload = payload + source = deployment(judge.url).replace('format = "openai_chat"', f'format = "{wire_format}"') + + decision = await Runner.from_toml(source).decide(request()) + + assert (decision.selected.target, decision.selected.model) == ("weak", "weak/model") + assert [(target.target, target.model) for target in decision.fallbacks] == [ + ("strong", "strong/model") + ] + assert decision.outcome.selected_model_ids == ["weak/model", "strong/model"] + assert decision.outcome.response is None + assert [call["model"] for call in judge.calls] == ["judge/model"] + (call,) = decision.calls + assert call.model == "judge/model" + assert call.is_success + expected = { + "input_tokens": 17 if wire_format == "anthropic_messages" else 10, + "cached_input_tokens": 5, + "cache_creation_input_tokens": 2, + "output_tokens": 8, + "total_tokens": 32 if wire_format == "anthropic_messages" else 25, + "reasoning_tokens": 3, + } + assert call.usage is not None + if has_usage: + assert call.usage == expected + else: + assert all(call.usage.get(field) is None for field in expected) + + async def test_passthrough_decision_never_calls_answer_model(judge: JudgeStub) -> None: runner = Runner.from_toml(deployment(judge.url)) decision = await runner.decide(request(model="fixed")) @@ -363,7 +445,9 @@ async def test_response_opt_in_preserves_answer_or_revision_without_final_comple judge: JudgeStub, verdict: str ) -> None: judge.response_text = verdict - source = deployment(judge.url).split("[routes.classifier]")[0] + """ + source = ( + deployment(judge.url).split("[routes.classifier]")[0] + + """ [routes.advisor] id = "auto" type = "advisor" @@ -371,6 +455,7 @@ async def test_response_opt_in_preserves_answer_or_revision_without_final_comple advisor_target = "judge" redo_feedback_prefix = "Feedback: " """ + ) original = request() decision = await Runner.from_toml(source).decide(original, allow_response=True) From b07685827394493b92b1b7806934aeb2e050265b Mon Sep 17 00:00:00 2001 From: ayushag Date: Wed, 30 Sep 2026 01:21:43 -0700 Subject: [PATCH 15/48] fix(python): mark public package as typed Signed-off-by: ayushag --- .github/workflows/package-portability.yml | 8 ++++++++ switchyard/py.typed | 1 + 2 files changed, 9 insertions(+) create mode 100644 switchyard/py.typed diff --git a/.github/workflows/package-portability.yml b/.github/workflows/package-portability.yml index 8615f8cdf..17f2ccf39 100644 --- a/.github/workflows/package-portability.yml +++ b/.github/workflows/package-portability.yml @@ -84,6 +84,7 @@ jobs: import os import sys import tempfile + from importlib.resources import files os.chdir(tempfile.mkdtemp(prefix="switchyard-wheel-smoke-")) workspace = os.environ.get("GITHUB_WORKSPACE") @@ -93,6 +94,9 @@ jobs: import switchyard_rust from switchyard_rust import _switchyard_rust + for package in ("switchyard", "switchyard_rust"): + assert files(package).joinpath("py.typed").is_file(), package + print("switchyard", switchyard.__version__) print("switchyard_rust", switchyard_rust.__file__) print("rust extension", _switchyard_rust.__name__) @@ -109,6 +113,7 @@ jobs: import os import sys import tempfile + from importlib.resources import files os.chdir(tempfile.mkdtemp(prefix="switchyard-wheel-smoke-")) workspace = os.environ.get("GITHUB_WORKSPACE") @@ -118,6 +123,9 @@ jobs: import switchyard_rust from switchyard_rust import _switchyard_rust + for package in ("switchyard", "switchyard_rust"): + assert files(package).joinpath("py.typed").is_file(), package + print("switchyard", switchyard.__version__) print("switchyard_rust", switchyard_rust.__file__) print("rust extension", _switchyard_rust.__name__) diff --git a/switchyard/py.typed b/switchyard/py.typed new file mode 100644 index 000000000..8b1378917 --- /dev/null +++ b/switchyard/py.typed @@ -0,0 +1 @@ + From ed782943bc1f88a20c5fb3054286c01dae5eaa25 Mon Sep 17 00:00:00 2001 From: ayushag Date: Wed, 30 Sep 2026 01:49:51 -0700 Subject: [PATCH 16/48] fix(sim): reject invalid task checksum evidence Signed-off-by: ayushag --- docs/simulation.md | 3 ++- switchyard/sim/models.py | 4 ++++ tests/sim/test_dataset.py | 6 ++++++ tests/sim/test_trajectory.py | 8 ++++++++ 4 files changed, 20 insertions(+), 1 deletion(-) diff --git a/docs/simulation.md b/docs/simulation.md index d761c364e..9f2e965bf 100644 --- a/docs/simulation.md +++ b/docs/simulation.md @@ -191,7 +191,8 @@ The recorded initial input itself may contain harness or environment details. `input_target` explicitly chooses which run supplies that input. Task names are matched within the supplied runs. Use `dataset="benchmark-v1"` -to namespace them. Conflicting task checksums are rejected. When a checksum is +to namespace them. Conflicting task checksums are rejected. Blank strings and +non-string checksums are rejected; use `None` when unavailable. When a checksum is missing, all initial user messages must match. Full system scaffolding may differ between agents. Repeated trials retain their identities; each target's reward, cost, and duration are averaged across all repeats of a task. Tasks then have diff --git a/switchyard/sim/models.py b/switchyard/sim/models.py index 501a1dd64..b2cf15b6a 100644 --- a/switchyard/sim/models.py +++ b/switchyard/sim/models.py @@ -36,6 +36,10 @@ def __post_init__(self) -> None: for name in ("task_id", "trial_id", "target"): if not isinstance(getattr(self, name), str) or not getattr(self, name).strip(): raise ValueError(f"{name} must be a non-empty string") + if self.task_checksum is not None and ( + not isinstance(self.task_checksum, str) or not self.task_checksum.strip() + ): + raise ValueError("task_checksum must be a non-empty string or None") for name in ("reward", "cost_usd", "duration_seconds"): value = getattr(self, name) try: diff --git a/tests/sim/test_dataset.py b/tests/sim/test_dataset.py index 456eacb28..676a23864 100644 --- a/tests/sim/test_dataset.py +++ b/tests/sim/test_dataset.py @@ -109,6 +109,12 @@ def test_checksums_reject_changed_tasks_even_when_names_match(): ) +@pytest.mark.parametrize("checksum", ["", " \t\n", 1, True, [], {}]) +def test_invalid_task_checksums_are_rejected(checksum): + with pytest.raises(ValueError, match="task_checksum must be a non-empty string or None"): + trial(task_checksum=checksum) + + def test_agent_wrappers_may_differ_and_input_target_is_explicit(): fast = trial(task_checksum="same") strong = replace( diff --git a/tests/sim/test_trajectory.py b/tests/sim/test_trajectory.py index bbe3e311b..afc827852 100644 --- a/tests/sim/test_trajectory.py +++ b/tests/sim/test_trajectory.py @@ -133,6 +133,14 @@ def convert(record: dict) -> Trajectory: assert HarborRun is Run +@pytest.mark.parametrize("checksum", [" \t\n", 1]) +def test_custom_projection_rejects_invalid_task_checksums(checksum) -> None: + with pytest.raises(ValueError, match="task_checksum must be a non-empty string or None"): + Trajectory.from_dict(atif()).to_trial( + task_id="task", trial_id="attempt", target="fast", task_checksum=checksum + ) + + @pytest.mark.parametrize("version", [[], {}, None, "ATIF-v2.0"]) def test_invalid_version_is_rejected_without_printing_recording(version: object) -> None: with pytest.raises(ValueError, match="schema_version") as error: From d08fea4654df75b87e88d22500bc89e98c4385db Mon Sep 17 00:00:00 2001 From: ayushag Date: Wed, 30 Sep 2026 02:02:45 -0700 Subject: [PATCH 17/48] test(python): cover usage retained after a later call fails Signed-off-by: ayushag --- tests/test_runner_bindings.py | 35 +++++++++++++++++++++++++++++++++-- 1 file changed, 33 insertions(+), 2 deletions(-) diff --git a/tests/test_runner_bindings.py b/tests/test_runner_bindings.py index 07c57e438..0ab2f124e 100644 --- a/tests/test_runner_bindings.py +++ b/tests/test_runner_bindings.py @@ -31,6 +31,7 @@ class JudgeStub: response_text: str | None = None response_payload: dict[str, Any] | None = None status: int = 200 + status_by_model: dict[str, int] = field(default_factory=dict) started: Event = field(default_factory=Event) release: Event | None = None completed: Event = field(default_factory=Event) @@ -46,6 +47,7 @@ def judge(monkeypatch: pytest.MonkeyPatch) -> Iterator[JudgeStub]: class Handler(BaseHTTPRequestHandler): def do_POST(self) -> None: body = json.loads(self.rfile.read(int(self.headers["Content-Length"]))) + status = stub.status_by_model.get(body["model"], stub.status) stub.calls.append(body) stub.request_headers.append( {name.lower(): value for name, value in self.headers.items()} @@ -71,7 +73,7 @@ def do_POST(self) -> None: } if stub.response_payload is not None: payload = stub.response_payload - elif stub.status == 200: + elif status == 200: payload = { "id": "judge-response", "model": body["model"], @@ -100,7 +102,7 @@ def do_POST(self) -> None: } } encoded = json.dumps(payload).encode() - self.send_response(stub.status) + self.send_response(status) self.send_header("Content-Type", "application/json") self.send_header("Content-Length", str(len(encoded))) self.end_headers() @@ -515,6 +517,35 @@ async def test_provider_failure_keeps_observations_and_safe_diagnostics(judge: J assert error.duration_seconds >= call.duration_seconds >= 0 +async def test_later_call_failure_preserves_earlier_call_usage(judge: JudgeStub) -> None: + judge.status_by_model["judge/model"] = 503 + source = ( + deployment(judge.url).split("[routes.classifier]")[0] + + """ +[routes.advisor] +id = "auto" +type = "advisor" +executor_target = "weak" +advisor_target = "judge" +""" + ) + with pytest.raises(DecisionError) as caught: + await Runner.from_toml(source).decide(request(), allow_response=True) + error = caught.value + assert (error.kind, error.upstream_status, error.target) == ( + "upstream_http", + 503, + "judge/model", + ) + assert [call["model"] for call in judge.calls] == ["weak/model", "judge/model"] + first, failed = error.calls + assert (first.model, failed.model) == ("weak/model", "judge/model") + assert first.is_success and first.usage["input_tokens"] == 12 + assert not failed.is_success and failed.usage is None + assert "provider-secret" not in str(error) + assert "private prompt" not in str(error) + + async def test_bad_request_and_configuration_fail_before_calls( judge: JudgeStub, tmp_path: Path ) -> None: From 37cd3f2a695f93fa2ad701b383b751dc0c9776c7 Mon Sep 17 00:00:00 2001 From: ayushag Date: Wed, 30 Sep 2026 02:10:22 -0700 Subject: [PATCH 18/48] docs(python): show configured routing decisions Signed-off-by: ayushag --- README.md | 4 ++- docs/getting_started.md | 3 ++ docs/simulation.md | 69 +++++++++++++++++++++++++++++++++++++---- 3 files changed, 69 insertions(+), 7 deletions(-) diff --git a/README.md b/README.md index 2a2420a43..9bd0cf889 100644 --- a/README.md +++ b/README.md @@ -41,7 +41,9 @@ Agent-specific guides are available for [pi](docs/integrations/pi.md) and ### Embed the library in your harness -[Embed the library in your harness](docs/getting_started.md#library-path) to run routing inside your Rust application. For Python, see the [embedding example](examples/libsy.py). +[Embed the library in your harness](docs/getting_started.md#library-path) to run routing inside your Rust application. +For Python, use the [configured decision API](docs/simulation.md#use-the-python-decision-api) +or the [algorithm stream example](examples/libsy.py). ## Routing algorithms diff --git a/docs/getting_started.md b/docs/getting_started.md index 900d30f5f..e430ca42a 100644 --- a/docs/getting_started.md +++ b/docs/getting_started.md @@ -7,6 +7,9 @@ Switchyard has two native Rust execution paths: - **Library path:** embed the routing algorithms directly in your own Rust application with `switchyard-libsy`. +From Python, use the [configured decision API](simulation.md#use-the-python-decision-api) +or [evaluate task routing against recorded runs](simulation.md). + ## Server Path Use this path when you want a standalone proxy for API clients or need to diff --git a/docs/simulation.md b/docs/simulation.md index 9f2e965bf..3dc860317 100644 --- a/docs/simulation.md +++ b/docs/simulation.md @@ -341,6 +341,69 @@ Cancellation stops local workers, but cannot revoke requests already received by a provider. Task concurrency bounds simultaneous decisions; an algorithm may make multiple provider calls inside a decision. +## Use the Python decision API + +You can call `switchyard.runner` directly when your application owns task +scheduling and scoring. Use the same `routes.toml` shown above. Requests use +Switchyard's normalized format: `model` is the route ID, and message content is +a list of typed blocks. + +```python +import asyncio +from uuid import uuid4 + +from switchyard.runner import DecisionError, Runner + +async def inspect_decision(): + runner = Runner.load("routes.toml") + request = { + "model": "auto", + "messages": [{ + "role": "user", + "content": [{"type": "text", "text": "Fix the parser's handling of empty input."}], + }], + } + try: + decision = await asyncio.wait_for( + runner.decide(request, headers={ + "x-switchyard-session-id": uuid4().hex, + "x-switchyard-session-final": "true", + }), + timeout=60, + ) + except DecisionError as error: + print(error.kind, error.upstream_status, error.target, error.duration_seconds) + for call in error.calls: + print(call.model, call.is_success, call.duration_seconds, call.usage) + raise + + print(decision.selected.target, decision.selected.model) + print([target.target for target in decision.fallbacks], decision.duration_seconds) + for call in decision.calls: + print(call.model, call.is_success, call.duration_seconds, call.usage) + return decision + +decision = asyncio.run(inspect_decision()) +``` + +The fixed route selects target key `fast` and model ID `fast-model`, with no model +calls. Classifier routes may make judge calls. `calls` contains completed logical +calls; durations include backend retries. `usage` is a mapping or `None`, and +missing token fields remain unknown. `DecisionError.target` names the failing +model, while `decision.selected.target` is the configured target key. + +`Runner.load` and `Runner.from_toml` use Switchyard's configuration parser. +Configuration and request validation failures raise `ValueError`. Execution +failures raise `DecisionError` with safe diagnostics and completed observations; +an application deadline can instead raise `asyncio.TimeoutError`. Cancellation +waits for local routing and its Python bridge to stop. + +Give independent tasks distinct session IDs. For successive turns of one session, +reuse its ID, await each decision in order, and mark only the last turn as final. +Use a fresh runner for independent experiments. `decision.outcome` retains the +existing `RoutingOutcome`. Algorithms that generate an answer while routing need +explicit `allow_response=True`; task simulation rejects them. + ## Library boundaries `Trajectory` preserves ATIF documents. `Trial` and `Run` represent projected @@ -355,12 +418,6 @@ Treat projected `Trial`, `Task`, and `Dataset` records as read-only. Their froze dataclasses still contain mutable nested mappings. If you change recorded input, create new trials and rebuild the dataset so pairing is validated again. -Native `Runner.load` and `Runner.from_toml` use Switchyard's configuration parser. -`await runner.decide(normalized_request, headers=...)` returns configured selected -and fallback targets, the existing `RoutingOutcome`, completed-call observations, -and elapsed routing time. Provider failures raise `DecisionError` with completed -observations and safe error metadata. Algorithms that generate an answer while -routing require explicit `allow_response=True`; task simulation rejects them. The task scorer also rejects answers and material conversation rewrites, because recorded outcomes cannot score those changes. From bf470e62dfa5018acdf2469e3fa501522a2fb8c8 Mon Sep 17 00:00:00 2001 From: ayushag Date: Wed, 30 Sep 2026 02:21:48 -0700 Subject: [PATCH 19/48] fix(sim): reject blank rejected-trial identities Signed-off-by: ayushag --- docs/simulation.md | 5 +++-- switchyard/sim/dataset.py | 5 ++--- tests/sim/test_dataset.py | 8 +++++--- 3 files changed, 10 insertions(+), 8 deletions(-) diff --git a/docs/simulation.md b/docs/simulation.md index 3dc860317..0d558a56a 100644 --- a/docs/simulation.md +++ b/docs/simulation.md @@ -217,8 +217,9 @@ print(dataset.coverage) The intersection excludes a task from every target if any target is missing or has an invalid trial for it. Issues and excluded task IDs remain in the report. -An invalid result with no recoverable task identity must be repaired before -pairing; otherwise a failed repeat could silently disappear from a task's mean. +An issue with a missing or blank task ID must be repaired before pairing; +otherwise a failed repeat could silently disappear from a task's mean. Custom +importers should use `None` when a rejected record's task cannot be identified. For a missing trajectory or copied continuation context, callers can supply `task_inputs={task_id: original_instruction}` to `load_harbor`. Invalid JSON, unsupported ATIF versions, and invalid fields read during input diff --git a/switchyard/sim/dataset.py b/switchyard/sim/dataset.py index 4517f0a22..d4b0c9c27 100644 --- a/switchyard/sim/dataset.py +++ b/switchyard/sim/dataset.py @@ -53,7 +53,7 @@ def from_runs( groups[trial.task_id].append(trial) grouped[target] = groups for issue in run.issues: - if issue.task_id is None: + if not isinstance(issue.task_id, str) or not issue.task_id.strip(): raise ValueError( f"input issue has no task identity: {issue.source}; " "repair or identify the rejected trial before pairing" @@ -66,8 +66,7 @@ def from_runs( "message": issue.message, } ) - if issue.task_id is not None: - invalid.add(issue.task_id) + invalid.add(issue.task_id) task_sets = [set(groups) for groups in grouped.values()] union = set.union(*task_sets) | invalid common = set.intersection(*task_sets) - invalid diff --git a/tests/sim/test_dataset.py b/tests/sim/test_dataset.py index 676a23864..97dc3c108 100644 --- a/tests/sim/test_dataset.py +++ b/tests/sim/test_dataset.py @@ -73,10 +73,12 @@ def test_invalid_repeat_excludes_whole_task_instead_of_improving_its_mean(): assert len(dataset.coverage["input_issues"]) == 1 -def test_unidentified_rejected_repeat_cannot_leave_a_favorable_mean(): - run = HarborRun((trial(reward=1),), (LoadIssue("missing/result.json", "cannot read"),)) +@pytest.mark.parametrize("task_id", [None, "", " \t\n"]) +@pytest.mark.parametrize("intersection", [False, True]) +def test_unidentified_rejected_repeat_cannot_leave_a_favorable_mean(task_id, intersection): + run = HarborRun((trial(reward=1),), (LoadIssue("missing/result.json", "cannot read", task_id),)) with pytest.raises(ValueError, match="no task identity"): - Dataset.from_runs({"fast": run}, input_target="fast", intersection=True) + Dataset.from_runs({"fast": run}, input_target="fast", intersection=intersection) def test_cost_source_mixture_is_visible_in_coverage(): From c56ab871b5defc609164c6c89bba01f4651028b7 Mon Sep 17 00:00:00 2001 From: ayushag Date: Wed, 30 Sep 2026 02:32:02 -0700 Subject: [PATCH 20/48] docs(sim): clarify result callback and write ownership Signed-off-by: ayushag --- docs/simulation.md | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/docs/simulation.md b/docs/simulation.md index 0d558a56a..384d50c15 100644 --- a/docs/simulation.md +++ b/docs/simulation.md @@ -335,9 +335,13 @@ If an output write fails, the last JSONL line may be incomplete. Earlier complet lines remain usable; check the exit status and final report before treating a run as complete. Use `--model-alias RECORDED=CONFIGURED` for an explicit model-ID equivalence. -The Python `on_result` callback runs after each completed task. It can update a -dashboard or write an application-owned result stream. Keep it short; a callback -failure stops evaluation and drains pending work. Each decision has a deadline. +The Python `on_result` callback runs synchronously on the event loop after each +completed task. Supply a regular function; asynchronous callbacks are not awaited. +It can update a dashboard or write an application-owned result stream. Keep it +short; an exception from the callback stops evaluation and drains pending work. +If the callback queues background writes, wait for them and handle their failures +before treating the records as saved. +Each decision has a deadline. Cancellation stops local workers, but cannot revoke requests already received by a provider. Task concurrency bounds simultaneous decisions; an algorithm may make multiple provider calls inside a decision. From fbb6b5e3b83aa7fce004e7009bde2183e8eaa29a Mon Sep 17 00:00:00 2001 From: ayushag Date: Wed, 30 Sep 2026 02:41:01 -0700 Subject: [PATCH 21/48] test(python): smoke public simulation imports in wheels Signed-off-by: ayushag --- .github/workflows/package-portability.yml | 8 ++++++++ .github/workflows/publish.yml | 8 ++++++++ 2 files changed, 16 insertions(+) diff --git a/.github/workflows/package-portability.yml b/.github/workflows/package-portability.yml index 17f2ccf39..a981cbe23 100644 --- a/.github/workflows/package-portability.yml +++ b/.github/workflows/package-portability.yml @@ -92,12 +92,16 @@ jobs: import switchyard import switchyard_rust + from switchyard.runner import Runner + from switchyard.sim import Trajectory from switchyard_rust import _switchyard_rust for package in ("switchyard", "switchyard_rust"): assert files(package).joinpath("py.typed").is_file(), package print("switchyard", switchyard.__version__) + print("runner", Runner) + print("trajectory", Trajectory) print("switchyard_rust", switchyard_rust.__file__) print("rust extension", _switchyard_rust.__name__) PY @@ -121,12 +125,16 @@ jobs: import switchyard import switchyard_rust + from switchyard.runner import Runner + from switchyard.sim import Trajectory from switchyard_rust import _switchyard_rust for package in ("switchyard", "switchyard_rust"): assert files(package).joinpath("py.typed").is_file(), package print("switchyard", switchyard.__version__) + print("runner", Runner) + print("trajectory", Trajectory) print("switchyard_rust", switchyard_rust.__file__) print("rust extension", _switchyard_rust.__name__) PY diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index f4bcea100..6974c8b97 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -278,10 +278,14 @@ jobs: import switchyard import switchyard_rust + from switchyard.runner import Runner + from switchyard.sim import Trajectory from switchyard_rust import _switchyard_rust from switchyard_rust.server import Server print("switchyard", switchyard.__version__) + print("runner", Runner) + print("trajectory", Trajectory) print("switchyard_rust", switchyard_rust.__file__) print("rust extension", _switchyard_rust.__name__) print("server", Server) @@ -307,10 +311,14 @@ jobs: import switchyard import switchyard_rust + from switchyard.runner import Runner + from switchyard.sim import Trajectory from switchyard_rust import _switchyard_rust from switchyard_rust.server import Server print("switchyard", switchyard.__version__) + print("runner", Runner) + print("trajectory", Trajectory) print("switchyard_rust", switchyard_rust.__file__) print("rust extension", _switchyard_rust.__name__) print("server", Server) From 148e60687a12d59b9fceffae577f82161fa3138b Mon Sep 17 00:00:00 2001 From: ayushag Date: Wed, 30 Sep 2026 02:50:47 -0700 Subject: [PATCH 22/48] test(sim): guard native-free data preparation Signed-off-by: ayushag --- tests/sim/test_trajectory.py | 49 ++++++++++++++++++++++++++++++++++++ 1 file changed, 49 insertions(+) diff --git a/tests/sim/test_trajectory.py b/tests/sim/test_trajectory.py index afc827852..f9acbf3ef 100644 --- a/tests/sim/test_trajectory.py +++ b/tests/sim/test_trajectory.py @@ -4,10 +4,14 @@ """Shared ATIF conversion preserves evidence while isolating task-routing input.""" import json +import subprocess +import sys from pathlib import Path +from textwrap import dedent import pytest +import switchyard from switchyard.sim import Dataset, HarborRun, Run, Trajectory, load_harbor from tests.sim.test_harbor import update, write_trial @@ -133,6 +137,51 @@ def convert(record: dict) -> Trajectory: assert HarborRun is Run +def test_data_preparation_does_not_require_native_or_harbor_packages(tmp_path: Path) -> None: + write_trial(tmp_path) + script = dedent(""" + import json + import sys + from pathlib import Path + + sys.path.insert(0, sys.argv[1]) + sys.modules.update(dict.fromkeys(("switchyard_rust", "switchyard.runner", "harbor"))) + from switchyard.sim import Dataset, Report, Result, Run, Trajectory, load_harbor + + run = load_harbor(sys.argv[2], target="recorded") + source = Path(sys.argv[2]) / "trial-a" / "agent" / "trajectory.json" + custom = Trajectory.from_dict(json.loads(source.read_text())).to_trial( + task_id=run.trials[0].task_id, trial_id="custom", target="custom", reward=1, + ) + dataset = Dataset.from_runs( + {"recorded": run, "custom": Run((custom,))}, input_target="custom", + ) + task = dataset.tasks[0] + report = Report(1, dataset.targets, run_id="offline", concurrency=1) + report.add(Result( + task.task_id, task.outcomes, target="custom", outcome=task.outcomes["custom"], + )) + assert report.complete + assert report.to_dict()["comparison"]["targets"] == {"recorded": 0.75, "custom": 1} + """) + completed = subprocess.run( + [ + sys.executable, + "-I", + "-S", + "-c", + script, + str(Path(switchyard.__file__).resolve().parent.parent), + str(tmp_path), + ], + cwd=tmp_path, + capture_output=True, + text=True, + timeout=30, + ) + assert completed.returncode == 0, completed.stderr + + @pytest.mark.parametrize("checksum", [" \t\n", 1]) def test_custom_projection_rejects_invalid_task_checksums(checksum) -> None: with pytest.raises(ValueError, match="task_checksum must be a non-empty string or None"): From e66b9f8106dcc05f9d9f69b3d2d07239781e87ad Mon Sep 17 00:00:00 2001 From: ayushag Date: Wed, 30 Sep 2026 03:00:30 -0700 Subject: [PATCH 23/48] fix(python): accept mapping headers in routing decisions Signed-off-by: ayushag --- crates/switchyard-py/src/runner_bindings.rs | 10 +++++++++- tests/test_runner_bindings.py | 21 ++++++++++++++++++++- 2 files changed, 29 insertions(+), 2 deletions(-) diff --git a/crates/switchyard-py/src/runner_bindings.rs b/crates/switchyard-py/src/runner_bindings.rs index 2c09677b2..933a6e7e4 100644 --- a/crates/switchyard-py/src/runner_bindings.rs +++ b/crates/switchyard-py/src/runner_bindings.rs @@ -13,6 +13,7 @@ use std::task::{Context, Poll}; use pyo3::create_exception; use pyo3::exceptions::{PyRuntimeError, PyValueError}; use pyo3::prelude::*; +use pyo3::types::{PyDict, PyMapping}; use pyo3_async_runtimes::TaskLocals; use pyo3_async_runtimes::generic::{ContextExt, Runtime}; use switchyard_llm_client::{LlmCallObservation, RunObservation, RunObserver}; @@ -292,9 +293,16 @@ impl PyRunner { slf: PyRef<'py, Self>, py: Python<'py>, request: &Bound<'py, PyAny>, - headers: Option>, + headers: Option<&Bound<'py, PyMapping>>, allow_response: bool, ) -> PyResult> { + let headers = headers + .map(|mapping| { + let owned = PyDict::new(py); + owned.update(mapping)?; + owned.extract::>() + }) + .transpose()?; // Start routing only when the coroutine runs, so cancellation before its // first step cannot leave an unowned native request running. py.import("switchyard_rust.runner")? diff --git a/tests/test_runner_bindings.py b/tests/test_runner_bindings.py index 0ab2f124e..ad08cfd70 100644 --- a/tests/test_runner_bindings.py +++ b/tests/test_runner_bindings.py @@ -9,11 +9,13 @@ import json import os import sys -from collections.abc import Iterator +from collections import UserDict +from collections.abc import Callable, Iterator, Mapping from dataclasses import dataclass, field from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer from pathlib import Path from threading import Event, Thread +from types import MappingProxyType from typing import Any import pytest @@ -325,6 +327,23 @@ async def test_affinity_and_observations_are_isolated_across_sessions_and_runner assert len(judge.calls) == 4 +@pytest.mark.parametrize("wrap", [MappingProxyType, UserDict]) +async def test_mapping_headers_are_copied_before_await( + judge: JudgeStub, wrap: Callable[[dict[str, str]], Mapping[str, str]] +) -> None: + runner = Runner.from_toml(deployment(judge.url)) + values = {**session("snapshot"), "x-request-id": "original"} + headers = wrap(values) + pending = runner.decide(MappingProxyType(request()), headers=headers) + mutable_headers = headers.data if isinstance(headers, UserDict) else values + mutable_headers["x-request-id"] = "changed" + + decision = await pending + assert decision.selected.target == "weak" + assert len(judge.calls) == 1 + assert judge.request_headers[0]["x-request-id"] == "original" + + @pytest.mark.parametrize("forward_auth", [False, True]) async def test_caller_headers_are_request_scoped_and_respect_configured_auth( judge: JudgeStub, forward_auth: bool From 8b2bc85667ce4c3385980c48a67a6c5d86765a60 Mon Sep 17 00:00:00 2001 From: ayushag Date: Wed, 30 Sep 2026 03:11:09 -0700 Subject: [PATCH 24/48] docs(sim): clarify repeat inputs and baseline validation Signed-off-by: ayushag --- docs/simulation.md | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/docs/simulation.md b/docs/simulation.md index 384d50c15..523ed5566 100644 --- a/docs/simulation.md +++ b/docs/simulation.md @@ -82,7 +82,9 @@ order must be reproducible. Concurrent scheduling can change which task receives each random draw. The configured route may select any recorded target, including a fixed-target -subset. Known recorded model IDs must match the configured model IDs. Use the +subset. Model validation covers that route's configured completion targets; their +known recorded model IDs must match the configured model IDs. Verify the model +and agent settings of other recorded targets before using their fixed baselines. Use the explicit `model_aliases={"recorded/provider/model": "configured/model"}` argument to `evaluate` or `score` when the two systems name the same model differently. Aliases are a caller assertion; the library does not guess equivalence by trimming model names. Missing model @@ -188,7 +190,9 @@ agent step. This matters for Codex, whose first user message can describe the environment and whose second contains the task. Completed agent messages, verifier rewards, model names, and trial IDs are not added to classifier input. The recorded initial input itself may contain harness or environment details. -`input_target` explicitly chooses which run supplies that input. +`input_target` explicitly chooses which run supplies that input. For repeated +trials, it uses the first trial after sorting `trial_id` as strings. Keep that +choice fixed when comparing policies; the recorded outcomes still average all repeats. Task names are matched within the supplied runs. Use `dataset="benchmark-v1"` to namespace them. Conflicting task checksums are rejected. Blank strings and From f1aeab4fba1c453fc488cbe5f2c86db0872bc83a Mon Sep 17 00:00:00 2001 From: ayushag Date: Wed, 30 Sep 2026 03:19:34 -0700 Subject: [PATCH 25/48] fix(sim): detect incomplete Harbor job copies Signed-off-by: ayushag --- docs/simulation.md | 6 ++++ switchyard/sim/harbor.py | 59 +++++++++++++++++++++++++++------------- tests/sim/test_harbor.py | 59 ++++++++++++++++++++++++++++++++++++++++ 3 files changed, 105 insertions(+), 19 deletions(-) diff --git a/docs/simulation.md b/docs/simulation.md index 523ed5566..7a5f745c8 100644 --- a/docs/simulation.md +++ b/docs/simulation.md @@ -221,6 +221,12 @@ print(dataset.coverage) The intersection excludes a task from every target if any target is missing or has an invalid trial for it. Issues and excluded task IDs remain in the report. +When a job summary supplies a valid `n_total_trials`, it must match the number of +discovered trial directories, including rejected trials. A mismatch becomes an +issue with no task identity, so even an entirely missing repeat remains visible. +Repair incomplete copies or wait for unfinished jobs before pairing. Missing or +malformed summaries cannot establish directory completeness; check archive integrity +separately. For an intentional subset, import its trial directories individually. An issue with a missing or blank task ID must be repaired before pairing; otherwise a failed repeat could silently disappear from a task's mean. Custom importers should use `None` when a rejected record's task cannot be identified. diff --git a/switchyard/sim/harbor.py b/switchyard/sim/harbor.py index 206d23247..9a7af34f6 100644 --- a/switchyard/sim/harbor.py +++ b/switchyard/sim/harbor.py @@ -58,12 +58,13 @@ def load_harbor( root = Path(path) if not root.is_dir(): raise FileNotFoundError(f"Harbor directory does not exist: {root}") - paths = _trial_paths(root) - if not paths: + paths, issues = _trial_paths(root) + if issues and on_error == "raise": + raise ValueError(f"{issues[0].source}: {issues[0].message}") + if not paths and not issues: raise ValueError(f"No Harbor trials found under {root}") trials: list[Trial] = [] - issues: list[LoadIssue] = [] trial_ids: set[str] = set() for result_path in paths: task_id = None @@ -86,34 +87,54 @@ def load_harbor( return Run(tuple(trials), tuple(issues)) -def _trial_paths(root: Path) -> list[Path]: +def _trial_paths(root: Path) -> tuple[list[Path], list[LoadIssue]]: """Inspect known Harbor layouts, excluding nested copies of trial metadata.""" result_path = root / "result.json" if (root / "agent").is_dir(): - return [result_path] + return [result_path], [] unreadable_result = False + summary = None if result_path.is_file(): try: - if "task_name" in _read_object(result_path): - return [result_path] + summary = _read_object(result_path) + if "task_name" in summary: + return [result_path], [] except (OSError, ValueError): unreadable_result = True jobs = root / "jobs" roots = sorted(p for p in jobs.iterdir() if p.is_dir()) if jobs.is_dir() else [root] - paths = [ - trial / "result.json" - for job in roots - for trial in sorted(job.iterdir()) - if trial.is_dir() - and ( - (trial / "result.json").is_file() - or (trial / "agent").is_dir() - or (trial / "config.json").is_file() - ) - ] + paths: list[Path] = [] + issues: list[LoadIssue] = [] + for job in roots: + candidates = [ + trial / "result.json" + for trial in sorted(job.iterdir()) + if trial.is_dir() + and ( + (trial / "result.json").is_file() + or (trial / "agent").is_dir() + or (trial / "config.json").is_file() + ) + ] + paths.extend(candidates) + job_result = job / "result.json" + job_summary = summary if job == root else None + if job != root and job_result.is_file(): + try: + job_summary = _read_object(job_result) + except (OSError, ValueError): + pass + expected = job_summary.get("n_total_trials") if job_summary is not None else None + if type(expected) is int and expected >= 0 and expected != len(candidates): + issues.append( + LoadIssue( + str(job_result), + f"job trial count mismatch: expected {expected}, found {len(candidates)}", + ) + ) # A damaged job summary must not hide valid trial directories. Without # children, keep the unreadable artifact visible as a standalone trial issue. - return paths or ([result_path] if unreadable_result else []) + return paths or ([result_path] if unreadable_result else []), issues def _read_object(path: Path) -> dict[str, Any]: diff --git a/tests/sim/test_harbor.py b/tests/sim/test_harbor.py index fe6220833..91e20dcae 100644 --- a/tests/sim/test_harbor.py +++ b/tests/sim/test_harbor.py @@ -8,6 +8,7 @@ import pytest +from switchyard.sim import Dataset from switchyard.sim.harbor import load_harbor @@ -170,6 +171,64 @@ def test_repeated_tasks_are_preserved_as_distinct_trials(tmp_path: Path) -> None assert trials[0].trial_id != trials[1].trial_id +@pytest.mark.parametrize("layout", ["job", "download"]) +@pytest.mark.parametrize("remaining", [0, 1]) +def test_missing_repeat_directory_is_not_silently_dropped( + tmp_path: Path, layout: str, remaining: int +) -> None: + job = tmp_path / "jobs" / "run" if layout == "download" else tmp_path + job.mkdir(parents=True, exist_ok=True) + if remaining: + write_trial(job, "retained") + summary = job / "result.json" + summary.write_text(json.dumps({"n_total_trials": 2})) + + with pytest.raises(ValueError, match=f"expected 2, found {remaining}"): + load_harbor(tmp_path, target="baseline") + run = load_harbor(tmp_path, target="baseline", on_error="record") + assert len(run.trials) == remaining + assert len(run.issues) == 1 + assert run.issues[0].source == str(summary) + assert run.issues[0].task_id is None + with pytest.raises(ValueError, match="no task identity"): + Dataset.from_runs({"baseline": run}, input_target="baseline", intersection=True) + + +def test_job_counts_are_checked_separately_including_empty_jobs(tmp_path: Path) -> None: + for name, count in (("retained", 0), ("empty", 1)): + job = tmp_path / "jobs" / name + job.mkdir(parents=True) + (job / "result.json").write_text(json.dumps({"n_total_trials": count})) + write_trial(tmp_path / "jobs" / "retained") + + run = load_harbor(tmp_path, target="baseline", on_error="record") + assert len(run.trials) == 1 + assert {issue.message for issue in run.issues} == { + "job trial count mismatch: expected 0, found 1", + "job trial count mismatch: expected 1, found 0", + } + + +def test_job_count_includes_rejected_trial_directories(tmp_path: Path) -> None: + invalid = write_trial(tmp_path, "invalid") + (invalid / "result.json").unlink() + write_trial(tmp_path, "valid") + (tmp_path / "result.json").write_text(json.dumps({"n_total_trials": 2})) + + run = load_harbor(tmp_path, target="baseline", on_error="record") + assert len(run.trials) == len(run.issues) == 1 + assert run.issues[0].source == str(invalid / "result.json") + + +@pytest.mark.parametrize("count", [None, True, -1, "2"]) +def test_unusable_job_count_does_not_hide_valid_trials(tmp_path: Path, count: object) -> None: + write_trial(tmp_path) + (tmp_path / "result.json").write_text(json.dumps({"n_total_trials": count})) + run = load_harbor(tmp_path, target="baseline") + assert len(run.trials) == 1 + assert run.issues == () + + def test_missing_measurements_remain_unknown_and_preserve_exception_type(tmp_path: Path) -> None: trial = write_trial(tmp_path) update( From 7eb48cf229b2238e12ea23d9b337e0dca697b8ab Mon Sep 17 00:00:00 2001 From: ayushag Date: Wed, 30 Sep 2026 03:48:28 -0700 Subject: [PATCH 26/48] fix(runner): scope target alias validation to decisions Signed-off-by: ayushag --- crates/switchyard-py/src/runner_bindings.rs | 1 + crates/switchyard-runner/src/config.rs | 27 +++-- crates/switchyard-runner/src/route.rs | 20 ++++ crates/switchyard-server/src/lib.rs | 3 + crates/switchyard-server/tests/server.rs | 112 ++++++++++++++++++++ docs/reference/toml_schema.md | 12 ++- docs/simulation.md | 7 +- tests/test_runner_bindings.py | 36 ++++++- 8 files changed, 191 insertions(+), 27 deletions(-) diff --git a/crates/switchyard-py/src/runner_bindings.rs b/crates/switchyard-py/src/runner_bindings.rs index 933a6e7e4..75ef234bc 100644 --- a/crates/switchyard-py/src/runner_bindings.rs +++ b/crates/switchyard-py/src/runner_bindings.rs @@ -435,6 +435,7 @@ fn decision_route<'a>( let route = runner .route(model) .ok_or_else(|| PyValueError::new_err(format!("unknown route model {model:?}")))?; + route.validate_decision_targets().map_err(config_error)?; if !allow_response && route.routing_answer_target().is_some() { return Err(PyValueError::new_err( "route can call an answer model while deciding; set allow_response=True to permit it", diff --git a/crates/switchyard-runner/src/config.rs b/crates/switchyard-runner/src/config.rs index 652ac1ceb..b0c796c17 100644 --- a/crates/switchyard-runner/src/config.rs +++ b/crates/switchyard-runner/src/config.rs @@ -244,17 +244,6 @@ impl DeploymentConfig { .into_iter() .filter_map(|name| self.decision_target(name)) .collect(); - let mut names_by_model = HashMap::new(); - for target in &decision_targets { - if let Some(first_name) = names_by_model.insert(&target.model, &target.target) - && first_name != &target.target - { - return Err(RunnerError::configuration(format!( - "route {route_name} completion targets {first_name} and {} both use model {}; routing decisions identify models, so use distinct model ids within this route or put these targets in separate routes", - target.target, target.model - ))); - } - } let names = config .algorithm .runtime_model_names(route_name) @@ -1577,7 +1566,7 @@ target = "smart" } #[test] - fn completion_target_aliases_in_one_route_are_rejected() { + fn completion_target_aliases_load_but_cannot_return_named_decisions() -> RunnerResult<()> { const ALIASED_TARGETS: &str = r#" schema_version = 1 [llm_clients.primary] @@ -1602,15 +1591,21 @@ llm_client = "primary" let configured = format!( "{ALIASED_TARGETS}\n[routes.shared]\nid = \"switchyard/shared\"\n{algorithm}" ); - let message = error_message(&configured); + let runner = runner_from_toml(&configured)?; + let message = runner + .route("switchyard/shared") + .expect("shared route should load") + .validate_decision_targets() + .expect_err("aliased completion targets cannot identify a named decision") + .to_string(); assert!( - message.contains("route shared") - && message.contains("completion targets first and second") + message.contains("completion targets first and second") && message.contains("shared/model") && message.contains("distinct model ids"), "{algorithm}: {message}" ); } + Ok(()) } #[test] @@ -1622,6 +1617,7 @@ llm_client = "primary" let route = runner .route("switchyard/passthrough") .expect("passthrough route should exist"); + route.validate_decision_targets()?; assert_eq!( route .decision_targets() @@ -1643,6 +1639,7 @@ llm_client = "primary" let route = runner .route("switchyard/classifier") .expect("classifier route should exist"); + route.validate_decision_targets()?; assert_eq!( route.models().models_for(&Category::Judge), route.models().models_for(&Category::Efficient) diff --git a/crates/switchyard-runner/src/route.rs b/crates/switchyard-runner/src/route.rs index 0b1d2ca1c..51f8db8b4 100644 --- a/crates/switchyard-runner/src/route.rs +++ b/crates/switchyard-runner/src/route.rs @@ -3,6 +3,7 @@ //! One configured algorithm and the clients that serve its targets. +use std::collections::HashMap; use std::error::Error; use std::sync::Arc; @@ -166,6 +167,25 @@ impl Route { &self.decision_targets } + /// Checks that model IDs identify unambiguous completion target names. + /// + /// Serving and raw model-ID decisions may use identical aliases. Consumers + /// returning named targets must check this before making routing-time calls. + pub fn validate_decision_targets(&self) -> Result<(), RunnerError> { + let mut names_by_model = HashMap::new(); + for target in &self.decision_targets { + if let Some(first_name) = names_by_model.insert(&target.model, &target.target) + && first_name != &target.target + { + return Err(RunnerError::configuration(format!( + "completion targets {first_name} and {} both use model {}; routing decisions identify models, so reuse one target key, use distinct model ids, or put these targets in separate routes", + target.target, target.model + ))); + } + } + Ok(()) + } + /// Returns model-list capability metadata. pub fn capabilities(&self) -> ModelCapabilities { self.capabilities diff --git a/crates/switchyard-server/src/lib.rs b/crates/switchyard-server/src/lib.rs index ca674f523..9606805f7 100644 --- a/crates/switchyard-server/src/lib.rs +++ b/crates/switchyard-server/src/lib.rs @@ -734,6 +734,9 @@ async fn decision( .map(ModelId::from) .unwrap_or_default(); + if let Err(error) = route.validate_decision_targets() { + return runner_error(error); + } let mut outcome = match route.decide(request).await { Ok(outcome) => outcome, Err(error) => return runner_error(error), diff --git a/crates/switchyard-server/tests/server.rs b/crates/switchyard-server/tests/server.rs index cd21d3c91..f9c363384 100644 --- a/crates/switchyard-server/tests/server.rs +++ b/crates/switchyard-server/tests/server.rs @@ -1826,6 +1826,118 @@ async fn responses_continuations_preserve_state_ownership() -> TestResult { Ok(()) } +#[tokio::test] +async fn identical_completion_aliases_serve_but_cannot_return_named_decisions() -> TestResult { + let upstream = MockUpstream::start().await?; + let app = build_switchyard_router(load_test_config(&format!( + r#" +schema_version = 1 +[llm_clients.upstream] +format = "openai_chat" +base_url = "{base_url}" +[targets] +parent = {{ id = "model/shared", llm_client = "upstream" }} +delegated = {{ id = "model/shared", llm_client = "upstream" }} +judge = {{ id = "model/classifier", llm_client = "upstream" }} +[routes.agent] +id = "switchyard/aliases" +type = "passthrough" +target = "parent" +[routes.agent.subagents] +type = "passthrough" +target = "delegated" +[routes.classified] +id = "switchyard/classify-aliases" +type = "llm_classifier" +classifier_target = "judge" +strong_target = "parent" +weak_target = "delegated" +base_threshold = 0.5 +[routes.fixed] +id = "switchyard/fixed" +type = "passthrough" +target = "parent" +"#, + base_url = upstream.base_url, + ))?); + + let delegated_headers = [ + ("x-claude-code-session-id", "parent-session"), + ("x-claude-code-agent-id", "delegated-agent"), + ]; + for headers in [&[][..], delegated_headers.as_slice()] { + let response = send_with_headers( + &app, + "POST", + "/v1/chat/completions", + Some(json!({ + "model": "switchyard/aliases", + "messages": [{"role": "user", "content": "hi"}] + })), + headers, + ) + .await?; + assert_eq!(response.status, StatusCode::OK); + assert_eq!( + response.headers["x-model-router-selected-model"], + "model/shared" + ); + assert_eq!(response.json()?["choices"][0]["message"]["content"], "ok"); + } + assert_eq!(upstream.models().await, ["model/shared", "model/shared"]); + upstream.calls.lock().await.clear(); + + for model in ["switchyard/aliases", "switchyard/classify-aliases"] { + let response = send( + &app, + "POST", + "/v1/decision", + Some(json!({ + "input_format": "openai_chat", + "request": { + "model": model, + "messages": [{"role": "user", "content": "hi"}] + } + })), + ) + .await?; + assert_eq!(response.status, StatusCode::INTERNAL_SERVER_ERROR); + let body = response.json()?; + assert_eq!(body["error"]["type"], "server_error"); + assert!( + body["error"]["message"] + .as_str() + .unwrap_or_default() + .contains("model/shared") + ); + assert!( + upstream.calls.lock().await.is_empty(), + "ambiguous decisions must fail before judge calls" + ); + } + + let response = send( + &app, + "POST", + "/v1/decision", + Some(json!({ + "input_format": "openai_chat", + "request": { + "model": "switchyard/fixed", + "messages": [{"role": "user", "content": "hi"}] + } + })), + ) + .await?; + assert_eq!(response.status, StatusCode::OK); + let body = response.json()?; + assert_eq!(body["selected"]["target"], "parent"); + assert_eq!(body["selected"]["model"], "model/shared"); + assert_eq!(body["fallbacks"], json!([])); + assert!(upstream.calls.lock().await.is_empty()); + Ok(()) +} + /// Decision-only routing returns callable metadata and preserves any answer produced while routing. #[tokio::test] async fn decision_returns_callable_target_and_routing_answer() -> TestResult { diff --git a/docs/reference/toml_schema.md b/docs/reference/toml_schema.md index 0b5f905e9..54a246529 100644 --- a/docs/reference/toml_schema.md +++ b/docs/reference/toml_schema.md @@ -112,11 +112,13 @@ This includes completion targets and targets used for judging or classification. same-model targets on different clients within a route because execution is keyed by model ID. Separate routes may use the same model ID on different clients. -Distinct completion target keys within one route must have distinct model IDs. -Routing decisions identify models, so aliases sharing an ID would make selected -and fallback target identity ambiguous. The same target key may be reused by the -parent and subagent routes. Judge-only targets may share a completion model ID -when the client and prompt rules below permit it. +The Python decision API and `/v1/decision` require distinct completion target keys +within the requested route to have distinct model IDs. Routing decisions identify +models, so aliases sharing an ID make selected and fallback target names ambiguous. +These APIs reject that route before provider calls. Identical aliases remain valid +for ordinary serving; other routes in the deployment can still return decisions. +The same target key may be reused by the parent and subagent routes. Judge-only +targets may share a completion model ID when the client and prompt rules below permit it. Each selected or fallback target is prepared from the routed request independently. A prompt configured for one target is therefore not carried into another target's fallback request. diff --git a/docs/simulation.md b/docs/simulation.md index 7a5f745c8..cee51bbb1 100644 --- a/docs/simulation.md +++ b/docs/simulation.md @@ -90,11 +90,12 @@ to `evaluate` or `score` when the two systems name the same model differently. A the library does not guess equivalence by trimming model names. Missing model metadata remains visible as unverified coverage. -Distinct completion targets in one route must have distinct model IDs. Native +For task evaluation, distinct completion targets in one route must have distinct model IDs. Native decisions identify models, so two target keys sharing an ID cannot be scored -separately; configuration loading rejects that ambiguity. Reuse one target key +separately; decision validation rejects that route before provider calls. Reuse one target key for the same candidate, or configure distinct served model IDs. Separate routes -can still use different target keys for the same model. +can still use different target keys for the same model. Identical aliases remain +valid for ordinary serving and do not prevent evaluating other routes in the deployment. ## Use ATIF or custom recordings diff --git a/tests/test_runner_bindings.py b/tests/test_runner_bindings.py index ad08cfd70..ff2f3035a 100644 --- a/tests/test_runner_bindings.py +++ b/tests/test_runner_bindings.py @@ -385,12 +385,27 @@ async def test_caller_headers_are_request_scoped_and_respect_configured_auth( assert "private prompt" not in str(caught.value) -@pytest.mark.parametrize("kind", ["classifier", "random", "random-zero", "random-reversed"]) -def test_ambiguous_completion_target_aliases_fail_before_calls( +@pytest.mark.parametrize( + "kind", ["classifier", "random", "random-zero", "random-reversed", "subagent"] +) +async def test_ambiguous_completion_target_aliases_fail_before_calls( judge: JudgeStub, tmp_path: Path, kind: str ) -> None: source = deployment(judge.url).replace('id = "strong/model"', 'id = "weak/model"') - if kind != "classifier": + if kind == "subagent": + source = ( + source.split("[routes.classifier]")[0] + + """ +[routes.shared] +id = "auto" +type = "passthrough" +target = "weak" +[routes.shared.subagents] +type = "passthrough" +target = "strong" +""" + ) + elif kind != "classifier": targets = '["strong", "weak"]' if kind == "random-reversed" else '["weak", "strong"]' weights = "[0, 1]" if kind == "random-zero" else "[1, 99]" source = ( @@ -404,11 +419,24 @@ def test_ambiguous_completion_target_aliases_fail_before_calls( seed = 1 """ ) + if kind != "classifier": + source += """ +[routes.fixed] +id = "fixed" +type = "passthrough" +target = "weak" +""" path = tmp_path / "routes.toml" path.write_text(source) for load in (lambda: Runner.from_toml(source), lambda: Runner.load(path)): + runner = load() with pytest.raises(ValueError, match="completion targets.*model"): - load() + runner.validate_decision_route("auto") + with pytest.raises(ValueError, match="completion targets.*model"): + await runner.decide(request()) + decision = await runner.decide(request(model="fixed")) + assert decision.selected.target == "weak" + assert decision.calls == [] assert judge.calls == [] From 5b1ae69166b79fc48d2c7e11e85be66e174b9c79 Mon Sep 17 00:00:00 2001 From: ayushag Date: Wed, 30 Sep 2026 04:01:40 -0700 Subject: [PATCH 27/48] docs(sim): retain rows for reproducible policy comparisons Signed-off-by: ayushag --- docs/simulation.md | 24 ++++++++++++++++++++++-- 1 file changed, 22 insertions(+), 2 deletions(-) diff --git a/docs/simulation.md b/docs/simulation.md index cee51bbb1..a7fcc6045 100644 --- a/docs/simulation.md +++ b/docs/simulation.md @@ -290,14 +290,17 @@ from switchyard.sim import evaluate async def compare(dataset): summaries = {} + rows = {} for name, path in {"fixed": "routes.toml", "candidate": "candidate.toml"}.items(): + rows[name] = [] report = await evaluate( dataset, Runner.load(path), route="auto", concurrency=1, + on_result=rows[name].append, ) summaries[name] = report.to_dict() - return summaries + return summaries, rows -summaries = asyncio.run(compare(dataset)) +summaries, rows = asyncio.run(compare(dataset)) for name, summary in summaries.items(): print(json.dumps({ "policy": name, @@ -316,6 +319,23 @@ supply the same `price_call` function to each evaluation to include classifier cost estimates. If coverage is partial, use `on_result` rows to compare the same task IDs across policies; matching counts alone do not establish a common cohort. +The example retains each policy's `Result` objects in `rows[name]`. For a partial +reward comparison, keep successful rows whose selected reward and every fixed-target +reward are known. Intersect those task IDs across policies, then recompute every +policy, fixed-target, and oracle mean on that intersection. Record the included and +excluded task IDs. Repeat this process separately for cost, requiring known routing +cost when comparing totals that include classifier calls. The saved rows contain +the recorded outcomes needed for these comparisons; no new routing calls are needed. +For large evaluations, use a synchronous callback to stream policy-tagged rows to +storage instead of retaining them in memory. + +Keep the original recordings or their immutable content identities, converter code +version, task splits, `input_target`, configuration snapshots, Switchyard revision, +evaluation options, and pricing rules alongside those rows. The report's `run_id` +identifies an evaluation; it does not fingerprint its inputs or policy. Python API +calls leave artifact storage to the caller. The Harbor CLI records configuration +hashes and input paths, so retain the referenced content as well. + Split recordings by task ID before tuning, keeping every repeat of a task in the same split. Use the development split to choose prompts, thresholds, or routing rules. Freeze that choice before evaluating a separate held-out dataset. Reusing From 13120b3373b6b79eb991eca6d705c0256e22ca19 Mon Sep 17 00:00:00 2001 From: ayushag Date: Wed, 30 Sep 2026 04:10:04 -0700 Subject: [PATCH 28/48] test(sim): cover native completion prompt scoring guards Signed-off-by: ayushag --- tests/sim/test_integration.py | 60 ++++++++++++++++++++++++++++++++++- 1 file changed, 59 insertions(+), 1 deletion(-) diff --git a/tests/sim/test_integration.py b/tests/sim/test_integration.py index 153dca662..9ae08f28d 100644 --- a/tests/sim/test_integration.py +++ b/tests/sim/test_integration.py @@ -10,7 +10,7 @@ import pytest from switchyard.runner import Runner -from switchyard.sim import Dataset, evaluate, load_harbor +from switchyard.sim import Dataset, evaluate, load_harbor, score from switchyard.sim.__main__ import main from tests.test_runner_bindings import JudgeStub, deployment from tests.test_runner_bindings import judge as judge @@ -93,6 +93,64 @@ async def test_harbor_to_classifier_scoring_and_matched_metrics(judge: JudgeStub assert not report.complete +@pytest.mark.parametrize("prompt_target", ["weak", "strong"]) +async def test_native_completion_prompt_only_invalidates_its_selected_recording( + judge: JudgeStub, tmp_path: Path, prompt_target: str +): + for target in ("weak", "strong"): + write_trial(tmp_path / target, "task", f"{target}/model", 1) + dataset = Dataset.from_runs( + {target: load_harbor(tmp_path / target, target=target) for target in ("weak", "strong")}, + input_target="weak", + ) + prompt = "ADDED_COMPLETION_PROMPT" + config = deployment(judge.url).replace( + f"[targets.{prompt_target}]", f'[targets.{prompt_target}]\nsystem_prompt = "{prompt}"' + ) + task = dataset.tasks[0] + decision = await Runner.from_toml(config).decide( + {"model": "auto", "messages": list(task.messages)} + ) + rejected = prompt_target == "weak" + assert decision.selected.target == "weak" + assert (prompt in json.dumps(decision.outcome.request)) is rejected + if rejected: + with pytest.raises(ValueError, match="rewritten task request"): + score(task, decision) + else: + assert score(task, decision).outcome == task.outcomes["weak"] + + rows = [] + report = await evaluate( + dataset, + Runner.from_toml(config), + route="auto", + on_result=rows.append, + price_call=lambda _: 0.01, + ) + (row,) = rows + assert row.target == "weak" and row.model == "weak/model" + assert (row.outcome is None) is rejected + assert (row.error is not None) is rejected + assert row.decision_id and row.evidence["source"] == "llm-classifier" + assert row.fallbacks == ("strong",) + assert row.algorithm == decision.outcome.metadata.algorithm + if rejected: + assert row.error == "recorded task outcomes cannot score a rewritten task request" + assert row.routing_calls == 1 and row.routing_failed_calls == 0 + assert row.routing_cost_usd == 0.01 + assert row.routing_usage["input_tokens"] == 12 + assert row.routing_usage["output_tokens"] == 8 + summary = report.to_dict() + assert summary["counts"]["scored"] == int(not rejected) + assert summary["counts"]["errors"] == int(rejected) + assert summary["comparison"]["tasks"] == int(not rejected) + assert summary["cost_comparison"]["tasks"] == int(not rejected) + assert summary["routing"]["cost_usd"]["total"] == 0.01 + assert [call["model"] for call in judge.calls] == ["judge/model", "judge/model"] + assert prompt not in json.dumps(judge.calls) + + def test_cli_writes_flushed_rows_manifest_and_report_without_copying_config( judge: JudgeStub, tmp_path: Path ): From cd2e62c88c387c37de36732ea489f2232434c568 Mon Sep 17 00:00:00 2001 From: ayushag Date: Wed, 30 Sep 2026 04:40:00 -0700 Subject: [PATCH 29/48] perf(runner): precompute decision target validation Signed-off-by: ayushag --- crates/switchyard-runner/src/config.rs | 10 +++++++-- crates/switchyard-runner/src/route.rs | 29 ++++++++++++++++---------- 2 files changed, 26 insertions(+), 13 deletions(-) diff --git a/crates/switchyard-runner/src/config.rs b/crates/switchyard-runner/src/config.rs index b0c796c17..039a10b04 100644 --- a/crates/switchyard-runner/src/config.rs +++ b/crates/switchyard-runner/src/config.rs @@ -1592,9 +1592,10 @@ llm_client = "primary" "{ALIASED_TARGETS}\n[routes.shared]\nid = \"switchyard/shared\"\n{algorithm}" ); let runner = runner_from_toml(&configured)?; - let message = runner + let route = runner .route("switchyard/shared") - .expect("shared route should load") + .expect("shared route should load"); + let message = route .validate_decision_targets() .expect_err("aliased completion targets cannot identify a named decision") .to_string(); @@ -1604,6 +1605,10 @@ llm_client = "primary" && message.contains("distinct model ids"), "{algorithm}: {message}" ); + assert_eq!( + route.validate_decision_targets().unwrap_err().to_string(), + message + ); } Ok(()) } @@ -1618,6 +1623,7 @@ llm_client = "primary" .route("switchyard/passthrough") .expect("passthrough route should exist"); route.validate_decision_targets()?; + route.validate_decision_targets()?; assert_eq!( route .decision_targets() diff --git a/crates/switchyard-runner/src/route.rs b/crates/switchyard-runner/src/route.rs index 51f8db8b4..fb4016c67 100644 --- a/crates/switchyard-runner/src/route.rs +++ b/crates/switchyard-runner/src/route.rs @@ -118,6 +118,8 @@ pub struct Route { anthropic_auxiliary_target: Option, responses_auxiliary_target: Option, decision_targets: Vec, + // Targets are immutable; only named-decision consumers surface this error. + decision_target_error: Option, models: Arc, } @@ -140,6 +142,18 @@ impl Route { decision_targets: Vec, models: RuntimeModels, ) -> Self { + let decision_target_error = { + let mut names_by_model = HashMap::new(); + decision_targets.iter().find_map(|target| { + let first_name = names_by_model.insert(&target.model, &target.target)?; + (first_name != &target.target).then(|| { + format!( + "completion targets {first_name} and {} both use model {}; routing decisions identify models, so reuse one target key, use distinct model ids, or put these targets in separate routes", + target.target, target.model + ) + }) + }) + }; Self { algorithm, clients, @@ -148,6 +162,7 @@ impl Route { anthropic_auxiliary_target, responses_auxiliary_target, decision_targets, + decision_target_error, models: Arc::new(models), } } @@ -172,18 +187,10 @@ impl Route { /// Serving and raw model-ID decisions may use identical aliases. Consumers /// returning named targets must check this before making routing-time calls. pub fn validate_decision_targets(&self) -> Result<(), RunnerError> { - let mut names_by_model = HashMap::new(); - for target in &self.decision_targets { - if let Some(first_name) = names_by_model.insert(&target.model, &target.target) - && first_name != &target.target - { - return Err(RunnerError::configuration(format!( - "completion targets {first_name} and {} both use model {}; routing decisions identify models, so reuse one target key, use distinct model ids, or put these targets in separate routes", - target.target, target.model - ))); - } + match &self.decision_target_error { + Some(message) => Err(RunnerError::configuration(message.clone())), + None => Ok(()), } - Ok(()) } /// Returns model-list capability metadata. From 41dc55b7dd7a26fe1e95d315632436f614412702 Mon Sep 17 00:00:00 2001 From: ayushag Date: Wed, 30 Sep 2026 04:55:57 -0700 Subject: [PATCH 30/48] fix(sim): preserve context when supplying missing task input Signed-off-by: ayushag --- docs/simulation.md | 4 ++++ switchyard/sim/trajectory.py | 10 ++++++---- tests/sim/test_integration.py | 33 +++++++++++++++++++++++++++++++++ tests/sim/test_trajectory.py | 22 ++++++++++++++++++++++ 4 files changed, 65 insertions(+), 4 deletions(-) diff --git a/docs/simulation.md b/docs/simulation.md index a7fcc6045..3ff9c4f8d 100644 --- a/docs/simulation.md +++ b/docs/simulation.md @@ -170,6 +170,10 @@ full history. The generator above therefore releases each full trajectory before reading the next record; `Run` retains only the trials. `HarborRun` is a compatibility alias for `Run`. Neither path requires Harbor to be installed. +When a recording has no nonblank initial user message, `task_input` appends the +original task text after any recorded initial system and user messages. Without +that fallback, projection fails. + Copied continuation context can contain earlier answers or progress summaries. If an initial step has `is_copied_context=true`, projection rejects it unless you provide the original task text with `to_trial(task_input=...)` or diff --git a/switchyard/sim/trajectory.py b/switchyard/sim/trajectory.py index db69d1c29..fb9c0e698 100644 --- a/switchyard/sim/trajectory.py +++ b/switchyard/sim/trajectory.py @@ -38,9 +38,9 @@ def to_dict(self) -> dict[str, Any]: def initial_messages(self, *, task_input: str | None = None) -> tuple[dict[str, object], ...]: """Extract text input before the first agent step, without later evidence. - ``task_input`` supplies original task text when the recording lacks it - or contains copied continuation context. Copied context may summarize - earlier execution, so it is never used as task-routing input. + ``task_input`` appends missing original task text to the initial context. + Copied continuation context may summarize earlier execution, so it is + replaced entirely by ``task_input`` and never used as task-routing input. """ return _initial_messages(self._data, task_input) @@ -133,7 +133,9 @@ def _initial_messages( if blocks: messages.append({"role": role, "content": blocks}) has_user_input |= role == "user" and any(block["text"].strip() for block in blocks) - return tuple(messages) if has_user_input else _task_messages(task_input) + if not has_user_input: + messages.extend(_task_messages(task_input)) + return tuple(messages) def _task_messages(text: str | None) -> tuple[dict[str, object], ...]: diff --git a/tests/sim/test_integration.py b/tests/sim/test_integration.py index 9ae08f28d..9ff2af018 100644 --- a/tests/sim/test_integration.py +++ b/tests/sim/test_integration.py @@ -93,6 +93,39 @@ async def test_harbor_to_classifier_scoring_and_matched_metrics(judge: JudgeStub assert not report.complete +async def test_missing_task_prompt_keeps_system_constraints_in_native_routing( + judge: JudgeStub, tmp_path: Path +): + for target in ("weak", "strong"): + write_trial(tmp_path / target, "task", f"{target}/model", float(target == "strong")) + path = tmp_path / "weak" / "task__attempt" / "agent" / "trajectory.json" + trajectory = json.loads(path.read_text()) + trajectory["steps"] = [ + {"source": "system", "message": "TASK_REQUIRES_STRONG"}, + {"source": "agent", "message": "FUTURE_ANSWER_MUST_NOT_LEAK"}, + ] + path.write_text(json.dumps(trajectory)) + dataset = Dataset.from_runs( + { + target: load_harbor(tmp_path / target, target=target, task_inputs={"task": "easy task"}) + for target in ("weak", "strong") + }, + input_target="weak", + ) + rows = [] + config = deployment(judge.url).replace( + 'type = "llm_classifier"', 'type = "llm_classifier"\nrecent_turn_window = 0' + ) + report = await evaluate(dataset, Runner.from_toml(config), route="auto", on_result=rows.append) + assert report.complete + assert rows[0].target == "strong" + assert rows[0].outcome.reward == 1 + assert [call["model"] for call in judge.calls] == ["judge/model"] + assert "TASK_REQUIRES_STRONG" in json.dumps(judge.calls) + assert "easy task" in json.dumps(judge.calls) + assert "FUTURE_ANSWER_MUST_NOT_LEAK" not in json.dumps(judge.calls) + + @pytest.mark.parametrize("prompt_target", ["weak", "strong"]) async def test_native_completion_prompt_only_invalidates_its_selected_recording( judge: JudgeStub, tmp_path: Path, prompt_target: str diff --git a/tests/sim/test_trajectory.py b/tests/sim/test_trajectory.py index f9acbf3ef..a5fe07618 100644 --- a/tests/sim/test_trajectory.py +++ b/tests/sim/test_trajectory.py @@ -212,6 +212,28 @@ def test_multimodal_input_is_preserved_but_cannot_be_scored_as_text() -> None: trajectory.initial_messages() +@pytest.mark.parametrize("blank_user", [False, True]) +def test_missing_task_input_preserves_recorded_initial_context(blank_user: bool) -> None: + data = atif() + prefix = [{"source": "system", "message": "Use the recorded constraints."}] + if blank_user: + prefix.append({"source": "user", "message": " \t"}) + data["steps"] = [*prefix, {"source": "agent", "message": "LATER_ANSWER"}] + trajectory = Trajectory.from_dict(data) + with pytest.raises(ValueError, match="missing initial task input"): + trajectory.initial_messages() + expected = tuple( + {"role": step["source"], "content": [{"type": "text", "text": step["message"]}]} + for step in [*prefix, {"source": "user", "message": "Canonical task."}] + ) + assert trajectory.initial_messages(task_input="Canonical task.") == expected + trial = trajectory.to_trial( + task_id="task", trial_id="attempt", target="fast", task_input="Canonical task." + ) + assert trial.messages == expected + assert trajectory.to_dict() == data + + def test_copied_continuation_context_requires_original_task_input() -> None: data = atif() data["steps"][1].update(is_copied_context=True, message="SUMMARY_OF_COMPLETED_WORK") From 9742f8ee86deaf14cbb3eb86f92c5495bda9c05e Mon Sep 17 00:00:00 2001 From: ayushag Date: Wed, 30 Sep 2026 05:16:18 -0700 Subject: [PATCH 31/48] fix(sim): keep CLI output outside recorded runs Signed-off-by: ayushag --- docs/simulation.md | 6 ++- switchyard/sim/__main__.py | 26 +++++++--- tests/sim/test_integration.py | 94 +++++++++++++++++++++++++++++++++++ 3 files changed, 119 insertions(+), 7 deletions(-) diff --git a/docs/simulation.md b/docs/simulation.md index 3ff9c4f8d..df1e6ca29 100644 --- a/docs/simulation.md +++ b/docs/simulation.md @@ -355,7 +355,11 @@ python -m switchyard.sim \ ``` This CLI imports Harbor layouts. Use the Python API for custom formats. -The output directory must be new. The CLI writes a `manifest.json` with the +The output directory must be new and outside every supplied recording directory. +Paths are resolved before this check, including symlink aliases and `..` components. +This prevents output folders from changing Harbor's trial discovery on later imports. +Existing output entries, including dangling symlinks, are rejected. +The CLI writes a `manifest.json` with the configuration hash, inputs, package version, and coverage, then flushes each completed row to `results.jsonl`. It writes `report.json` after evaluation returns. The hash covers the exact TOML bytes loaded. Invalid routes and recorded-model diff --git a/switchyard/sim/__main__.py b/switchyard/sim/__main__.py index 22a447c05..7889bb43f 100644 --- a/switchyard/sim/__main__.py +++ b/switchyard/sim/__main__.py @@ -20,6 +20,14 @@ from .evaluate import _aliases, _validate_options, _validate_route +def _resolve_path(path: Path) -> Path: + try: + return path.resolve() + except RuntimeError as error: + # Python < 3.13 reports symlink loops as RuntimeError rather than OSError. + raise ValueError(f"cannot resolve path: {path}") from error + + def main(argv: list[str] | None = None) -> int: parser = argparse.ArgumentParser( description="Evaluate Switchyard task routing on recorded Harbor runs." @@ -39,7 +47,7 @@ def main(argv: list[str] | None = None) -> int: "--input-target", required=True, help="Target whose initial agent input the router sees" ) parser.add_argument( - "--output", required=True, type=Path, help="New output directory (must not already exist)" + "--output", required=True, type=Path, help="New output directory outside all recorded runs" ) parser.add_argument("--dataset", help="Optional task namespace") parser.add_argument("--reward-key", default="reward") @@ -71,7 +79,7 @@ def main(argv: list[str] | None = None) -> int: target, separator, path = value.partition("=") if not separator or not target or not path or target in paths: parser.error("each --run must be a unique TARGET=PATH") - paths[target] = Path(path).resolve() + paths[target] = Path(path) aliases = {} for value in args.model_alias: recorded, separator, configured = value.partition("=") @@ -80,6 +88,12 @@ def main(argv: list[str] | None = None) -> int: aliases[recorded] = configured try: _validate_options(args.concurrency, args.timeout) + paths = {target: _resolve_path(path) for target, path in paths.items()} + if args.output.is_symlink(): + raise FileExistsError(f"output already exists: {args.output}") + output = _resolve_path(args.output) + if any(output.is_relative_to(path) for path in paths.values()): + raise ValueError("output directory must be outside every recorded run directory") runs = { target: load_harbor( path, @@ -97,7 +111,7 @@ def main(argv: list[str] | None = None) -> int: config_bytes = args.config.read_bytes() runner = Runner.from_toml(config_bytes.decode("utf-8")) _validate_route(dataset, runner, args.route, _aliases(aliases)) - args.output.mkdir(parents=True, exist_ok=False) + output.mkdir(parents=True, exist_ok=False) manifest = { "schema_version": 1, "switchyard_version": __version__, @@ -112,10 +126,10 @@ def main(argv: list[str] | None = None) -> int: "timeout_seconds": args.timeout, "coverage": dict(dataset.coverage), } - (args.output / "manifest.json").write_text( + (output / "manifest.json").write_text( json.dumps(manifest, indent=2, allow_nan=False) + "\n" ) - with (args.output / "results.jsonl").open("x", encoding="utf-8") as stream: + with (output / "results.jsonl").open("x", encoding="utf-8") as stream: def save(result: Result) -> None: stream.write(json.dumps(asdict(result), allow_nan=False) + "\n") @@ -132,7 +146,7 @@ def save(result: Result) -> None: model_aliases=aliases, ) ) - (args.output / "report.json").write_text( + (output / "report.json").write_text( json.dumps(report.to_dict(), indent=2, allow_nan=False) + "\n" ) except (OSError, ValueError) as error: diff --git a/tests/sim/test_integration.py b/tests/sim/test_integration.py index 9ff2af018..e4eecdc29 100644 --- a/tests/sim/test_integration.py +++ b/tests/sim/test_integration.py @@ -217,6 +217,100 @@ def test_cli_writes_flushed_rows_manifest_and_report_without_copying_config( assert (output / "report.json").read_text() == original +@pytest.fixture +def cli_recording(judge: JudgeStub, tmp_path: Path): + root = tmp_path / "recordings" + write_trial(root, "task", "weak/model", 1) + config = tmp_path / "routes.toml" + config.write_text(deployment(judge.url)) + + def invoke(run: Path, output: Path) -> int: + return main( + [ + "--config", + str(config), + "--route", + "fixed", + "--run", + f"weak={run}", + "--input-target", + "weak", + "--output", + str(output), + ] + ) + + return root, invoke + + +@pytest.mark.parametrize( + "placement", ["jobs", "agent", "new-parent/agent", "evaluation", "output-alias", "input-alias"] +) +def test_cli_rejects_output_inside_recordings(cli_recording, placement, capsys, judge): + root, invoke = cli_recording + run = root + output = root / placement + if placement in ("output-alias", "input-alias"): + alias = root.parent / "recording-alias" + alias.symlink_to(root, target_is_directory=True) + if placement == "output-alias": + output = alias / "jobs" + else: + run = alias + output = root / "jobs" + before = sorted(root.rglob("*")) + assert invoke(run, output) == 2 + assert "outside every recorded run directory" in capsys.readouterr().err + assert sorted(root.rglob("*")) == before + assert len(load_harbor(root, target="weak").trials) == 1 + assert judge.calls == [] + + +@pytest.mark.parametrize("placement", ["sibling", "parent-traversal"]) +def test_cli_resolves_outside_output_without_creating_input_directories( + cli_recording, placement, judge +): + root, invoke = cli_recording + output = ( + root.parent / "recordings-evaluation" + if placement == "sibling" + else root / "agent" / ".." / ".." / "evaluation" + ) + before = sorted(root.rglob("*")) + assert invoke(root, output) == 0 + assert sorted(root.rglob("*")) == before + assert len(load_harbor(root, target="weak").trials) == 1 + report = json.loads((output.resolve() / "report.json").read_text()) + assert report["complete"] and report["counts"]["scored"] == 1 + assert judge.calls == [] + + +def test_cli_rejects_dangling_output_symlink(cli_recording, capsys, judge): + root, invoke = cli_recording + output = root.parent / "evaluation" + destination = root.parent / "missing" + output.symlink_to(destination, target_is_directory=True) + assert invoke(root, output) == 2 + assert capsys.readouterr().err.startswith("switchyard.sim:") + assert output.is_symlink() and not destination.exists() + assert judge.calls == [] + + +@pytest.mark.parametrize("location", ["run", "output-parent"]) +def test_cli_reports_cyclic_paths_as_file_errors(cli_recording, location, capsys, judge): + root, invoke = cli_recording + loop = root.parent / "loop" + loop.symlink_to(loop, target_is_directory=True) + run = loop if location == "run" else root + output = root.parent / "evaluation" if location == "run" else loop / "evaluation" + assert invoke(run, output) == 2 + error = capsys.readouterr().err + assert error.startswith("switchyard.sim:") and "Traceback" not in error + assert not (root.parent / "evaluation").exists() + assert len(load_harbor(root, target="weak").trials) == 1 + assert judge.calls == [] + + def test_cli_exits_incomplete_for_missing_reward(judge: JudgeStub, tmp_path: Path): write_trial(tmp_path / "recordings", "task", "weak/model", None) config = tmp_path / "routes.toml" From 3256e4402b4b3d4f2e85ea7d9485738b1d4a91ef Mon Sep 17 00:00:00 2001 From: ayushag Date: Wed, 30 Sep 2026 05:25:41 -0700 Subject: [PATCH 32/48] fix(sim): reject duplicate Harbor trial directory aliases Signed-off-by: ayushag --- docs/simulation.md | 2 ++ switchyard/sim/harbor.py | 5 +++++ tests/sim/test_harbor.py | 46 ++++++++++++++++++++++++++++++++++++++++ 3 files changed, 53 insertions(+) diff --git a/docs/simulation.md b/docs/simulation.md index df1e6ca29..6026be5ed 100644 --- a/docs/simulation.md +++ b/docs/simulation.md @@ -206,6 +206,8 @@ missing, all initial user messages must match. Full system scaffolding may diffe between agents. Repeated trials retain their identities; each target's reward, cost, and duration are averaged across all repeats of a task. Tasks then have equal weight regardless of their number of repeats. +Within one import, duplicate trial IDs and directory aliases resolving to the same +trial directory are rejected, so symlinks cannot count one recorded trial twice. Missing rewards, cost, duration, and token counts stay unknown. A task measurement is unknown if any of its repeats lacks that measurement. Agent errors remain on diff --git a/switchyard/sim/harbor.py b/switchyard/sim/harbor.py index 9a7af34f6..88abac6f6 100644 --- a/switchyard/sim/harbor.py +++ b/switchyard/sim/harbor.py @@ -66,6 +66,7 @@ def load_harbor( trials: list[Trial] = [] trial_ids: set[str] = set() + trial_dirs: set[Path] = set() for result_path in paths: task_id = None try: @@ -77,7 +78,11 @@ def load_harbor( ) if trial.trial_id in trial_ids: raise ValueError("duplicate trial ID") + trial_dir = result_path.parent.resolve() + if trial_dir in trial_dirs: + raise ValueError("duplicate trial directory") trial_ids.add(trial.trial_id) + trial_dirs.add(trial_dir) trials.append(trial) except (OSError, ValueError) as error: message = str(error) if isinstance(error, ValueError) else "cannot read trial artifact" diff --git a/tests/sim/test_harbor.py b/tests/sim/test_harbor.py index 91e20dcae..8c5002846 100644 --- a/tests/sim/test_harbor.py +++ b/tests/sim/test_harbor.py @@ -171,6 +171,52 @@ def test_repeated_tasks_are_preserved_as_distinct_trials(tmp_path: Path) -> None assert trials[0].trial_id != trials[1].trial_id +def test_distinct_trial_directories_can_share_result_metadata(tmp_path: Path) -> None: + first = write_trial(tmp_path, "first") + second = write_trial(tmp_path, "second") + update(first / "result.json", id=None) + (second / "result.json").unlink() + (second / "result.json").symlink_to(first / "result.json") + run = load_harbor(tmp_path, target="baseline") + assert run.issues == () + assert [trial.trial_id for trial in run.trials] == ["first", "second"] + + +@pytest.mark.parametrize("layout", ["job", "download"]) +@pytest.mark.parametrize("id_mode", ["absent", "null"]) +def test_trial_directory_alias_cannot_bias_repeat_outcomes( + tmp_path: Path, layout: str, id_mode: str +) -> None: + job = tmp_path / "jobs" / "run" if layout == "download" else tmp_path + for name, reward in (("repeat-1", 1), ("repeat-2", 0)): + path = write_trial(job, name) / "result.json" + result = json.loads(path.read_text()) + result.pop("id") + if id_mode == "null": + result["id"] = None + result["verifier_result"]["rewards"]["reward"] = reward + path.write_text(json.dumps(result)) + write_trial(job, "clean", task="unaffected") + run = load_harbor(tmp_path, target="baseline", dataset="suite") + dataset = Dataset.from_runs({"baseline": run}, input_target="baseline") + assert dataset.tasks[0].outcomes["baseline"].reward == 0.5 + alias = job / "repeat-1-alias" + alias.symlink_to(job / "repeat-1", target_is_directory=True) + + with pytest.raises(ValueError, match="duplicate trial directory"): + load_harbor(tmp_path, target="baseline", dataset="suite") + run = load_harbor(tmp_path, target="baseline", dataset="suite", on_error="record") + assert len(run.trials) == 3 + (issue,) = run.issues + assert issue.source == str(alias / "result.json") + assert issue.task_id == "suite/benchmark/task-a" + with pytest.raises(ValueError, match="incomplete cohort"): + Dataset.from_runs({"baseline": run}, input_target="baseline") + dataset = Dataset.from_runs({"baseline": run}, input_target="baseline", intersection=True) + assert [task.task_id for task in dataset.tasks] == ["suite/unaffected"] + assert dataset.coverage["excluded_task_ids"] == [issue.task_id] + + @pytest.mark.parametrize("layout", ["job", "download"]) @pytest.mark.parametrize("remaining", [0, 1]) def test_missing_repeat_directory_is_not_silently_dropped( From 4474e4551c4c1af959929b1c1cca2e67b169fad3 Mon Sep 17 00:00:00 2001 From: ayushag Date: Wed, 30 Sep 2026 05:51:50 -0700 Subject: [PATCH 33/48] docs(sim): explain importer and dataset memory ownership Signed-off-by: ayushag --- docs/simulation.md | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/docs/simulation.md b/docs/simulation.md index 6026be5ed..2126db092 100644 --- a/docs/simulation.md +++ b/docs/simulation.md @@ -456,9 +456,13 @@ explicit `allow_response=True`; task simulation rejects them. task evidence. `Dataset.from_runs` validates and pairs it. `score(task, decision)` is a pure scorer for a saved native decision. `evaluate` owns bounded scheduling and calls that scorer. `Report.add` accumulates -results without retaining full trajectories or result rows. Import processes -one trajectory file at a time; paired task inputs and small outcome records stay -in memory. +results without retaining full trajectories or result rows. Harbor import decodes +one complete trajectory JSON file at a time, then keeps each accepted trial's initial +input and metadata. A dataset retains those trials for every included task, across +targets and repeats. `input_target` chooses the routing input without discarding the +other trial inputs. +Memory therefore depends on both the largest trajectory being decoded and the +retained trial inputs. Concurrency bounds in-flight decisions, not the loaded dataset size. Treat projected `Trial`, `Task`, and `Dataset` records as read-only. Their frozen dataclasses still contain mutable nested mappings. If you change recorded input, From ca7f236dbb873edc5fa30de6ca8bae27cd6b7d0f Mon Sep 17 00:00:00 2001 From: ayushag Date: Wed, 30 Sep 2026 06:05:06 -0700 Subject: [PATCH 34/48] perf(sim): release excluded trials before CLI routing Signed-off-by: ayushag --- docs/simulation.md | 2 + switchyard/sim/__main__.py | 1 + tests/sim/test_integration.py | 74 +++++++++++++++++++++++++++++++++++ 3 files changed, 77 insertions(+) diff --git a/docs/simulation.md b/docs/simulation.md index 2126db092..7e1389d2f 100644 --- a/docs/simulation.md +++ b/docs/simulation.md @@ -461,6 +461,8 @@ one complete trajectory JSON file at a time, then keeps each accepted trial's in input and metadata. A dataset retains those trials for every included task, across targets and repeats. `input_target` chooses the routing input without discarding the other trial inputs. +The CLI releases excluded trial inputs after pairing; library callers must release +their original `Run` objects when they no longer need them. Memory therefore depends on both the largest trajectory being decoded and the retained trial inputs. Concurrency bounds in-flight decisions, not the loaded dataset size. diff --git a/switchyard/sim/__main__.py b/switchyard/sim/__main__.py index 7889bb43f..64d41d355 100644 --- a/switchyard/sim/__main__.py +++ b/switchyard/sim/__main__.py @@ -108,6 +108,7 @@ def main(argv: list[str] | None = None) -> int: dataset = Dataset.from_runs( runs, input_target=args.input_target, intersection=args.intersection ) + del runs # Dataset owns included trials; release excluded inputs before routing. config_bytes = args.config.read_bytes() runner = Runner.from_toml(config_bytes.decode("utf-8")) _validate_route(dataset, runner, args.route, _aliases(aliases)) diff --git a/tests/sim/test_integration.py b/tests/sim/test_integration.py index e4eecdc29..e8db5707d 100644 --- a/tests/sim/test_integration.py +++ b/tests/sim/test_integration.py @@ -5,6 +5,7 @@ import hashlib import json +import weakref from pathlib import Path import pytest @@ -217,6 +218,79 @@ def test_cli_writes_flushed_rows_manifest_and_report_without_copying_config( assert (output / "report.json").read_text() == original +def test_cli_releases_excluded_trials_before_routing( + judge: JudgeStub, tmp_path: Path, monkeypatch: pytest.MonkeyPatch +): + for target in ("weak", "strong"): + write_trial(tmp_path / target, "shared", f"{target}/model", 1) + write_trial(tmp_path / target, "bad", f"{target}/model", 0) + write_trial(tmp_path / "weak", "weak-only", "weak/model", 0) + write_trial(tmp_path / "weak", "bad-repeat", "weak/model", 0) + invalid = tmp_path / "weak" / "bad-repeat__attempt" / "result.json" + data = json.loads(invalid.read_text()) + data.update(task_name="bad", verifier_result={"rewards": {"reward": "invalid"}}) + invalid.write_text(json.dumps(data)) + config = tmp_path / "routes.toml" + config.write_text(deployment(judge.url)) + run_refs = [] + trial_refs = [] + + def capture(*args, **kwargs): + run = load_harbor(*args, **kwargs) + run_refs.append(weakref.ref(run)) + trial_refs.extend((trial.task_id, weakref.ref(trial)) for trial in run.trials) + return run + + async def verify(dataset, *args, **kwargs): + assert len(run_refs) == 2 and all(ref() is None for ref in run_refs) + assert sum(name != "shared" for name, _ in trial_refs) == 3 + assert all(ref() is None for name, ref in trial_refs if name != "shared") + (task,) = dataset.tasks + assert task.task_id == "shared" + assert {id(trial) for trials in task.trials.values() for trial in trials} == { + id(ref()) for name, ref in trial_refs if name == "shared" + } + return await evaluate(dataset, *args, **kwargs) + + monkeypatch.setattr("switchyard.sim.__main__.load_harbor", capture) + monkeypatch.setattr("switchyard.sim.__main__.evaluate", verify) + output = tmp_path / "evaluation" + assert ( + main( + [ + "--config", + str(config), + "--route", + "fixed", + "--run", + f"weak={tmp_path / 'weak'}", + "--run", + f"strong={tmp_path / 'strong'}", + "--input-target", + "weak", + "--intersection", + "--skip-invalid", + "--output", + str(output), + ] + ) + == 0 + ) + manifest = json.loads((output / "manifest.json").read_text()) + report = json.loads((output / "report.json").read_text()) + coverage = manifest["coverage"] + assert coverage.items() <= report["coverage"].items() + assert coverage["tasks_seen"] == 3 and coverage["tasks_included"] == 1 + assert coverage["excluded_task_ids"] == ["bad", "weak-only"] + assert coverage["cost_sources_by_target"] == {"weak": {"result": 3}, "strong": {"result": 2}} + assert coverage["input_issues"][0]["task_id"] == "bad" + assert coverage["input_issues"][0]["source"] == str(invalid) + (row,) = [json.loads(line) for line in (output / "results.jsonl").read_text().splitlines()] + assert row["task_id"] == "shared" and row["target"] == "weak" + assert row["outcome"]["reward"] == 1 and report["complete"] + assert judge.calls == [] + + @pytest.fixture def cli_recording(judge: JudgeStub, tmp_path: Path): root = tmp_path / "recordings" From 155bcb2c1b96fbdce0c25e76649f8323f2c4caa1 Mon Sep 17 00:00:00 2001 From: ayushag Date: Wed, 30 Sep 2026 06:27:26 -0700 Subject: [PATCH 35/48] fix(sim): preserve finite means and signed reward totals Signed-off-by: ayushag --- docs/simulation.md | 5 +++ switchyard/sim/models.py | 13 ++++-- switchyard/sim/report.py | 91 +++++++++++++++++++++++++++++----------- tests/sim/test_report.py | 65 ++++++++++++++++++++++++++++ 4 files changed, 145 insertions(+), 29 deletions(-) diff --git a/docs/simulation.md b/docs/simulation.md index 7e1389d2f..1a36cb383 100644 --- a/docs/simulation.md +++ b/docs/simulation.md @@ -252,6 +252,11 @@ and means are `null`; `observed_total` retains the known portion. `complete` means every expected task has a scored reward and no evaluation error. Inspect each cost and usage field's coverage separately. +Signed rewards are supported. Totals preserve small values when larger positive +and negative rewards cancel, regardless of task completion order. Published +aggregates remain floating-point numbers. `Report.to_dict()` raises `ValueError` +if an exposed aggregate exceeds the finite floating-point range. + Native routing failures retain `routing_error_kind`, `routing_error_status`, and `routing_error_target` on each result row. The status is an upstream HTTP code when available; the target identifies the failing model, such as the classifier. diff --git a/switchyard/sim/models.py b/switchyard/sim/models.py index b2cf15b6a..5e5c59f75 100644 --- a/switchyard/sim/models.py +++ b/switchyard/sim/models.py @@ -6,6 +6,7 @@ from __future__ import annotations import math +import statistics from collections.abc import Mapping from dataclasses import dataclass, field @@ -100,10 +101,14 @@ def from_trials(cls, trials: tuple[Trial, ...]) -> Outcome: def measurement(name: str) -> tuple[float | None, int]: values = [getattr(trial, name) for trial in trials if getattr(trial, name) is not None] - return ( - math.fsum(values) / len(trials) if len(values) == len(trials) else None, - len(values), - ) + if len(values) != len(trials): + return None, len(values) + try: + mean = math.fsum(values) / len(trials) + except OverflowError: + # A finite mean can have a sum outside the floating-point range. + mean = float(statistics.mean(values)) + return mean, len(values) reward, reward_count = measurement("reward") cost, cost_count = measurement("cost_usd") diff --git a/switchyard/sim/report.py b/switchyard/sim/report.py index 064479124..fedfd6e50 100644 --- a/switchyard/sim/report.py +++ b/switchyard/sim/report.py @@ -9,28 +9,56 @@ from collections import Counter from collections.abc import Mapping from copy import deepcopy -from dataclasses import dataclass +from dataclasses import dataclass, field +from fractions import Fraction from .models import Result +def _as_float(value: Fraction) -> float: + try: + return float(value) + except OverflowError as error: + raise ValueError("report aggregate exceeds finite floating-point range") from error + + +@dataclass +class _Sum: + """Exact sum of finite ints/floats; binary denominators are powers of two.""" + + numerator: int = 0 + denominator: int = 1 + + def add(self, value: float) -> None: + numerator, denominator = value.as_integer_ratio() + if denominator > self.denominator: + self.numerator *= denominator // self.denominator + self.denominator = denominator + self.numerator += numerator * (self.denominator // denominator) + + @property + def fraction(self) -> Fraction: + return Fraction(self.numerator, self.denominator) + + @dataclass class _Measurement: known: int = 0 - total: float = 0.0 + total: _Sum = field(default_factory=_Sum) def add(self, value: float | None) -> None: if value is not None: self.known += 1 - self.total += value + self.total.add(value) def summary(self, count: int) -> dict[str, int | float | None]: complete = count > 0 and self.known == count + total = self.total.fraction return { "known_tasks": self.known, - "observed_total": self.total if self.known else None, - "total": self.total if complete else None, - "mean": self.total / count if complete else None, + "observed_total": _as_float(total) if self.known else None, + "total": _as_float(total) if complete else None, + "mean": _as_float(total / count) if complete else None, } @@ -95,12 +123,12 @@ def __init__( self._routing_usage: dict[str, _Measurement] = {} self._estimated_cost = _Measurement() self._comparison_tasks = 0 - self._comparison_routed = 0.0 - self._comparison_targets = dict.fromkeys(targets, 0.0) - self._comparison_best = 0.0 + self._comparison_routed = _Sum() + self._comparison_targets = {target: _Sum() for target in targets} + self._comparison_best = _Sum() self._cost_comparison_tasks = 0 - self._cost_comparison_routed = 0.0 - self._cost_comparison_targets = dict.fromkeys(targets, 0.0) + self._cost_comparison_routed = _Sum() + self._cost_comparison_targets = {target: _Sum() for target in targets} self._cost_comparison_routing = _Measurement() @property @@ -171,7 +199,8 @@ def add(self, result: Result) -> None: and outcome.cost_usd is not None and result.routing_cost_usd is not None ): - self._estimated_cost.add(outcome.cost_usd + result.routing_cost_usd) + self._estimated_cost.add(outcome.cost_usd) + self._estimated_cost.total.add(result.routing_cost_usd) if outcome is not None and outcome.reward is not None: rewards = [ @@ -181,10 +210,10 @@ def add(self, result: Result) -> None: if all(reward is not None for reward in rewards): known_rewards = [reward for reward in rewards if reward is not None] self._comparison_tasks += 1 - self._comparison_routed += outcome.reward + self._comparison_routed.add(outcome.reward) for target, reward in zip(self.targets, known_rewards, strict=True): - self._comparison_targets[target] += reward - self._comparison_best += max(known_rewards) + self._comparison_targets[target].add(reward) + self._comparison_best.add(max(known_rewards)) if outcome is not None and outcome.cost_usd is not None: costs = [ @@ -193,17 +222,19 @@ def add(self, result: Result) -> None: ] if all(cost is not None for cost in costs): self._cost_comparison_tasks += 1 - self._cost_comparison_routed += outcome.cost_usd + self._cost_comparison_routed.add(outcome.cost_usd) for target, cost in zip(self.targets, costs, strict=True): assert cost is not None - self._cost_comparison_targets[target] += cost + self._cost_comparison_targets[target].add(cost) self._cost_comparison_routing.add(result.routing_cost_usd) def to_dict(self) -> dict[str, object]: """Return an independent JSON-compatible snapshot with explicit coverage.""" count = self._comparison_tasks cost_count = self._cost_comparison_tasks - routed_cost = self._cost_comparison_routed / cost_count if cost_count else None + routed_cost = ( + _as_float(self._cost_comparison_routed.fraction / cost_count) if cost_count else None + ) routing_cost = self._cost_comparison_routing.summary(cost_count)["mean"] estimated = self._estimated_cost.summary(self.processed) if self.processed != self.expected: @@ -231,10 +262,10 @@ def to_dict(self) -> dict[str, object]: "seconds": self._routing_seconds.summary(self.processed), "calls": self._routing_calls.summary(self.processed)["total"], "known_call_tasks": self._routing_calls.known, - "observed_calls": self._routing_calls.total, + "observed_calls": _as_float(self._routing_calls.total.fraction), "failed_calls": self._routing_failed_calls.summary(self.processed)["total"], "known_failed_call_tasks": self._routing_failed_calls.known, - "observed_failed_calls": self._routing_failed_calls.total, + "observed_failed_calls": _as_float(self._routing_failed_calls.total.fraction), "cost_usd": self._routing_cost.summary(self.processed), "usage": { name: value.summary(self.processed) @@ -244,22 +275,32 @@ def to_dict(self) -> dict[str, object]: "estimated_cost_usd": estimated, "comparison": { "tasks": count, - "routed_mean_reward": self._comparison_routed / count if count else None, + "routed_mean_reward": _as_float(self._comparison_routed.fraction / count) + if count + else None, "targets": { - target: total / count if count else None + target: _as_float(total.fraction / count) if count else None for target, total in self._comparison_targets.items() }, - "empirical_oracle_mean_reward": self._comparison_best / count if count else None, + "empirical_oracle_mean_reward": _as_float(self._comparison_best.fraction / count) + if count + else None, }, "cost_comparison": { "tasks": cost_count, "routed_mean_cost_usd": routed_cost, "targets": { - target: total / cost_count if cost_count else None + target: _as_float(total.fraction / cost_count) if cost_count else None for target, total in self._cost_comparison_targets.items() }, "routing_mean_cost_usd": routing_cost, - "routed_mean_cost_with_routing_usd": routed_cost + routing_cost + "routed_mean_cost_with_routing_usd": _as_float( + ( + self._cost_comparison_routed.fraction + + self._cost_comparison_routing.total.fraction + ) + / cost_count + ) if routed_cost is not None and routing_cost is not None else None, }, diff --git a/tests/sim/test_report.py b/tests/sim/test_report.py index 484b96c58..c2e63a0bd 100644 --- a/tests/sim/test_report.py +++ b/tests/sim/test_report.py @@ -5,6 +5,7 @@ import json from dataclasses import replace +from itertools import permutations import pytest @@ -52,6 +53,70 @@ def report(expected): return Report(expected, ("fast", "strong"), run_id="run", concurrency=2) +def test_repeat_means_remain_finite_when_intermediate_sum_overflows(): + value = outcome([1e308, 1e308], cost=1e308, duration=1e308) + assert value.reward == value.cost_usd == value.duration_seconds == 1e308 + assert outcome([1e308, 1e308, -1e308]).reward == pytest.approx(1e308 / 3) + + +@pytest.mark.parametrize("scale,residual", [(1e16, 1.0), (1e308, 1e-300), (1.0, 5e-324)]) +@pytest.mark.parametrize("order", list(permutations(range(3)))) +def test_signed_reward_totals_preserve_residual_independent_of_completion_order( + scale, residual, order +): + summary = report(3) + rewards = (scale, residual, -scale) + for index in order: + reward = rewards[index] + recorded = outcome([reward]) + summary.add(result(str(index), fast=recorded, strong=recorded)) + data = summary.to_dict() + assert data["recorded"]["reward"]["total"] == residual + assert data["recorded"]["reward"]["mean"] == residual / 3 + assert data["comparison"] == { + "tasks": 3, + "routed_mean_reward": residual / 3, + "targets": {"fast": residual / 3, "strong": residual / 3}, + "empirical_oracle_mean_reward": residual / 3, + } + json.dumps(data, allow_nan=False) + + +def test_unrepresentable_snapshot_is_explicit_and_does_not_corrupt_later_totals(): + summary = report(3) + for index in range(2): + recorded = outcome([1e308]) + summary.add(result(str(index), fast=recorded, strong=recorded)) + with pytest.raises(ValueError, match="aggregate exceeds finite"): + summary.to_dict() + recorded = outcome([-1e308]) + summary.add(result("last", fast=recorded, strong=recorded)) + data = summary.to_dict() + assert data["recorded"]["reward"]["total"] == 1e308 + assert data["comparison"]["routed_mean_reward"] == pytest.approx(1e308 / 3) + assert data["complete"] + json.dumps(data, allow_nan=False) + + +def test_comparison_means_do_not_require_representable_baseline_totals(): + summary = report(2) + for index in range(2): + summary.add(result(str(index), strong=outcome([1e308], cost=1e308))) + data = summary.to_dict() + assert data["comparison"]["targets"]["strong"] == 1e308 + assert data["comparison"]["empirical_oracle_mean_reward"] == 1e308 + assert data["cost_comparison"]["targets"]["strong"] == 1e308 + json.dumps(data, allow_nan=False) + + +def test_combined_cost_overflow_is_reported_at_snapshot_without_rejecting_result(): + summary = report(1) + summary.add(replace(result("one", fast=outcome([1], cost=1e308)), routing_cost_usd=1e308)) + assert summary.processed == 1 and summary.complete + with pytest.raises(ValueError, match="aggregate exceeds finite"): + summary.to_dict() + + def test_repeats_are_averaged_within_task_and_comparisons_share_cohort(): summary = report(3) summary.add(result("one", fast=outcome([0, 1, 1, 0]))) From 3f2e66ef955f464da41d31cecd51e408ed683d75 Mon Sep 17 00:00:00 2001 From: ayushag Date: Wed, 30 Sep 2026 06:56:31 -0700 Subject: [PATCH 36/48] docs(sim): clarify source setup and classifier input context Signed-off-by: ayushag --- .../llm_classifier_routing.md | 10 +++++++++- docs/simulation.md | 17 +++++++++++++++++ 2 files changed, 26 insertions(+), 1 deletion(-) diff --git a/docs/routing_algorithms/llm_classifier_routing.md b/docs/routing_algorithms/llm_classifier_routing.md index de35015ff..be82d708f 100644 --- a/docs/routing_algorithms/llm_classifier_routing.md +++ b/docs/routing_algorithms/llm_classifier_routing.md @@ -119,13 +119,21 @@ for the server merge behavior. |---|---|---| | `base_threshold` | required | Lowest `p_solve` that routes a supported task to `weak_target`. Must be between `0` and `1`. | | `threshold_step` | `0.0` | Amount added for each boundary step. Must be finite and non-negative, and `base_threshold + 2 * threshold_step` must not exceed `1`. | -| `recent_turn_window` | unset | When unset, the judge sees the opening user task and the latest user message when they differ. When set to `N`, it sees the opening user task and the last `N` conversation messages after that task. `0` keeps only the opening task. Client system and developer instructions are not shown to the judge. | +| `recent_turn_window` | unset | When unset, the judge sees the opening user task and the latest user message when they differ. When set to `N`, it keeps the opening task and the last `N` conversation messages after it, widening the window to keep tool call/result pairs together. `0` excludes later conversation. System/developer message handling is described below. | | `classify_trigger` | `every_request` | When the judge runs. `every_request` judges every request, tool continuations included. `user_turn` judges each new user message and holds that target across the tool calls between. `new_session` judges once and reuses that target for the session. | | `message_hash_fallback` | `false` | When session metadata is absent, keys affinity from the first user-message text. Requires `classify_trigger = "new_session"` or `"user_turn"`. | | `prompt` | packaged capability prompt | Replaces the classifier's system prompt. The packaged verdict schema and routing policy remain active. | | `response_format_type` | `json_schema` | Structured-output mode for capability and escalation judges. Use `json_object` for providers without JSON Schema support. | | `max_output_tokens` | `4096` | Maximum completion tokens available to the classifier verdict. Must be at least `1`. | +The judge replaces the request's separate `instructions` field with its own +prompt. HTTP decoders place client system and developer instructions in that +field, so those instructions are excluded. Native Python requests may instead +contain `system` or `developer` entries in normalized `messages`. A configured +`recent_turn_window`, including `0`, preserves those entries; leaving it unset +selects user task messages only. The simulator supplies recorded initial context +through `messages`. + ### Override the classifier prompt Set `prompt` on the route when the packaged capability rubric does not describe diff --git a/docs/simulation.md b/docs/simulation.md index 1a36cb383..605bc08ca 100644 --- a/docs/simulation.md +++ b/docs/simulation.md @@ -10,6 +10,17 @@ public `switchyard.runner` Python interface. Switchyard's native bindings load the existing TOML configuration and execute routing calls. No local server or Harbor installation is needed. +In an active Python 3.10+ virtual environment, install from the root of a source +checkout containing `switchyard/sim`: + +```bash +python -m pip install . +``` + +This builds the native extension using the Rust toolchain pinned in +[`rust-toolchain.toml`](../rust-toolchain.toml). See the +[installation requirements](../INSTALLATION.md#requirements) for CPU requirements. + ## Start with paired recordings Each target names a complete recorded model and agent configuration. Its name @@ -46,6 +57,12 @@ route ID passed to `evaluate` or `--route`. `fast` and `strong` are target keys by the dataset and `--run`; their `id` fields name the actual models. For a live classifier, use the [Task routing configuration](routing_algorithms/llm_classifier_routing.md). +For `llm_classifier`, the default judge input contains only the opening and latest +user messages. In the native Python path, set `recent_turn_window = 0` to keep +recorded system messages alongside the opening user task. See the +[classifier input rules](routing_algorithms/llm_classifier_routing.md#tuning-options) +when choosing which context the judge should use. + ```python import asyncio From 496bbd5ab0058052b7e621100b5ce0052ce61190 Mon Sep 17 00:00:00 2001 From: ayushag Date: Wed, 30 Sep 2026 07:31:03 -0700 Subject: [PATCH 37/48] test(python): cover composite child routing decisions Signed-off-by: ayushag --- tests/test_runner_bindings.py | 132 ++++++++++++++++++++++++++++++++++ 1 file changed, 132 insertions(+) diff --git a/tests/test_runner_bindings.py b/tests/test_runner_bindings.py index ff2f3035a..9eff34a4d 100644 --- a/tests/test_runner_bindings.py +++ b/tests/test_runner_bindings.py @@ -168,6 +168,48 @@ def deployment(url: str, *, escalation: bool = False) -> str: """ +def composite_deployment(url: str) -> str: + source = deployment(url).split("[routes.classifier]")[0] + for target in ("child_judge", "worker", "reviewer", "backup"): + source += f'\n[targets.{target}]\nid = "{target}/model"\nllm_client = "provider"\n' + return ( + source + + """ +[routes.composite] +id = "auto" +type = "composite" +[routes.composite.classifier] +target = "judge" +base_threshold = 0.5 +classify_trigger = "new_session" +[routes.composite.stage] +capable_target = "strong" +efficient_target = "weak" +confidence_threshold = 0.5 +[routes.composite.subagents] +type = "llm_classifier" +mode = "custom" +default_target = "efficient" +classify_trigger = "new_session" +prompt = "Select a target for this delegated task." +response_schema = ''' +{ + "type": "object", + "properties": {"target": {"type": "string", "enum": ["capable", "efficient"]}}, + "required": ["target"], + "additionalProperties": false +} +''' +policy = { type = "target_selector", selector = "/target" } +[routes.composite.subagents.models] +judge = ["child_judge"] +capable = ["reviewer", "backup"] +efficient = ["worker"] +any = ["worker", "reviewer", "backup"] +""" + ) + + def request(text: str = "easy task", *, model: str = "auto") -> dict[str, object]: return { "model": model, @@ -306,6 +348,96 @@ async def test_passthrough_decision_never_calls_answer_model(judge: JudgeStub) - assert judge.calls == [] +@pytest.mark.parametrize( + ("verdict", "selected", "fallbacks"), + [ + pytest.param('{"target":"capable"}', "reviewer", ["backup", "worker"], id="selected"), + pytest.param("not valid routing JSON", "worker", ["reviewer", "backup"], id="default"), + ], +) +async def test_composite_child_decisions_preserve_target_scope_and_fallback_order( + judge: JudgeStub, verdict: str, selected: str, fallbacks: list[str] +) -> None: + runner = Runner.from_toml(composite_deployment(judge.url)) + targets = runner.validate_decision_route("auto") + assert {(target.target, target.model) for target in targets} == { + (target, f"{target}/model") for target in ("strong", "weak", "worker", "reviewer", "backup") + } + assert judge.calls == [] + + parent_request = request("parent task") + parent = await runner.decide( + parent_request, + headers={ + **session(f"parent-{selected}"), + "x-switchyard-session-final": "true", + }, + ) + assert (parent.selected.target, parent.selected.model) == ("weak", "weak/model") + assert [(target.target, target.model) for target in parent.fallbacks] == [ + ("strong", "strong/model") + ] + assert parent.outcome.selected_model_ids == ["weak/model", "strong/model"] + assert [call["model"] for call in judge.calls] == ["judge/model"] + + judge.response_text = verdict + child_request = request("child task") + child = await runner.decide( + child_request, + headers={ + **session(f"child-{selected}"), + "x-switchyard-is-subagent": "true", + "x-switchyard-agent-id": "child", + "x-switchyard-session-final": "true", + }, + ) + assert (child.selected.target, child.selected.model) == (selected, f"{selected}/model") + assert [(target.target, target.model) for target in child.fallbacks] == [ + (target, f"{target}/model") for target in fallbacks + ] + assert child.outcome.selected_model_ids == [ + f"{target}/model" for target in (selected, *fallbacks) + ] + assert [call["model"] for call in judge.calls] == ["judge/model", "child_judge/model"] + assert parent_request == request("parent task") + assert child_request == request("child task") + for decision, original, judge_model in ( + (parent, parent_request, "judge/model"), + (child, child_request, "child_judge/model"), + ): + assert decision.outcome.response is None + assert decision.outcome.request["model"] == decision.selected.model + assert decision.outcome.request["messages"] == original["messages"] + (call,) = decision.calls + assert call.model == judge_model + assert call.is_success + assert call.usage == { + "input_tokens": 12, + "cached_input_tokens": 5, + "cache_creation_input_tokens": None, + "output_tokens": 8, + "total_tokens": 25, + "reasoning_tokens": 3, + } + assert decision.duration_seconds >= call.duration_seconds >= 0 + + +def test_composite_child_escalation_is_rejected_before_calls(judge: JudgeStub) -> None: + source = composite_deployment(judge.url).split("[routes.composite.subagents]")[0] + source += """ +[routes.composite.subagents] +type = "llm_classifier" +mode = "escalation" +classifier_target = "child_judge" +strong_target = "reviewer" +weak_target = "worker" +escalation = { confirmations = 2 } +""" + with pytest.raises(ValueError, match="subagents llm_classifier only supports mode custom"): + Runner.from_toml(source) + assert judge.calls == [] + + async def test_affinity_and_observations_are_isolated_across_sessions_and_runners( judge: JudgeStub, ) -> None: From 39dba2b7d04331ad8769c1c9346cac7ad9df71e8 Mon Sep 17 00:00:00 2001 From: ayushag Date: Wed, 30 Sep 2026 07:36:05 -0700 Subject: [PATCH 38/48] docs(sim): clarify provider configuration scoring boundary Signed-off-by: ayushag --- docs/simulation.md | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/docs/simulation.md b/docs/simulation.md index 605bc08ca..1210b1d76 100644 --- a/docs/simulation.md +++ b/docs/simulation.md @@ -492,8 +492,13 @@ Treat projected `Trial`, `Task`, and `Dataset` records as read-only. Their froze dataclasses still contain mutable nested mappings. If you change recorded input, create new trials and rebuild the dataset so pairing is validated again. -The task scorer also rejects answers and material conversation rewrites, because -recorded outcomes cannot score those changes. +The task scorer rejects routing-time answers and conversation rewrites visible in +`decision.outcome.request`, such as a selected target's `system_prompt`. It does +not reconstruct the final provider payload. Target `extra_body`, +`omit_body_fields`, and `reasoning_effort` are applied when a provider call is sent; +for example, `extra_body.instructions` can add instructions to an OpenAI Responses +request after routing. Verify that the completion target's settings match its +recorded run before reusing the outcome. Future trajectory replay can send successive normalized histories through the same public Runner interface and supply its own scorer. Task simulation makes no From 6242afe6b4d315baf0c9ef73b1adffefedc93f25 Mon Sep 17 00:00:00 2001 From: ayushag Date: Wed, 30 Sep 2026 07:51:48 -0700 Subject: [PATCH 39/48] fix(sim): retain import options in evaluation manifests Signed-off-by: ayushag --- docs/simulation.md | 5 +++-- switchyard/sim/__main__.py | 2 ++ tests/sim/test_integration.py | 9 ++++++++- 3 files changed, 13 insertions(+), 3 deletions(-) diff --git a/docs/simulation.md b/docs/simulation.md index 1210b1d76..c1b0723b5 100644 --- a/docs/simulation.md +++ b/docs/simulation.md @@ -384,8 +384,9 @@ Paths are resolved before this check, including symlink aliases and `..` compone This prevents output folders from changing Harbor's trial discovery on later imports. Existing output entries, including dangling symlinks, are rejected. The CLI writes a `manifest.json` with the -configuration hash, inputs, package version, and coverage, then flushes each -completed row to `results.jsonl`. It writes `report.json` after evaluation returns. +configuration hash, inputs, task namespace, import-error policy, package version, +and coverage, then flushes each completed row to `results.jsonl`. +It writes `report.json` after evaluation returns. The hash covers the exact TOML bytes loaded. Invalid routes and recorded-model mismatches fail before output creation, leaving the path available for a corrected run. The deployment source and credentials are not copied. diff --git a/switchyard/sim/__main__.py b/switchyard/sim/__main__.py index 64d41d355..040b38cf3 100644 --- a/switchyard/sim/__main__.py +++ b/switchyard/sim/__main__.py @@ -120,8 +120,10 @@ def main(argv: list[str] | None = None) -> int: "route": args.route, "runs": {target: str(path) for target, path in paths.items()}, "input_target": args.input_target, + "dataset": args.dataset, "reward_key": args.reward_key, "cost_source": args.cost_source, + "skip_invalid": args.skip_invalid, "model_aliases": aliases, "concurrency": args.concurrency, "timeout_seconds": args.timeout, diff --git a/tests/sim/test_integration.py b/tests/sim/test_integration.py index e8db5707d..d4b459310 100644 --- a/tests/sim/test_integration.py +++ b/tests/sim/test_integration.py @@ -185,8 +185,9 @@ async def test_native_completion_prompt_only_invalidates_its_selected_recording( assert prompt not in json.dumps(judge.calls) +@pytest.mark.parametrize("namespace", [None, "benchmark/v1"]) def test_cli_writes_flushed_rows_manifest_and_report_without_copying_config( - judge: JudgeStub, tmp_path: Path + judge: JudgeStub, tmp_path: Path, namespace: str | None ): write_trial(tmp_path / "recordings", "task", "weak/model", 1) config = tmp_path / "routes.toml" @@ -204,13 +205,18 @@ def test_cli_writes_flushed_rows_manifest_and_report_without_copying_config( "--output", str(output), ] + if namespace is not None: + args.extend(["--dataset", namespace]) assert main(args) == 0 assert judge.calls == [] report = json.loads((output / "report.json").read_text()) rows = [json.loads(line) for line in (output / "results.jsonl").read_text().splitlines()] manifest = json.loads((output / "manifest.json").read_text()) assert report["complete"] and len(rows) == 1 + assert rows[0]["task_id"] == (f"{namespace}/task" if namespace else "task") assert rows[0]["outcome"]["reward"] == 1 + assert manifest["dataset"] == namespace + assert manifest["skip_invalid"] is False assert len(manifest["config_sha256"]) == 64 assert "provider-secret" not in "".join(path.read_text() for path in output.iterdir()) original = (output / "report.json").read_text() @@ -278,6 +284,7 @@ async def verify(dataset, *args, **kwargs): ) manifest = json.loads((output / "manifest.json").read_text()) report = json.loads((output / "report.json").read_text()) + assert manifest["skip_invalid"] is True coverage = manifest["coverage"] assert coverage.items() <= report["coverage"].items() assert coverage["tasks_seen"] == 3 and coverage["tasks_included"] == 1 From f54655a82477c2131c9acb98b50a7526c6ae563a Mon Sep 17 00:00:00 2001 From: ayushag Date: Wed, 30 Sep 2026 08:31:00 -0700 Subject: [PATCH 40/48] test(python): cover concurrent decision request ownership Signed-off-by: ayushag --- docs/simulation.md | 6 +++++ tests/test_runner_bindings.py | 47 +++++++++++++++++++++++++++++++++++ 2 files changed, 53 insertions(+) diff --git a/docs/simulation.md b/docs/simulation.md index c1b0723b5..ff65e15e5 100644 --- a/docs/simulation.md +++ b/docs/simulation.md @@ -461,6 +461,12 @@ calls; durations include backend retries. `usage` is a mapping or `None`, and missing token fields remain unknown. `DecisionError.target` names the failing model, while `decision.selected.target` is the configured target key. +A decision starts when its coroutine runs. Keep the request and its nested data +unchanged until the await completes, and use separate request objects for +concurrent decisions. The returned `decision.outcome.request` is a mutable Python +mapping; use `copy.deepcopy` before editing it if you need to preserve the decision +for scoring. + `Runner.load` and `Runner.from_toml` use Switchyard's configuration parser. Configuration and request validation failures raise `ValueError`. Execution failures raise `DecisionError` with safe diagnostics and completed observations; diff --git a/tests/test_runner_bindings.py b/tests/test_runner_bindings.py index 9eff34a4d..f77677659 100644 --- a/tests/test_runner_bindings.py +++ b/tests/test_runner_bindings.py @@ -476,6 +476,53 @@ async def test_mapping_headers_are_copied_before_await( assert judge.request_headers[0]["x-request-id"] == "original" +async def test_in_flight_decisions_own_nested_requests_and_independent_outcomes( + judge: JudgeStub, +) -> None: + judge.release = Event() + runner = Runner.from_toml(deployment(judge.url)) + original = request() + tasks = [] + try: + for index, text in enumerate(("easy task", "TASK_REQUIRES_STRONG")): + original["messages"][0]["content"][0]["text"] = text + judge.started.clear() + tasks.append( + asyncio.create_task( + runner.decide( + original, + headers={ + **session(f"ownership-{index}"), + "x-switchyard-session-final": "true", + }, + ) + ) + ) + assert await asyncio.to_thread(judge.started.wait, 5) + + assert all(not task.done() for task in tasks) + original["messages"][0]["content"][0]["text"] = "caller changed the input again" + judge.release.set() + first, second = await asyncio.wait_for(asyncio.gather(*tasks), timeout=5) + + assert first.selected.target == "weak" + assert second.selected.target == "strong" + assert first.outcome.request["messages"] == request("easy task")["messages"] + assert second.outcome.request["messages"] == request("TASK_REQUIRES_STRONG")["messages"] + assert len(judge.calls) == 2 + + first.outcome.request["messages"][0]["content"][0]["text"] = "changed first outcome" + assert first.outcome.request["messages"] == request("changed first outcome")["messages"] + assert second.outcome.request["messages"] == request("TASK_REQUIRES_STRONG")["messages"] + assert original == request("caller changed the input again") + finally: + judge.release.set() + for task in tasks: + if not task.done(): + task.cancel() + await asyncio.gather(*tasks, return_exceptions=True) + + @pytest.mark.parametrize("forward_auth", [False, True]) async def test_caller_headers_are_request_scoped_and_respect_configured_auth( judge: JudgeStub, forward_auth: bool From cd23e0059ab49686d2ef76895c1e5745014ae443 Mon Sep 17 00:00:00 2001 From: ayushag Date: Wed, 30 Sep 2026 08:46:26 -0700 Subject: [PATCH 41/48] perf(sim): avoid copying rows before JSON encoding Signed-off-by: ayushag --- switchyard/sim/__main__.py | 12 ++++++--- tests/sim/test_integration.py | 51 +++++++++++++++++++++++++++++++++-- 2 files changed, 58 insertions(+), 5 deletions(-) diff --git a/switchyard/sim/__main__.py b/switchyard/sim/__main__.py index 040b38cf3..b9c8971c9 100644 --- a/switchyard/sim/__main__.py +++ b/switchyard/sim/__main__.py @@ -10,16 +10,22 @@ import hashlib import json import sys -from dataclasses import asdict +from dataclasses import fields from pathlib import Path from switchyard import __version__ from switchyard.runner import Runner -from . import Dataset, Result, evaluate, load_harbor +from . import Dataset, Outcome, Result, evaluate, load_harbor from .evaluate import _aliases, _validate_options, _validate_route +def _json_record(value: object) -> dict[str, object]: + if isinstance(value, (Result, Outcome)): + return {field.name: getattr(value, field.name) for field in fields(value)} + raise TypeError(f"Object of type {type(value).__name__} is not JSON serializable") + + def _resolve_path(path: Path) -> Path: try: return path.resolve() @@ -135,7 +141,7 @@ def main(argv: list[str] | None = None) -> int: with (output / "results.jsonl").open("x", encoding="utf-8") as stream: def save(result: Result) -> None: - stream.write(json.dumps(asdict(result), allow_nan=False) + "\n") + stream.write(json.dumps(result, default=_json_record, allow_nan=False) + "\n") stream.flush() report = asyncio.run( diff --git a/tests/sim/test_integration.py b/tests/sim/test_integration.py index d4b459310..fb4baa30c 100644 --- a/tests/sim/test_integration.py +++ b/tests/sim/test_integration.py @@ -6,13 +6,14 @@ import hashlib import json import weakref +from dataclasses import asdict from pathlib import Path import pytest from switchyard.runner import Runner -from switchyard.sim import Dataset, evaluate, load_harbor, score -from switchyard.sim.__main__ import main +from switchyard.sim import Dataset, Outcome, Result, evaluate, load_harbor, score +from switchyard.sim.__main__ import _json_record, main from tests.test_runner_bindings import JudgeStub, deployment from tests.test_runner_bindings import judge as judge @@ -185,6 +186,52 @@ async def test_native_completion_prompt_only_invalidates_its_selected_recording( assert prompt not in json.dumps(judge.calls) +@pytest.mark.parametrize("failed", [False, True], ids=["success", "failure"]) +def test_cli_json_rows_preserve_exact_serialization(failed): + selected = Outcome( + trials=2, + reward=-0.0, + cost_usd=None, + duration_seconds=5e-324, + reward_trials=2, + cost_trials=1, + duration_trials=2, + ) + ungraded = Outcome( + trials=1, + reward=None, + cost_usd=1e308, + duration_seconds=None, + reward_trials=0, + cost_trials=1, + duration_trials=0, + ) + result = Result( + task_id="benchmark/\u03c0", + baselines={"weak": selected, "strong": ungraded}, + target=None if failed else "weak", + model=None if failed else "weak/model", + outcome=None if failed else selected, + decision_id=None if failed else "decision-1", + algorithm=None if failed else "llm-classifier", + evidence=None + if failed + else {"verdict": {"label": "\u590d\u6742", "values": [True, None, -0.0, 5e-324, 1e308]}}, + fallbacks=() if failed else ("strong",), + routing_seconds=0.125, + routing_calls=1, + routing_failed_calls=int(failed), + routing_usage={"input_tokens": None, "output_tokens": 0}, + error="routing failed: DecisionError" if failed else None, + routing_error_kind="upstream_http" if failed else None, + routing_error_status=503 if failed else None, + routing_error_target="judge/model" if failed else None, + ) + assert json.dumps(result, default=_json_record, allow_nan=False) == json.dumps( + asdict(result), allow_nan=False + ) + + @pytest.mark.parametrize("namespace", [None, "benchmark/v1"]) def test_cli_writes_flushed_rows_manifest_and_report_without_copying_config( judge: JudgeStub, tmp_path: Path, namespace: str | None From 1880a60eae419d779378140aa83f91b56b2d97d0 Mon Sep 17 00:00:00 2001 From: ayushag Date: Wed, 30 Sep 2026 09:05:24 -0700 Subject: [PATCH 42/48] docs(sim): document architecture and extension boundaries Signed-off-by: ayushag --- README.md | 1 + docs/simulation-architecture.md | 154 ++++++++++++++++++++++++++++++++ docs/simulation.md | 7 +- mkdocs.yml | 1 + 4 files changed, 161 insertions(+), 2 deletions(-) create mode 100644 docs/simulation-architecture.md diff --git a/README.md b/README.md index 9bd0cf889..8b431873a 100644 --- a/README.md +++ b/README.md @@ -66,6 +66,7 @@ The [routing overview](docs/routing_algorithms/overview.md) retains the full cat Results depend on the benchmark, model pool, serving stack, and routing configuration. Use the Python [task routing evaluator](docs/simulation.md) to compare routing decisions against ATIF trajectories, custom recordings, or Harbor Claude and Codex runs. +See its [architecture](docs/simulation-architecture.md) for the ATIF converter boundary, routing and scoring flow, and future replay extension. For latency and routing overhead testing, see [Soak Testing](docs/operations/soak_test.md). ### Further reading diff --git a/docs/simulation-architecture.md b/docs/simulation-architecture.md new file mode 100644 index 000000000..940ed30bf --- /dev/null +++ b/docs/simulation-architecture.md @@ -0,0 +1,154 @@ +# Switchyard-Sim architecture + +`switchyard.sim` helps users compare Switchyard routing policies on their own +recorded workflows. It routes the input available before a task starts, then +scores the selected target using completed runs of that task. The simulator is +Python; routing executes through Switchyard's public Python interface and existing +native runtime. Both run in the same process. + +See the [evaluation guide](simulation.md) for installation, runnable examples, +import rules, and metric definitions. This page describes the implemented task +evaluator and the boundary available for future trajectory replay. + +## Data flow + +Importers converge on trials before task pairing. Custom converters retain a full +ATIF document in `Trajectory`; Harbor projects directly from its artifact files. + +```mermaid +flowchart TD + custom["Custom recording"] --> converter["User converter"] + atif["ATIF document"] --> trajectory["Trajectory"] + converter --> trajectory + trajectory -->|"to_trial + outcome metadata"| run["Run / Trial"] + harbor["Harbor artifacts"] --> loader["load_harbor"] + loader --> run + run --> dataset["Dataset: paired tasks"] +``` + +`evaluate` schedules tasks through the following flow. The two dataset edges +separate what the router sees from the evidence used to score its choice. + +```mermaid +flowchart TD + dataset["Dataset"] -->|"initial messages"| api["Python Runner.decide"] + api <-->|"request / decision"| native["Native Switchyard runtime"] + config["TOML configuration"] --> native + native <-->|"optional routing calls"| provider["Judge provider"] + dataset -->|"recorded outcomes"| scoring["Task scoring"] + api -->|"decision or failure"| scoring + scoring --> result["Result"] + result --> report["Report"] + result --> sink["on_result / CLI JSONL"] +``` + +Recorded rewards and completed agent steps stay outside the routing request. +Fixed and random routes need no provider calls. A classifier may call its judge; +the task evaluator does not run the selected completion model, agent environment, +or verifier. Its score is an estimate from comparable recorded runs. + +## Components and ownership + +| Component | Owns | +| --- | --- | +| [`trajectory.py`](../switchyard/sim/trajectory.py) | An independent copy of a full ATIF document; extraction of initial text messages; projection into a `Trial` with explicit identity and outcome metadata. | +| [`harbor.py`](../switchyard/sim/harbor.py) | Harbor artifact discovery, validation, verifier/accounting metadata, and import issues. It uses the shared initial-input projection directly, avoiding a second copy of the full trajectory. | +| [`models.py`](../switchyard/sim/models.py), [`dataset.py`](../switchyard/sim/dataset.py) | `Trial`, `Run`, `Task`, `Outcome`, and `Result`; task pairing, comparable-input checks, repeat aggregation, and selection of the routing input. | +| [`evaluate.py`](../switchyard/sim/evaluate.py) | Route compatibility checks, bounded asynchronous scheduling, deadlines, isolated task sessions, scoring, and progressive result delivery. Public `score(task, decision)` exposes the same scoring rules without making calls. | +| [`switchyard.runner`](../switchyard/runner.py), [Python bindings](../switchyard_rust/runner.py) | The public normalized decision API and asynchronous bridge into the existing runtime. Simulation uses this interface rather than duplicating routing configuration or algorithms. | +| [Native runner](../crates/switchyard-runner/src/runner.rs), [routing algorithms](../crates/libsy/src/lib.rs) | Configuration, routing state, algorithms, and routing-time calls through Switchyard's clients and format translation. | +| [`report.py`](../switchyard/sim/report.py), [CLI](../switchyard/sim/__main__.py) | Incremental metrics and coverage; the CLI adds a manifest, flushed result rows, and a final report. | + +Artifact conversion, pairing, and report aggregation use the Python standard +library. The native runtime is needed when making routing decisions. Harbor is +an input format, not an installed dependency. + +## From recordings to one scored task + +1. **Import evidence.** A trial contains the original initial messages, target, + task and trial identities, recorded model, and available reward, cost, duration, + and usage. Unknown measurements stay unknown. Imports fail by default; + tolerant importers retain rejected records in `Run.issues`. +2. **Build a comparable cohort.** `Dataset.from_runs` validates task identity and + input compatibility. Pairing requires matching task sets by default; explicit + intersection records exclusions. Outcomes average repeats within each + task/target. Tasks have equal weight in the report. `input_target` chooses which + run supplies the routing input; repeats use the first string-sorted trial ID. +3. **Validate and route.** `evaluate` checks the route's targets and known model + identities before scheduling work. Each task gets a copied initial request, + a distinct final-session ID, and a deadline covering the complete decision. + `Runner.decide(..., allow_response=False)` returns a selection, fallback + choices, decision evidence, and completed routing-call observations. +4. **Score the selection.** The scorer checks that the decision can be matched to + recorded outcomes, including model identity and visible conversation changes. + It attaches the selected target's outcome and all fixed-target baselines to a + `Result`. Routing failures and unsupported selections become error results in + `evaluate`; standalone `score` rejects unsupported decisions with `ValueError`. +5. **Report progressively.** The coordinator adds each result to `Report`, then + invokes the synchronous `on_result` callback. Routing telemetry and costs stay + separate from recorded task estimates. Matched comparisons use their stated + observed cohorts; missing reward or cost is never treated as zero. + +`evaluate` and `score` share internal scoring routines. Applications that already +own scheduling can use `Runner.decide`, `score`, and `Report.add` directly. + +## Extension boundaries + +### Custom formats use ATIF + +A converter is a plain function returning `Trajectory`. It translates the source +format into ATIF; `Trajectory.to_trial` extracts task input using the same rules as +Harbor. The caller supplies task identity, outcomes, and recorded model metadata +explicitly. The [converter example](simulation.md#use-atif-or-custom-recordings) +shows the complete path into `Run` and `Dataset`. + +`Trajectory` preserves all steps and extension fields for other consumers. +`to_trial` retains only initial input and supplied metadata. Validation covers the +ATIF version, steps container, and fields read for input projection; it is not a +complete ATIF schema validator. Initial copied continuation context requires the +original task input to replace it. Unmarked leaked context remains the converter +author's responsibility. + +### Routing, pricing, and result storage + +Configure routing policies through Switchyard's existing TOML and `Runner`. +Adding a format converter does not require a new routing algorithm. Supplying a +`price_call` function lets a library caller estimate routing-call cost from its +own rates and observed usage; the simulator has no built-in pricing catalog. +Use `on_result` for a dashboard or an application-owned result store. + +### Future trajectory replay + +Replay is not implemented. Full ATIF documents and the public decision API provide +the reusable boundaries: a replay coordinator could construct successive +histories, call the same `Runner`, and pass execution results to a replay-specific +scorer. It would also need to own environment state, tool execution, session +ordering, verification, and recovery. + +Recorded task outcomes cannot establish what happens after a mid-trajectory model +switch. That requires new execution or a separately validated replay method. +Keeping replay outside the current task scorer avoids assigning recorded rewards +to behavior that never occurred. + +## Operational contracts + +- **Memory:** Harbor decodes one full trajectory at a time. A dataset retains the + included trials and initial inputs across targets and repeats. `Report` retains + aggregates and seen task IDs, not result rows. Concurrency bounds active + decisions, not dataset memory. Release unused `Run` objects after pairing. +- **State and concurrency:** use a fresh `Runner` for independent experiments. + Task sessions are distinct and final. Seeded random routing requires serial + evaluation when task-to-draw assignment must be reproducible. One decision may + make several routing-time provider calls. +- **Failure handling:** callback errors and cancellation stop scheduling and drain + pending local work. Already received provider requests cannot be revoked. Keep + callbacks short; applications that queue writes must handle their completion. +- **Artifacts:** the CLI writes into a new output directory outside the inputs. + Its manifest records configuration identity and import options; JSONL rows are + flushed as tasks finish. Partial output remains inspectable after interruption. + There is no automatic resume. See [saved artifacts](simulation.md#save-progressive-results). +- **Validity:** the caller must establish equivalent task, agent, model, and + provider settings across recordings. The scorer checks the normalized decision + request, not every transport-time provider override. See + [library boundaries](simulation.md#library-boundaries) before interpreting + comparisons as workflow improvements. diff --git a/docs/simulation.md b/docs/simulation.md index ff65e15e5..7c687c801 100644 --- a/docs/simulation.md +++ b/docs/simulation.md @@ -10,6 +10,9 @@ public `switchyard.runner` Python interface. Switchyard's native bindings load the existing TOML configuration and execute routing calls. No local server or Harbor installation is needed. +See the [simulator architecture](simulation-architecture.md) for the data flow, +component ownership, and extension boundaries for custom formats and future replay. + In an active Python 3.10+ virtual environment, install from the root of a source checkout containing `switchyard/sim`: @@ -484,8 +487,8 @@ explicit `allow_response=True`; task simulation rejects them. `Trajectory` preserves ATIF documents. `Trial` and `Run` represent projected task evidence. `Dataset.from_runs` validates and pairs it. `score(task, decision)` is a pure scorer for a saved native decision. -`evaluate` owns bounded scheduling and calls that scorer. `Report.add` accumulates -results without retaining full trajectories or result rows. Harbor import decodes +`evaluate` owns bounded scheduling and shares the same scoring routines. +`Report.add` accumulates results without retaining full trajectories or result rows. Harbor import decodes one complete trajectory JSON file at a time, then keeps each accepted trial's initial input and metadata. A dataset retains those trials for every included task, across targets and repeats. `input_target` chooses the routing input without discarding the diff --git a/mkdocs.yml b/mkdocs.yml index 69029eb60..f6f3c5186 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -41,6 +41,7 @@ nav: - Advisor-Gate Routing: routing_algorithms/advisor_gate_routing.md - Operations: - Task routing evaluation: simulation.md + - Simulator architecture: simulation-architecture.md - Context-Window Handling: operations/context_window.md - Soak Testing: operations/soak_test.md - Reference: From 55561784de1616f8e1e0bbb059bf82cee5d8320b Mon Sep 17 00:00:00 2001 From: ayushag Date: Wed, 30 Sep 2026 09:26:37 -0700 Subject: [PATCH 43/48] fix(python): redact invalid Unicode header values Signed-off-by: ayushag --- crates/switchyard-py/src/runner_bindings.rs | 11 +++++++++-- docs/simulation.md | 3 ++- tests/test_runner_bindings.py | 18 ++++++++++++++++++ 3 files changed, 29 insertions(+), 3 deletions(-) diff --git a/crates/switchyard-py/src/runner_bindings.rs b/crates/switchyard-py/src/runner_bindings.rs index 75ef234bc..422888bd0 100644 --- a/crates/switchyard-py/src/runner_bindings.rs +++ b/crates/switchyard-py/src/runner_bindings.rs @@ -11,7 +11,7 @@ use std::sync::{Arc, Condvar, Mutex}; use std::task::{Context, Poll}; use pyo3::create_exception; -use pyo3::exceptions::{PyRuntimeError, PyValueError}; +use pyo3::exceptions::{PyRuntimeError, PyUnicodeError, PyValueError}; use pyo3::prelude::*; use pyo3::types::{PyDict, PyMapping}; use pyo3_async_runtimes::TaskLocals; @@ -300,7 +300,14 @@ impl PyRunner { .map(|mapping| { let owned = PyDict::new(py); owned.update(mapping)?; - owned.extract::>() + owned.extract::>().map_err(|error| { + if error.is_instance_of::(py) { + // Unicode errors retain the complete input in their args. + PyValueError::new_err("header names and values must be valid UTF-8") + } else { + error + } + }) }) .transpose()?; // Start routing only when the coroutine runs, so cancellation before its diff --git a/docs/simulation.md b/docs/simulation.md index 7c687c801..fc3d240d4 100644 --- a/docs/simulation.md +++ b/docs/simulation.md @@ -471,7 +471,8 @@ mapping; use `copy.deepcopy` before editing it if you need to preserve the decis for scoring. `Runner.load` and `Runner.from_toml` use Switchyard's configuration parser. -Configuration and request validation failures raise `ValueError`. Execution +Configuration and request validation failures raise `ValueError`; incompatible +Python argument types can also raise `TypeError`. Execution failures raise `DecisionError` with safe diagnostics and completed observations; an application deadline can instead raise `asyncio.TimeoutError`. Cancellation waits for local routing and its Python bridge to stop. diff --git a/tests/test_runner_bindings.py b/tests/test_runner_bindings.py index f77677659..ce11e710d 100644 --- a/tests/test_runner_bindings.py +++ b/tests/test_runner_bindings.py @@ -772,6 +772,24 @@ async def test_later_call_failure_preserves_earlier_call_usage(judge: JudgeStub) assert "private prompt" not in str(error) +@pytest.mark.parametrize("component", ["name", "value"]) +def test_invalid_unicode_headers_do_not_expose_caller_data( + judge: JudgeStub, component: str +) -> None: + runner = Runner.from_toml(deployment(judge.url)) + secret = "private-header-value" + invalid = secret + "\udcff" + headers = {invalid: "value"} if component == "name" else {"authorization": invalid} + with pytest.raises(ValueError) as caught: + runner.decide(request(), headers=headers) + error = caught.value + for diagnostic in (str(error), repr(error), repr(error.args)): + assert secret not in diagnostic + assert error.__cause__ is None and error.__context__ is None + assert "valid UTF-8" in str(error) + assert judge.calls == [] + + async def test_bad_request_and_configuration_fail_before_calls( judge: JudgeStub, tmp_path: Path ) -> None: From 41371ba1cd55c09b52beb759befd38c1d1135862 Mon Sep 17 00:00:00 2001 From: ayushag Date: Wed, 30 Sep 2026 09:45:03 -0700 Subject: [PATCH 44/48] docs(sim): explain unscored tasks and routing failures Signed-off-by: ayushag --- docs/simulation.md | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/docs/simulation.md b/docs/simulation.md index fc3d240d4..7e13b430d 100644 --- a/docs/simulation.md +++ b/docs/simulation.md @@ -272,6 +272,20 @@ and means are `null`; `observed_total` retains the known portion. `complete` means every expected task has a scored reward and no evaluation error. Inspect each cost and usage field's coverage separately. +For tasks processed by `evaluate`, the counters distinguish these cases: + +| Result | `routed` | `scored` | `errors` | +| --- | --- | --- | --- | +| Valid selection with a known outcome reward, including zero | 1 | 1 | 0 | +| Valid selection with an unknown outcome reward | 1 | 0 | 0 | +| Target selected, but scoring rejects the decision, such as a rewritten task input | 1 | 0 | 1 | +| Routing fails before returning a selection | 0 | 0 | 1 | + +The last three cases also increment `unscored`. A zero reward is an observed +outcome; a missing reward is not a failed routing decision. A task with unknown +reward can still enter the cost comparison when the selected outcome and every +fixed target have known costs. + Signed rewards are supported. Totals preserve small values when larger positive and negative rewards cancel, regardless of task completion order. Published aggregates remain floating-point numbers. `Report.to_dict()` raises `ValueError` @@ -282,6 +296,9 @@ Native routing failures retain `routing_error_kind`, `routing_error_status`, and when available; the target identifies the failing model, such as the classifier. These fields distinguish failures such as HTTP 401 and 503 without storing provider response bodies. Other exception details remain suppressed. +A whole-decision timeout returns no completed-call observations, so its call +counts and routing cost remain unknown. The provider may already have received +or completed work. Reward comparisons use one common cohort where the routed outcome and every fixed-target reward are observed. The empirical recorded-outcome oracle selects From c5352fdf26d0d535eb3590266101ec50e4f59e86 Mon Sep 17 00:00:00 2001 From: ayushag Date: Wed, 30 Sep 2026 09:50:13 -0700 Subject: [PATCH 45/48] fix(python): redact invalid Unicode configuration sources Signed-off-by: ayushag --- crates/switchyard-py/src/runner_bindings.rs | 15 ++++++++-- tests/test_runner_bindings.py | 32 +++++++++++++++++++++ 2 files changed, 45 insertions(+), 2 deletions(-) diff --git a/crates/switchyard-py/src/runner_bindings.rs b/crates/switchyard-py/src/runner_bindings.rs index 422888bd0..6f606f05a 100644 --- a/crates/switchyard-py/src/runner_bindings.rs +++ b/crates/switchyard-py/src/runner_bindings.rs @@ -13,7 +13,7 @@ use std::task::{Context, Poll}; use pyo3::create_exception; use pyo3::exceptions::{PyRuntimeError, PyUnicodeError, PyValueError}; use pyo3::prelude::*; -use pyo3::types::{PyDict, PyMapping}; +use pyo3::types::{PyDict, PyMapping, PyString}; use pyo3_async_runtimes::TaskLocals; use pyo3_async_runtimes::generic::{ContextExt, Runtime}; use switchyard_llm_client::{LlmCallObservation, RunObservation, RunObserver}; @@ -257,7 +257,18 @@ impl PyRunner { /// Load and validate a native deployment TOML document. #[staticmethod] - fn from_toml(py: Python<'_>, source: String) -> PyResult { + fn from_toml(py: Python<'_>, source: &Bound<'_, PyString>) -> PyResult { + let source = source + .to_cow() + .map_err(|error| { + if error.is_instance_of::(py) { + // Unicode errors retain the complete configuration in their args. + PyValueError::new_err("deployment TOML must be valid UTF-8") + } else { + error + } + })? + .into_owned(); py.detach(move || { let _guard = pyo3_async_runtimes::tokio::get_runtime().enter(); Runner::from_toml(&source) diff --git a/tests/test_runner_bindings.py b/tests/test_runner_bindings.py index ce11e710d..7ae3fd212 100644 --- a/tests/test_runner_bindings.py +++ b/tests/test_runner_bindings.py @@ -790,6 +790,38 @@ def test_invalid_unicode_headers_do_not_expose_caller_data( assert judge.calls == [] +@pytest.mark.parametrize("string_subclass", [False, True]) +def test_invalid_unicode_configuration_does_not_expose_source( + judge: JudgeStub, string_subclass: bool +) -> None: + class Source(str): + pass + + secret = "private-configuration-value" + source = deployment(judge.url) + f"\n# {secret}\udcff" + with pytest.raises(ValueError) as caught: + Runner.from_toml(Source(source) if string_subclass else source) + error = caught.value + for diagnostic in (str(error), repr(error), repr(error.args)): + assert secret not in diagnostic + assert error.__cause__ is None and error.__context__ is None + assert "valid UTF-8" in str(error) + assert judge.calls == [] + + +def test_configuration_string_conversion_preserves_supported_types(judge: JudgeStub) -> None: + class Source(str): + pass + + source = deployment(judge.url) + "\n# café 路線" + runner = Runner.from_toml(Source(source)) + assert runner.validate_decision_route("fixed")[0].target == "weak" + for invalid in (None, 42, {}, source.encode("utf-8")): + with pytest.raises(TypeError): + Runner.from_toml(invalid) + assert judge.calls == [] + + async def test_bad_request_and_configuration_fail_before_calls( judge: JudgeStub, tmp_path: Path ) -> None: From fe1f2abc9ada115f8e2190fd03c552d1a91630d7 Mon Sep 17 00:00:00 2001 From: ayushag Date: Wed, 30 Sep 2026 09:59:36 -0700 Subject: [PATCH 46/48] fix(sim): retain issues for excessively nested JSON Signed-off-by: ayushag --- switchyard/sim/harbor.py | 2 +- tests/sim/test_harbor.py | 41 ++++++++++++++++++++++++++++++++++++++++ 2 files changed, 42 insertions(+), 1 deletion(-) diff --git a/switchyard/sim/harbor.py b/switchyard/sim/harbor.py index 88abac6f6..1e10e00f6 100644 --- a/switchyard/sim/harbor.py +++ b/switchyard/sim/harbor.py @@ -146,7 +146,7 @@ def _read_object(path: Path) -> dict[str, Any]: try: with path.open(encoding="utf-8") as stream: value = json.load(stream) - except (json.JSONDecodeError, UnicodeError): + except (json.JSONDecodeError, UnicodeError, RecursionError): raise ValueError(f"invalid JSON in {path.name}") from None return _object(value, path.name) diff --git a/tests/sim/test_harbor.py b/tests/sim/test_harbor.py index 8c5002846..e9b8e2e90 100644 --- a/tests/sim/test_harbor.py +++ b/tests/sim/test_harbor.py @@ -358,6 +358,47 @@ def test_malformed_trajectory_records_safe_error_and_task_identity(tmp_path: Pat assert "PRIVATE_TRACE" not in run.issues[0].message +@pytest.mark.parametrize("artifact", ["result.json", "agent/trajectory.json"]) +def test_excessive_json_nesting_records_issue_without_losing_valid_trials( + tmp_path: Path, artifact: str +) -> None: + invalid = write_trial(tmp_path, "invalid") + write_trial(tmp_path, "valid", task="benchmark/task-b") + path = invalid / artifact + nested = "[" * 10_000 + '"PRIVATE_TRACE"' + "]" * 10_000 + path.write_text(path.read_text()[:-1] + ', "unused":' + nested + "}") + + with pytest.raises(ValueError, match=f"invalid JSON in {path.name}"): + load_harbor(tmp_path, target="baseline", dataset="suite") + run = load_harbor(tmp_path, target="baseline", dataset="suite", on_error="record") + assert [trial.task_id for trial in run.trials] == ["suite/benchmark/task-b"] + assert len(run.issues) == 1 + issue = run.issues[0] + assert issue.message == f"invalid JSON in {path.name}" + assert issue.source == str(invalid / "result.json") + if artifact == "result.json": + assert issue.task_id is None + with pytest.raises(ValueError, match="no task identity"): + Dataset.from_runs({"baseline": run}, input_target="baseline", intersection=True) + else: + assert issue.task_id == "suite/benchmark/task-a" + dataset = Dataset.from_runs({"baseline": run}, input_target="baseline", intersection=True) + assert dataset.coverage["excluded_task_ids"] == [issue.task_id] + + +@pytest.mark.parametrize("layout", ["job", "download"]) +def test_excessive_json_nesting_in_summary_keeps_discoverable_trials( + tmp_path: Path, layout: str +) -> None: + job = tmp_path / "jobs" / "run" if layout == "download" else tmp_path + write_trial(job) + nested = "[" * 10_000 + "0" + "]" * 10_000 + (job / "result.json").write_text('{"unused":' + nested + "}") + run = load_harbor(tmp_path, target="baseline", on_error="record") + assert [trial.task_id for trial in run.trials] == ["benchmark/task-a"] + assert run.issues == () + + def test_trial_without_result_is_an_issue_not_an_invisible_exclusion(tmp_path: Path) -> None: trial = write_trial(tmp_path) (trial / "result.json").unlink() From e20b2d423e908239eb3229aba9b775e3f1c31b07 Mon Sep 17 00:00:00 2001 From: ayushag Date: Wed, 30 Sep 2026 10:08:58 -0700 Subject: [PATCH 47/48] test(sim): respect interpreter JSON nesting limits Signed-off-by: ayushag --- tests/sim/test_harbor.py | 51 +++++++++++++++++++++++++++++++++------- 1 file changed, 43 insertions(+), 8 deletions(-) diff --git a/tests/sim/test_harbor.py b/tests/sim/test_harbor.py index e9b8e2e90..41b7543c9 100644 --- a/tests/sim/test_harbor.py +++ b/tests/sim/test_harbor.py @@ -84,6 +84,17 @@ def update(path: Path, **values: object) -> None: path.write_text(json.dumps(row)) +def inject_json_recursion_error(path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + original_load = json.load + + def load(stream, *args, **kwargs): + if Path(stream.name) == path: + raise RecursionError("PRIVATE_TRACE") + return original_load(stream, *args, **kwargs) + + monkeypatch.setattr(json, "load", load) + + @pytest.mark.parametrize( "version,codex", [("ATIF-v1.7", False), ("ATIF-v1.7", True), ("ATIF-v1.5", True)] ) @@ -359,14 +370,13 @@ def test_malformed_trajectory_records_safe_error_and_task_identity(tmp_path: Pat @pytest.mark.parametrize("artifact", ["result.json", "agent/trajectory.json"]) -def test_excessive_json_nesting_records_issue_without_losing_valid_trials( - tmp_path: Path, artifact: str +def test_json_decoder_recursion_error_records_issue_without_losing_valid_trials( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch, artifact: str ) -> None: invalid = write_trial(tmp_path, "invalid") write_trial(tmp_path, "valid", task="benchmark/task-b") path = invalid / artifact - nested = "[" * 10_000 + '"PRIVATE_TRACE"' + "]" * 10_000 - path.write_text(path.read_text()[:-1] + ', "unused":' + nested + "}") + inject_json_recursion_error(path, monkeypatch) with pytest.raises(ValueError, match=f"invalid JSON in {path.name}"): load_harbor(tmp_path, target="baseline", dataset="suite") @@ -387,18 +397,43 @@ def test_excessive_json_nesting_records_issue_without_losing_valid_trials( @pytest.mark.parametrize("layout", ["job", "download"]) -def test_excessive_json_nesting_in_summary_keeps_discoverable_trials( - tmp_path: Path, layout: str +def test_json_decoder_recursion_error_in_summary_keeps_discoverable_trials( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch, layout: str ) -> None: job = tmp_path / "jobs" / "run" if layout == "download" else tmp_path write_trial(job) - nested = "[" * 10_000 + "0" + "]" * 10_000 - (job / "result.json").write_text('{"unused":' + nested + "}") + path = job / "result.json" + path.write_text('{"n_total_trials": 1}') + inject_json_recursion_error(path, monkeypatch) run = load_harbor(tmp_path, target="baseline", on_error="record") assert [trial.task_id for trial in run.trials] == ["benchmark/task-a"] assert run.issues == () +def test_json_nesting_follows_the_interpreters_decoder_limit(tmp_path: Path) -> None: + trial = write_trial(tmp_path) + path = trial / "result.json" + nested = "[" * 10_000 + "0" + "]" * 10_000 + payload = path.read_text()[:-1] + ', "unused":' + nested + "}" + path.write_text(payload) + try: + json.loads(payload) + except RecursionError: + supported = False + else: + supported = True + + run = load_harbor(tmp_path, target="baseline", on_error="record") + if supported: + assert [trial.task_id for trial in run.trials] == ["benchmark/task-a"] + assert run.issues == () + else: + assert run.trials == () + assert len(run.issues) == 1 + assert run.issues[0].message == "invalid JSON in result.json" + assert run.issues[0].task_id is None + + def test_trial_without_result_is_an_issue_not_an_invisible_exclusion(tmp_path: Path) -> None: trial = write_trial(tmp_path) (trial / "result.json").unlink() From 8201d1a9ee6b497a75a5913b861640a3418402ee Mon Sep 17 00:00:00 2001 From: ayushag Date: Wed, 30 Sep 2026 10:47:48 -0700 Subject: [PATCH 48/48] fix(python): bound recursive native conversion Signed-off-by: ayushag --- crates/switchyard-py/src/lib.rs | 1 + crates/switchyard-py/src/py_serde.rs | 6 +- crates/switchyard-py/src/serde_depth.rs | 325 ++++++++++++++++++++++++ docs/simulation.md | 4 +- tests/test_runner_bindings.py | 107 ++++++++ 5 files changed, 440 insertions(+), 3 deletions(-) create mode 100644 crates/switchyard-py/src/serde_depth.rs diff --git a/crates/switchyard-py/src/lib.rs b/crates/switchyard-py/src/lib.rs index 29fa2e883..93c45cc8c 100644 --- a/crates/switchyard-py/src/lib.rs +++ b/crates/switchyard-py/src/lib.rs @@ -7,6 +7,7 @@ mod errors; mod libsy_bindings; mod py_serde; mod runner_bindings; +mod serde_depth; mod server_bindings; #[pymodule] diff --git a/crates/switchyard-py/src/py_serde.rs b/crates/switchyard-py/src/py_serde.rs index 8a79e16bd..03e293b85 100644 --- a/crates/switchyard-py/src/py_serde.rs +++ b/crates/switchyard-py/src/py_serde.rs @@ -6,13 +6,15 @@ use pyo3::exceptions::PyValueError; use pyo3::prelude::*; use pyo3::types::PyDict; -use pythonize::{depythonize, pythonize}; +use pythonize::{Depythonizer, pythonize}; use serde::{Serialize, de::DeserializeOwned}; +use crate::serde_depth; + /// Converts a Python mapping-like object into a Serde-owned Rust value. pub(crate) fn from_python(value: &Bound<'_, PyAny>) -> PyResult { let normalized = jsonable_python(value)?; - depythonize(normalized.bind(value.py())) + serde_depth::deserialize(&mut Depythonizer::from_object(normalized.bind(value.py()))) .map_err(|error| PyValueError::new_err(error.to_string())) } diff --git a/crates/switchyard-py/src/serde_depth.rs b/crates/switchyard-py/src/serde_depth.rs new file mode 100644 index 000000000..d076fd9ea --- /dev/null +++ b/crates/switchyard-py/src/serde_depth.rs @@ -0,0 +1,325 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +//! Bounds the actual Serde traversal without inspecting Python objects twice. + +use std::fmt; + +use serde::{Deserialize, Deserializer, de}; + +const MAX_DEPTH: usize = 128; +const DEPTH_ERROR: &str = "Python value exceeds supported nesting depth (128)"; + +pub(crate) fn deserialize<'de, T, D>(deserializer: D) -> Result +where + T: Deserialize<'de>, + D: Deserializer<'de>, +{ + T::deserialize(Limited { + inner: deserializer, + remaining: MAX_DEPTH, + }) +} + +// The budget belongs to each branch. Wide containers do not consume the depth +// available to their siblings, and ignored values are never traversed. +struct Limited { + inner: T, + remaining: usize, +} + +macro_rules! deserialize_methods { + ($($method:ident $(($($arg:ident: $ty:ty),*))?),* $(,)?) => { + $( + fn $method(self, $($($arg: $ty,)*)? visitor: V) -> Result + where + V: de::Visitor<'de>, + { + let remaining = self.remaining.checked_sub(1).ok_or_else(|| { + ::custom(DEPTH_ERROR) + })?; + self.inner.$method($($($arg,)*)? Limited { inner: visitor, remaining }) + } + )* + }; +} + +impl<'de, D: Deserializer<'de>> Deserializer<'de> for Limited { + type Error = D::Error; + + deserialize_methods! { + deserialize_any, deserialize_bool, + deserialize_i8, deserialize_i16, deserialize_i32, deserialize_i64, deserialize_i128, + deserialize_u8, deserialize_u16, deserialize_u32, deserialize_u64, deserialize_u128, + deserialize_f32, deserialize_f64, deserialize_char, deserialize_str, deserialize_string, + deserialize_bytes, deserialize_byte_buf, deserialize_option, deserialize_unit, + deserialize_unit_struct(name: &'static str), + deserialize_newtype_struct(name: &'static str), + deserialize_seq, deserialize_tuple(len: usize), + deserialize_tuple_struct(name: &'static str, len: usize), deserialize_map, + deserialize_struct(name: &'static str, fields: &'static [&'static str]), + deserialize_enum(name: &'static str, variants: &'static [&'static str]), + deserialize_identifier, deserialize_ignored_any, + } + + fn is_human_readable(&self) -> bool { + self.inner.is_human_readable() + } +} + +macro_rules! scalar_visits { + ($($method:ident $(($value:ident: $ty:ty))?),* $(,)?) => { + $( + fn $method(self $(, $value: $ty)?) -> Result + where + E: de::Error, + { + self.inner.$method($($value)?) + } + )* + }; +} + +impl<'de, V: de::Visitor<'de>> de::Visitor<'de> for Limited { + type Value = V::Value; + + fn expecting(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + self.inner.expecting(formatter) + } + + scalar_visits! { + visit_bool(value: bool), + visit_i8(value: i8), visit_i16(value: i16), visit_i32(value: i32), + visit_i64(value: i64), visit_i128(value: i128), + visit_u8(value: u8), visit_u16(value: u16), visit_u32(value: u32), + visit_u64(value: u64), visit_u128(value: u128), + visit_f32(value: f32), visit_f64(value: f64), visit_char(value: char), + visit_str(value: &str), visit_borrowed_str(value: &'de str), visit_string(value: String), + visit_bytes(value: &[u8]), visit_borrowed_bytes(value: &'de [u8]), visit_byte_buf(value: Vec), + visit_none, visit_unit, + } + + fn visit_some>(self, deserializer: D) -> Result { + self.inner.visit_some(Limited { + inner: deserializer, + remaining: self.remaining, + }) + } + + fn visit_newtype_struct>( + self, + deserializer: D, + ) -> Result { + self.inner.visit_newtype_struct(Limited { + inner: deserializer, + remaining: self.remaining, + }) + } + + fn visit_seq>(self, access: A) -> Result { + self.inner.visit_seq(Limited { + inner: access, + remaining: self.remaining, + }) + } + + fn visit_map>(self, access: A) -> Result { + self.inner.visit_map(Limited { + inner: access, + remaining: self.remaining, + }) + } + + fn visit_enum>(self, access: A) -> Result { + self.inner.visit_enum(Limited { + inner: access, + remaining: self.remaining, + }) + } +} + +impl<'de, S: de::DeserializeSeed<'de>> de::DeserializeSeed<'de> for Limited { + type Value = S::Value; + + fn deserialize>(self, deserializer: D) -> Result { + self.inner.deserialize(Limited { + inner: deserializer, + remaining: self.remaining, + }) + } +} + +impl<'de, A: de::SeqAccess<'de>> de::SeqAccess<'de> for Limited { + type Error = A::Error; + + fn next_element_seed>( + &mut self, + seed: S, + ) -> Result, Self::Error> { + self.inner.next_element_seed(Limited { + inner: seed, + remaining: self.remaining, + }) + } + + fn size_hint(&self) -> Option { + self.inner.size_hint() + } +} + +impl<'de, A: de::MapAccess<'de>> de::MapAccess<'de> for Limited { + type Error = A::Error; + + fn next_key_seed>( + &mut self, + seed: S, + ) -> Result, Self::Error> { + self.inner.next_key_seed(Limited { + inner: seed, + remaining: self.remaining, + }) + } + + fn next_value_seed>( + &mut self, + seed: S, + ) -> Result { + self.inner.next_value_seed(Limited { + inner: seed, + remaining: self.remaining, + }) + } + + fn size_hint(&self) -> Option { + self.inner.size_hint() + } +} + +impl<'de, A: de::EnumAccess<'de>> de::EnumAccess<'de> for Limited { + type Error = A::Error; + type Variant = Limited; + + fn variant_seed>( + self, + seed: S, + ) -> Result<(S::Value, Self::Variant), Self::Error> { + let (value, variant) = self.inner.variant_seed(Limited { + inner: seed, + remaining: self.remaining, + })?; + Ok(( + value, + Limited { + inner: variant, + remaining: self.remaining, + }, + )) + } +} + +impl<'de, A: de::VariantAccess<'de>> de::VariantAccess<'de> for Limited { + type Error = A::Error; + + fn unit_variant(self) -> Result<(), Self::Error> { + self.inner.unit_variant() + } + + fn newtype_variant_seed>( + self, + seed: S, + ) -> Result { + self.inner.newtype_variant_seed(Limited { + inner: seed, + remaining: self.remaining, + }) + } + + fn tuple_variant>( + self, + len: usize, + visitor: V, + ) -> Result { + self.inner.tuple_variant( + len, + Limited { + inner: visitor, + remaining: self.remaining, + }, + ) + } + + fn struct_variant>( + self, + fields: &'static [&'static str], + visitor: V, + ) -> Result { + self.inner.struct_variant( + fields, + Limited { + inner: visitor, + remaining: self.remaining, + }, + ) + } +} + +#[cfg(test)] +mod tests { + use super::{DEPTH_ERROR, MAX_DEPTH, deserialize}; + use serde::Deserialize; + use serde_json::{Value, json}; + + fn nested_arrays(depth: usize) -> Value { + (0..depth).fold(Value::Null, |value, _| Value::Array(vec![value])) + } + + #[test] + fn depth_is_limited_per_branch() { + let input = Value::Array(vec![ + nested_arrays(MAX_DEPTH - 2), + nested_arrays(MAX_DEPTH - 2), + ]); + assert_eq!(deserialize::(input.clone()).unwrap(), input); + + let error = deserialize::(nested_arrays(MAX_DEPTH)).unwrap_err(); + assert!(error.to_string().contains(DEPTH_ERROR)); + } + + #[test] + fn preserves_compound_enum_dispatch() { + #[derive(Debug, Deserialize, PartialEq)] + struct Newtype(Option>); + + #[derive(Debug, Deserialize, PartialEq)] + enum Example { + Unit, + Newtype(Newtype), + Tuple(u8, bool), + Struct { value: Newtype }, + } + + let cases = [ + (json!("Unit"), Example::Unit), + ( + json!({"Newtype": [1, 2]}), + Example::Newtype(Newtype(Some(vec![1, 2]))), + ), + (json!({"Tuple": [3, true]}), Example::Tuple(3, true)), + ( + json!({"Struct": {"value": null}}), + Example::Struct { + value: Newtype(None), + }, + ), + ]; + for (input, expected) in cases { + assert_eq!(deserialize::(input).unwrap(), expected); + } + } + + #[test] + fn preserves_borrowed_strings() { + let mut input = serde_json::Deserializer::from_str("\"borrowed\""); + assert_eq!(deserialize::<&str, _>(&mut input).unwrap(), "borrowed"); + } +} diff --git a/docs/simulation.md b/docs/simulation.md index 7e13b430d..1f334c312 100644 --- a/docs/simulation.md +++ b/docs/simulation.md @@ -489,7 +489,9 @@ for scoring. `Runner.load` and `Runner.from_toml` use Switchyard's configuration parser. Configuration and request validation failures raise `ValueError`; incompatible -Python argument types can also raise `TypeError`. Execution +Python argument types can also raise `TypeError`. Native conversion uses a fixed +nesting limit and rejects cyclic or excessively nested values it consumes with +`ValueError`. Shared values that do not form a cycle remain supported. Execution failures raise `DecisionError` with safe diagnostics and completed observations; an application deadline can instead raise `asyncio.TimeoutError`. Cancellation waits for local routing and its Python bridge to stop. diff --git a/tests/test_runner_bindings.py b/tests/test_runner_bindings.py index 7ae3fd212..0af6d6f9d 100644 --- a/tests/test_runner_bindings.py +++ b/tests/test_runner_bindings.py @@ -848,6 +848,113 @@ async def test_bad_request_and_configuration_fail_before_calls( assert judge.calls == [] +@pytest.mark.parametrize( + "case", ["dict", "list", "user-dict", "dataclass", "deep", "tool-result", "schema"] +) +async def test_recursive_python_inputs_raise_without_terminating_process( + judge: JudgeStub, tmp_path: Path, case: str +) -> None: + config = tmp_path / "routes.toml" + config.write_text(deployment(judge.url)) + script = r""" +import asyncio +import sys +from collections import UserDict +from dataclasses import dataclass + +if sys.platform != "win32": + import resource + resource.setrlimit(resource.RLIMIT_CORE, (0, 0)) + +from switchyard.libsy import CustomClassifierConfig +from switchyard.runner import Runner + +runner = Runner.load(sys.argv[1]) +case = sys.argv[2] +if case == "list": + value = [] + value.append(value) +elif case == "dataclass": + @dataclass + class Node: + child: object = None + value = Node() + value.child = value +elif case == "deep": + value = {} + for _ in range(4096): + value = {"nested": value} +elif case == "tool-result": + value = {"type": "tool_result", "tool_call_id": "call", "content": []} + value["content"].append(value) +else: + value = UserDict() if case == "user-dict" else {} + value["nested"] = value + +async def main(): + request = {"model": "fixed", "messages": []} + if case == "tool-result": + request["messages"] = [{"role": "tool", "content": [value]}] + else: + request["output"] = {"response_format": value} + try: + if case == "schema": + CustomClassifierConfig("Select a target.", value, "/target") + else: + await runner.decide(request) + except ValueError as error: + assert "nesting" in str(error) + assert error.__cause__ is None and error.__context__ is None + else: + raise AssertionError("recursive input was accepted") + # The same interpreter and runner remain usable after rejecting the input. + decision = await runner.decide({"model": "fixed", "messages": []}) + assert decision.selected.target == "weak" and not decision.calls + +asyncio.run(main()) +""" + process = await asyncio.create_subprocess_exec( + sys.executable, + "-c", + script, + str(config), + case, + stdout=asyncio.subprocess.PIPE, + stderr=asyncio.subprocess.PIPE, + ) + try: + stdout, stderr = await asyncio.wait_for(process.communicate(), timeout=15) + assert process.returncode == 0, stderr.decode() + assert not stdout and not stderr + assert judge.calls == [] + finally: + if process.returncode is None: + process.kill() + await process.wait() + + +async def test_python_conversion_preserves_shared_values_and_ignored_fields( + judge: JudgeStub, +) -> None: + shared: dict[str, object] = {"type": "string"} + for _ in range(16): + shared = {"nested": shared} + ignored: dict[str, object] = {} + ignored["self"] = ignored + original = { + **request(model="fixed"), + "output": {"response_format": {"left": shared, "right": shared}}, + "ignored_field": ignored, + } + decision = await Runner.from_toml(deployment(judge.url)).decide(original) + assert decision.outcome.request["output"]["response_format"] == { + "left": shared, + "right": shared, + } + assert "ignored_field" not in decision.outcome.request + assert not decision.calls and judge.calls == [] + + def test_configuration_diagnostics_name_missing_target_and_environment( judge: JudgeStub, monkeypatch: pytest.MonkeyPatch ) -> None: