feat: switchyard sim

.github/workflows/package-portability.yml

@@ -84,16 +84,24 @@ jobs:
           import os
           import sys
           import tempfile
+          from importlib.resources import files
 
           os.chdir(tempfile.mkdtemp(prefix="switchyard-wheel-smoke-"))
           workspace = os.environ.get("GITHUB_WORKSPACE")
           sys.path = [entry for entry in sys.path if entry not in ("", workspace)]
 
           import switchyard
           import switchyard_rust
+          from switchyard.runner import Runner
+          from switchyard.sim import Trajectory
           from switchyard_rust import _switchyard_rust
 
+          for package in ("switchyard", "switchyard_rust"):
+              assert files(package).joinpath("py.typed").is_file(), package
+
           print("switchyard", switchyard.__version__)
+          print("runner", Runner)
+          print("trajectory", Trajectory)
           print("switchyard_rust", switchyard_rust.__file__)
           print("rust extension", _switchyard_rust.__name__)
           PY
@@ -109,16 +117,24 @@ jobs:
           import os
           import sys
           import tempfile
+          from importlib.resources import files
 
           os.chdir(tempfile.mkdtemp(prefix="switchyard-wheel-smoke-"))
           workspace = os.environ.get("GITHUB_WORKSPACE")
           sys.path = [entry for entry in sys.path if entry not in ("", workspace)]
 
           import switchyard
           import switchyard_rust
+          from switchyard.runner import Runner
+          from switchyard.sim import Trajectory
           from switchyard_rust import _switchyard_rust
 
+          for package in ("switchyard", "switchyard_rust"):
+              assert files(package).joinpath("py.typed").is_file(), package
+
           print("switchyard", switchyard.__version__)
+          print("runner", Runner)
+          print("trajectory", Trajectory)
           print("switchyard_rust", switchyard_rust.__file__)
           print("rust extension", _switchyard_rust.__name__)
           PY

.github/workflows/publish.yml

@@ -278,10 +278,14 @@ jobs:
 
           import switchyard
           import switchyard_rust
+          from switchyard.runner import Runner
+          from switchyard.sim import Trajectory
           from switchyard_rust import _switchyard_rust
           from switchyard_rust.server import Server
 
           print("switchyard", switchyard.__version__)
+          print("runner", Runner)
+          print("trajectory", Trajectory)
           print("switchyard_rust", switchyard_rust.__file__)
           print("rust extension", _switchyard_rust.__name__)
           print("server", Server)
@@ -307,10 +311,14 @@ jobs:
 
           import switchyard
           import switchyard_rust
+          from switchyard.runner import Runner
+          from switchyard.sim import Trajectory
           from switchyard_rust import _switchyard_rust
           from switchyard_rust.server import Server
 
           print("switchyard", switchyard.__version__)
+          print("runner", Runner)
+          print("trajectory", Trajectory)
           print("switchyard_rust", switchyard_rust.__file__)
           print("rust extension", _switchyard_rust.__name__)
           print("server", Server)

README.md

@@ -41,7 +41,9 @@ Agent-specific guides are available for [pi](docs/integrations/pi.md) and
 
 ### Embed the library in your harness
 
-[Embed the library in your harness](docs/getting_started.md#library-path) to run routing inside your Rust application. For Python, see the [embedding example](examples/libsy.py).
+[Embed the library in your harness](docs/getting_started.md#library-path) to run routing inside your Rust application.
+For Python, use the [configured decision API](docs/simulation.md#use-the-python-decision-api)
+or the [algorithm stream example](examples/libsy.py).
 
 ## Routing algorithms
 
@@ -63,6 +65,8 @@ The [routing overview](docs/routing_algorithms/overview.md) retains the full cat
 ![Task completion versus cost for Switchyard classification, stage, and escalation routing, compared with Opus 4.8 and GLM 5.2 single-model baselines.](assets/switchyard-cost-accuracy.png)
 
 Results depend on the benchmark, model pool, serving stack, and routing configuration.
+Use the Python [task routing evaluator](docs/simulation.md) to compare routing decisions against ATIF trajectories, custom recordings, or Harbor Claude and Codex runs.
+See its [architecture](docs/simulation-architecture.md) for the ATIF converter boundary, routing and scoring flow, and future replay extension.
 For latency and routing overhead testing, see [Soak Testing](docs/operations/soak_test.md).
 
 ### Further reading

crates/libsy-llm-client/src/lib.rs

@@ -30,7 +30,7 @@ pub use client::{AuxiliaryOperation, ModelConfig, TranslatingLlmClient};
 pub use error::{LlmClientError, Result};
 pub use observation::{LlmCallObservation, RunObservation, RunObserver};
 pub use raw::RawResponse;
-pub use run::{ClientRouter, decide, run};
+pub use run::{ClientRouter, decide, decide_with_observer, run};
 pub use switchyard_translation::RawEventStream;
 
 /// Registers process-wide compatibility gauges with the global meter provider.

crates/libsy-llm-client/src/observation.rs

@@ -22,7 +22,7 @@ pub struct LlmCallObservation {
     pub usage: Option<Usage>,
 }
 
-/// Events emitted inline while [`crate::run`] serves a routing request.
+/// Events emitted inline by [`crate::run`] and [`crate::decide_with_observer`].
 #[derive(Clone, Debug)]
 pub enum RunObservation {
     /// Metadata attached to the completed routing outcome.
@@ -31,7 +31,7 @@ pub enum RunObservation {
     LlmCall(LlmCallObservation),
     /// A completed terminal model call made from the routing outcome.
     AnswerCall(LlmCallObservation),
-    /// Routing time recorded by the `switchyard.routing_overhead_ms` metric.
+    /// Elapsed routing time, including model calls required by the algorithm.
     RoutingOverhead(Duration),
 }
 

crates/libsy-llm-client/src/run.rs

@@ -131,23 +131,57 @@ pub async fn decide(
     request: Request,
     models: Arc<RuntimeModels>,
 ) -> Result<RoutingOutcome> {
+    decide_with_observer(algorithm, clients, request, models, None).await
+}
+
+/// Resolve a decision and report its completed calls and elapsed routing time.
+///
+/// Observations belong to this invocation, including calls completed before a failure.
+/// A routing-time response is reported as an answer call when it becomes the outcome.
+pub async fn decide_with_observer(
+    algorithm: Arc<dyn Algorithm>,
+    clients: ClientRouter,
+    request: Request,
+    models: Arc<RuntimeModels>,
+    observer: Option<RunObserver>,
+) -> Result<RoutingOutcome> {
+    let started = Instant::now();
     let routing_clients = clients.clone();
-    let mut outcome = match clients.stored_state_owner(&request) {
-        Some(owner) => continue_on(owner, algorithm.name(), request),
+    let observations = observer.as_ref().map(|_| Arc::new(Mutex::new(Vec::new())));
+    let outcome = match clients.stored_state_owner(&request) {
+        Some(owner) => Ok(continue_on(owner, algorithm.name(), request)),
         None => {
-            drive(algorithm, request, models, move |call| {
-                serve(routing_clients.clone(), call, None)
+            drive(algorithm, request, models, {
+                let observations = observations.clone();
+                move |call| serve(routing_clients.clone(), call, observations.clone())
             })
-            .await?
+            .await
         }
     };
-    let selected_model_id = outcome.selected_model_id()?.clone();
-    outcome.request = clients.prepare_completion_request(outcome.request, &selected_model_id);
-    outcome.response = outcome
-        .response
-        .map(|response| clients.remember_state_owner(&outcome.request, response))
-        .transpose()?;
-    Ok(outcome)
+    let answered_model = outcome
+        .as_ref()
+        .ok()
+        .and_then(|outcome| outcome.response.as_ref())
+        .and_then(Response::served_model);
+    emit_routing_observations(&observer, &observations, answered_model);
+    let outcome = outcome.and_then(|mut outcome| {
+        let selected_model_id = outcome.selected_model_id()?.clone();
+        outcome.request = clients.prepare_completion_request(outcome.request, &selected_model_id);
+        outcome.response = outcome
+            .response
+            .map(|response| clients.remember_state_owner(&outcome.request, response))
+            .transpose()?;
+        Ok(outcome)
+    });
+    if let Some(observer) = observer {
+        if let Ok(outcome) = &outcome
+            && let Some(metadata) = &outcome.metadata
+        {
+            observer(RunObservation::Outcome(metadata.clone()));
+        }
+        observer(RunObservation::RoutingOverhead(started.elapsed()));
+    }
+    outcome
 }
 
 /// Emits completed routing calls after the outcome reveals whether one response became the answer.
@@ -566,6 +600,11 @@ enum Routing {
 }
 
 impl ClientRouter {
+    /// Configured target that may produce an answer during routing.
+    pub fn routing_answer_target(&self) -> Option<&ModelId> {
+        self.inner.routing_answer_target.as_ref()
+    }
+
     /// Build a router over `model name -> client`, for targets spread across providers.
     pub fn new(by_model: HashMap<ModelId, Arc<dyn RoutedLlmClient>>) -> Self {
         Self::new_with_target_prompts(by_model, HashMap::new(), None)
@@ -1817,6 +1856,70 @@ mod tests {
         Ok(())
     }
 
+    #[tokio::test]
+    async fn observed_decision_does_not_call_a_selected_completion() -> Result<()> {
+        let client = Arc::new(CandidateClient {
+            calls: Mutex::new(Vec::new()),
+            requests: Mutex::new(Vec::new()),
+            first: FirstOutcome::StreamSuccess,
+        });
+        let observations = Arc::new(Mutex::new(Vec::new()));
+        let captured = Arc::clone(&observations);
+        let observer: RunObserver = Arc::new(move |event| captured.lock().push(event));
+
+        let outcome = decide_with_observer(
+            Arc::new(CandidateAlgorithm {}),
+            ClientRouter::single(client.clone()),
+            request(),
+            to_category_map(&["weak", "strong"]),
+            Some(observer),
+        )
+        .await?;
+
+        assert_eq!(outcome.selected_model_id()?, "weak");
+        assert!(outcome.response.is_none());
+        assert!(client.calls.lock().is_empty());
+        assert!(matches!(
+            &observations.lock()[..],
+            [
+                RunObservation::Outcome(_),
+                RunObservation::RoutingOverhead(_)
+            ]
+        ));
+        Ok(())
+    }
+
+    #[tokio::test]
+    async fn observed_decision_reports_completed_calls_on_failure() {
+        let client = Arc::new(CandidateClient {
+            calls: Mutex::new(Vec::new()),
+            requests: Mutex::new(Vec::new()),
+            first: FirstOutcome::Unauthorized,
+        });
+        let observations = Arc::new(Mutex::new(Vec::new()));
+        let captured = Arc::clone(&observations);
+        let observer: RunObserver = Arc::new(move |event| captured.lock().push(event));
+
+        let result = decide_with_observer(
+            Arc::new(AnsweredAlgorithm {
+                model: "weak".into(),
+            }),
+            ClientRouter::single(client.clone()),
+            request(),
+            to_category_map(&["weak", "strong"]),
+            Some(observer),
+        )
+        .await;
+
+        assert!(result.is_err());
+        assert_eq!(&*client.calls.lock(), &[ModelId::from("weak")]);
+        assert!(matches!(
+            &observations.lock()[..],
+            [RunObservation::LlmCall(call), RunObservation::RoutingOverhead(_)]
+                if call.selected_model == "weak" && !call.is_success && call.usage.is_none()
+        ));
+    }
+
     #[tokio::test]
     async fn decision_prompts_a_routing_response_target() -> Result<()> {
         let client = Arc::new(CandidateClient {