diff --git a/crates/libsy-llm-client/src/backend.rs b/crates/libsy-llm-client/src/backend.rs index 064b13546..63a9ecfa3 100644 --- a/crates/libsy-llm-client/src/backend.rs +++ b/crates/libsy-llm-client/src/backend.rs @@ -31,6 +31,7 @@ const OPENAI_OVERFLOW_PHRASES: &[&str] = &[ "please reduce the length of the input", "exceeds the maximum allowed input length", "exceeds the maximum allowed length", + "exceeds the available context size", "is longer than the model's context length", ]; @@ -196,6 +197,10 @@ impl Backend { } } + pub(crate) fn base_url(&self) -> &str { + &self.config().base_url + } + /// The fully resolved upstream URL for this backend's endpoint. /// /// Tolerates base URLs that already include the provider path (or a bare diff --git a/crates/libsy-llm-client/src/client.rs b/crates/libsy-llm-client/src/client.rs index f2d0301dd..6dd493f45 100644 --- a/crates/libsy-llm-client/src/client.rs +++ b/crates/libsy-llm-client/src/client.rs @@ -183,6 +183,51 @@ impl TranslatingLlmClient { self.backend_for(model, operation.wire_format()).is_some() } + /// Reads llama.cpp model properties through the backend's configured auth and headers. + /// + /// Redirects are disabled so credentials cannot move to another origin. + pub async fn get_model_properties( + &self, + model: &ModelId, + format: WireFormat, + timeout: Duration, + ) -> Result { + let backend = + self.backend_for(model, format) + .ok_or_else(|| LlmClientError::Configuration { + message: format!("model {model} has no backend for {format:?}"), + })?; + let mut url = reqwest::Url::parse(backend.base_url()).map_err(|error| { + LlmClientError::Configuration { + message: format!("model {model} has an invalid backend URL: {error}"), + } + })?; + let root = url + .path() + .trim_end_matches('/') + .strip_suffix("/v1") + .unwrap_or_else(|| url.path().trim_end_matches('/')) + .to_string(); + url.set_path(&format!("{}/props", root.trim_end_matches('/'))); + url.set_query(None); + + let builder = self.forward_auth_client.get(url).timeout(timeout); + let builder = apply_extra_headers(builder, backend); + let response = backend + .apply_auth(builder) + .send() + .await + .map_err(convert_reqwest_error)? + .error_for_status() + .map_err(convert_reqwest_error)?; + response + .json() + .await + .map_err(|source| LlmClientError::InvalidResponse { + source: Box::new(source), + }) + } + /// Calls a model-bearing auxiliary provider operation. /// /// Returns an error when the model has no compatible backend or the upstream @@ -259,6 +304,9 @@ impl TranslatingLlmClient { if matches!(backend, Backend::Anthropic(_)) { strip_anthropic_incompatible_fields(&mut body); strip_unsigned_thinking_blocks(&mut body); + if nvidia_inference_api_backend(backend) { + strip_anthropic_tool_strict(&mut body); + } } omit_configured_body_fields(&mut body, backend.omit_body_fields()); merge_extra_body(&mut body, backend.extra_body()); @@ -1047,6 +1095,24 @@ fn strip_anthropic_incompatible_fields(body: &mut Value) { } } +// NVIDIA Inference API's Anthropic-compatible schema rejects the strict tool-use +// extension that native Anthropic accepts. +fn nvidia_inference_api_backend(backend: &Backend) -> bool { + reqwest::Url::parse(backend.base_url()) + .ok() + .and_then(|url| url.host_str().map(ToOwned::to_owned)) + .is_some_and(|host| host.eq_ignore_ascii_case("inference-api.nvidia.com")) +} + +fn strip_anthropic_tool_strict(body: &mut Value) { + let Some(tools) = body.get_mut("tools").and_then(Value::as_array_mut) else { + return; + }; + for tool in tools.iter_mut().filter_map(Value::as_object_mut) { + tool.remove("strict"); + } +} + // Removes replayed `thinking` blocks that carry no signature. // // Anthropic requires signed thinking blocks on replay. A router can serve earlier @@ -1446,6 +1512,33 @@ mod tests { )] } + #[tokio::test] + async fn model_properties_uses_the_backends_configured_auth() + -> std::result::Result<(), Box> { + let server = MockServer::start().await; + Mock::given(method("GET")) + .and(path("/props")) + .and(wiremock::matchers::header("authorization", "Bearer secret")) + .respond_with(ResponseTemplate::new(200).set_body_json(json!({ + "default_generation_settings": {"n_ctx": 65_536} + }))) + .expect(1) + .mount(&server) + .await; + let client = TranslatingLlmClient::new(&chat_map(&format!("{}/v1", server.uri())))?; + + let properties = client + .get_model_properties( + &ModelId::from("gpt"), + WireFormat::OpenAiChat, + Duration::from_secs(1), + ) + .await?; + + assert_eq!(properties["default_generation_settings"]["n_ctx"], 65_536); + Ok(()) + } + fn chat_map_with_retries(base_url: &str, max_retries: u32) -> Vec { vec![ModelConfig::new( "gpt", @@ -1539,6 +1632,18 @@ mod tests { assert!(!convert_reqwest_error(error).to_string().contains("CANARY")); } + #[test] + fn nvidia_anthropic_tools_drop_unsupported_strict_field() { + let backend = Backend::Anthropic(config("https://inference-api.nvidia.com")); + assert!(nvidia_inference_api_backend(&backend)); + assert!(!nvidia_inference_api_backend(&Backend::Anthropic(config( + "https://inference-api.nvidia.com.example.test" + )))); + let mut body = json!({"tools": [{"name": "a", "input_schema": {}, "strict": true}]}); + strip_anthropic_tool_strict(&mut body); + assert!(body["tools"][0].get("strict").is_none()); + } + #[test] fn anthropic_prompt_caching_marks_final_message() { let caller_managed = json!({ @@ -2630,12 +2735,14 @@ mod tests { } #[tokio::test] - async fn context_overflow_400_is_mapped() + async fn llama_cpp_context_overflow_400_is_mapped() -> std::result::Result<(), Box> { let server = MockServer::start().await; Mock::given(method("POST")) .respond_with(ResponseTemplate::new(400).set_body_json(json!({ - "error": {"code": "context_length_exceeded", "message": "too big"} + "error": { + "message": "request (6016 tokens) exceeds the available context size (4096 tokens), try increasing it" + } }))) .mount(&server) .await; diff --git a/crates/switchyard-runner/src/algorithm.rs b/crates/switchyard-runner/src/algorithm.rs index e742d48ba..2a981a0fc 100644 --- a/crates/switchyard-runner/src/algorithm.rs +++ b/crates/switchyard-runner/src/algorithm.rs @@ -10,12 +10,13 @@ use std::path::PathBuf; use std::sync::Arc; use libsy::{ - AdvisorGate, AdvisorGateConfig, Algorithm, ClassifierContractConfig, ClassifierResponseFormat, - ClassifyTrigger, CompositeRouter, CompositeRouterConfig, CustomClassifierConfig, - CustomClassifierPolicy, EscalationJudgeConfig, GateTrigger, HandoffNoteConfig, - LlmClassifierConfig, LlmFallback, LlmTaskClassifier, Noop, Passthrough, PickerMode, - PlanExecute, PlanExecuteConfig, Random, StageRouter, StageRouterConfig, SubagentRouter, - SubagentRouterConfig, TaskClassifierConfig, ToolSemantics, + AdvisorGate, AdvisorGateConfig, Algorithm, BreakerConfig, ClassifierContractConfig, + ClassifierResponseFormat, ClassifyTrigger, CompositeRouter, CompositeRouterConfig, + CustomClassifierConfig, CustomClassifierPolicy, EscalationJudgeConfig, GateTrigger, + HandoffNoteConfig, LlmClassifierConfig, LlmFallback, LlmTaskClassifier, Noop, Passthrough, + PickerMode, PlanExecute, PlanExecuteConfig, Random, StageRouter, StageRouterConfig, + SubagentRouter, SubagentRouterConfig, TaskClassifierConfig, ToolSemantics, Vgr, VgrConfig, + VgrTargets, }; use serde::Deserialize; use switchyard_protocol::{Category, ModelId}; @@ -428,6 +429,10 @@ pub enum AlgorithmSpec { #[serde(default = "default_fail_open")] fail_open: bool, }, + Vgr { + #[serde(flatten)] + config: VgrRouteConfig, + }, /// Routes using a checkpoint-backed prefill classifier. PrefillRouter { /// Target names in checkpoint output order. @@ -446,6 +451,43 @@ pub enum AlgorithmSpec { }, } +#[derive(Clone, Debug, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct VgrRouteConfig { + pub local_target: String, + pub cloud_target: String, + /// Defaults to `local_target`. + #[serde(default)] + pub judge_target: Option, + #[serde(default)] + pub cloud_judge_target: Option, + #[serde(default)] + pub mode: VgrModeConfig, + #[serde(default)] + pub active_approval: Option, + #[serde(default = "default_vgr_deadline")] + pub deadline_seconds: f64, + #[serde(default = "default_true")] + pub task_typing: bool, + #[serde(default = "default_vgr_breaker_threshold")] + pub breaker_threshold: u32, + #[serde(default = "default_vgr_breaker_cooldown")] + pub breaker_cooldown_seconds: f64, + /// Clean trailing tool results that let a long run recover on the cloud judge. + #[serde(default)] + pub confirmed_recovery_min_clean_tail: Option, +} + +#[derive(Clone, Copy, Debug, Default, Deserialize, Eq, PartialEq)] +#[serde(rename_all = "snake_case")] +pub enum VgrModeConfig { + #[default] + Off, + Evaluate, + Shadow, + Active, +} + /// What fires an advisor route's review. #[derive(Clone, Debug, Default, Deserialize, PartialEq)] #[serde(rename_all = "snake_case")] @@ -606,6 +648,9 @@ impl AlgorithmSpec { Self::Advisor { executor_target, .. } => vec![executor_target], + Self::Vgr { config } => { + vec![config.local_target.as_str(), config.cloud_target.as_str()] + } Self::PrefillRouter { targets, .. } => targets.iter().map(String::as_str).collect(), } } @@ -642,6 +687,10 @@ impl AlgorithmSpec { names.push(&classifier.target); } Self::Advisor { advisor_target, .. } => names.push(advisor_target), + Self::Vgr { config } => { + names.extend(config.judge_target.as_deref()); + names.extend(config.cloud_judge_target.as_deref()); + } _ => {} } // A sub-agent classifier calls its own judge, which is never a completion target. @@ -738,6 +787,22 @@ impl AlgorithmSpec { (Category::Any, vec![executor_target.clone()]), (Category::Judge, vec![advisor_target.clone()]), ]), + Self::Vgr { config } => { + let judge = config.judge_target.as_ref().unwrap_or(&config.local_target); + let judges = std::iter::once(judge) + .chain(&config.cloud_judge_target) + .cloned() + .collect(); + category_models([ + (Category::Efficient, vec![config.local_target.clone()]), + (Category::Capable, vec![config.cloud_target.clone()]), + ( + Category::Any, + vec![config.local_target.clone(), config.cloud_target.clone()], + ), + (Category::Judge, judges), + ]) + } }; let subagents = match self { @@ -771,6 +836,13 @@ impl AlgorithmSpec { advisor_target, .. } => Some((executor_target, advisor_target)), + Self::Vgr { config } => Some(( + &config.local_target, + config + .judge_target + .as_deref() + .unwrap_or(&config.local_target), + )), Self::Noop { .. } | Self::Random { .. } | Self::Passthrough { .. } @@ -1412,6 +1484,7 @@ fn build_algorithm( })?; Ok(Arc::new(algorithm)) } + AlgorithmSpec::Vgr { config } => build_vgr(route_name, config, targets), AlgorithmSpec::PrefillRouter { targets: names, checkpoint, @@ -1457,6 +1530,64 @@ fn build_algorithm( } } +fn build_vgr( + route_name: &str, + config: &VgrRouteConfig, + targets: &BTreeMap, +) -> AlgorithmResult> { + let local = resolve_target_model_id(route_name, &config.local_target, targets)?; + let cloud = resolve_target_model_id(route_name, &config.cloud_target, targets)?; + if local == cloud { + return Err(AlgorithmConfigError::new(format!( + "vgr route {route_name} requires distinct local and cloud model IDs" + ))); + } + let optional = |name: &Option| { + name.as_deref() + .map(|name| resolve_target_model_id(route_name, name, targets)) + .transpose() + }; + let mut runtime = VgrConfig::new(local, cloud); + runtime.targets = VgrTargets { + judge: optional(&config.judge_target)?, + cloud_judge: optional(&config.cloud_judge_target)?, + ..runtime.targets + }; + runtime.mode = match config.mode { + VgrModeConfig::Off => libsy::ServingMode::Off, + VgrModeConfig::Evaluate => libsy::ServingMode::Evaluate, + VgrModeConfig::Shadow => libsy::ServingMode::Shadow, + VgrModeConfig::Active => libsy::ServingMode::Active { + approval: config.active_approval.clone().unwrap_or_default(), + }, + }; + runtime.deadline = duration(route_name, "deadline_seconds", config.deadline_seconds)?; + runtime.task_typing = config.task_typing; + runtime.confirmed_recovery_min_clean_tail = config.confirmed_recovery_min_clean_tail; + runtime.breaker = BreakerConfig { + threshold: config.breaker_threshold, + cooldown: duration( + route_name, + "breaker_cooldown_seconds", + config.breaker_cooldown_seconds, + )?, + }; + Vgr::new(runtime) + .map(|algorithm| Arc::new(algorithm) as Arc) + .map_err(|error| { + AlgorithmConfigError::with_source(format!("vgr route {route_name}: {error}"), error) + }) +} + +fn duration(route: &str, field: &str, seconds: f64) -> AlgorithmResult { + std::time::Duration::try_from_secs_f64(seconds).map_err(|error| { + AlgorithmConfigError::with_source( + format!("vgr route {route}: {field} must be a non-negative number"), + error, + ) + }) +} + const fn default_max_reviews() -> u32 { 1 } @@ -1492,6 +1623,22 @@ fn warn_single_target_classifier(route_name: &str, models: &CategoryModelConfig) } } +const fn default_vgr_deadline() -> f64 { + 30.0 +} + +const fn default_true() -> bool { + true +} + +const fn default_vgr_breaker_threshold() -> u32 { + 5 +} + +const fn default_vgr_breaker_cooldown() -> f64 { + 30.0 +} + fn resolve_target_model_id( route_name: &str, name: &str, diff --git a/crates/switchyard-runner/src/config.rs b/crates/switchyard-runner/src/config.rs index 05f03e2ce..543c52611 100644 --- a/crates/switchyard-runner/src/config.rs +++ b/crates/switchyard-runner/src/config.rs @@ -19,6 +19,7 @@ use switchyard_llm_client::{ }; use switchyard_protocol::{Category, ModelId, RoutedLlmClient, WireFormat}; +use crate::runner::ModelPropertiesProbe; use crate::{ AlgorithmSpec, AuxiliaryTarget, CallerAuthKind, DecisionTarget, ModelCapabilities, Route, Runner, RunnerError, @@ -214,6 +215,7 @@ impl DeploymentConfig { let targets = self.build_targets(); let fallback_base_url = self.fallback_base_url()?; let mut routes = Vec::with_capacity(self.routes.len()); + let mut model_properties_probes = BTreeMap::new(); for (route_name, config) in &self.routes { for target_name in config.callable_target_names() { self.targets.get(target_name).ok_or_else(|| { @@ -232,6 +234,11 @@ impl DeploymentConfig { .algorithm .build(route_name, &targets) .map_err(|error| RunnerError::configuration_source(error.to_string(), error))?; + if matches!(&config.algorithm, AlgorithmSpec::Vgr { .. }) + && let Some(probe) = self.build_model_properties_probe(config, &clients) + { + model_properties_probes.insert(config.id.clone(), probe); + } let (route_clients, caller_auth) = self.build_route_clients(route_name, config, &clients)?; let anthropic_auxiliary_target = @@ -265,7 +272,8 @@ impl DeploymentConfig { } let runner = Runner::new(routes) .with_fallback_url(fallback_base_url) - .with_provider_api_keys(provider_api_keys); + .with_provider_api_keys(provider_api_keys) + .with_model_properties_probes(model_properties_probes); Ok(runner) } @@ -476,6 +484,22 @@ impl DeploymentConfig { Ok(Some(config.base_url.as_str().to_string())) } + fn build_model_properties_probe( + &self, + route: &RouteConfig, + clients: &BTreeMap>, + ) -> Option { + let local_target_name = route.routing_target_names().into_iter().next()?; + let local_target = self.targets.get(local_target_name)?; + let client_config = self.llm_clients.get(&local_target.llm_client)?; + let client = clients.get(&local_target.llm_client)?; + Some(ModelPropertiesProbe::new( + local_target.id.clone(), + client_config.format.wire_format(), + Arc::clone(client), + )) + } + fn build_anthropic_auxiliary_target( &self, route: &RouteConfig, @@ -908,6 +932,38 @@ target = "strong" } } + fn vgr_config() -> String { + format!( + r#"{VALID_CONFIG} + +[routes.vgr] +id = "switchyard/vgr" +type = "vgr" +local_target = "weak" +cloud_target = "strong" +judge_target = "classifier" +mode = "active" +active_approval = "prospective-validation-and-canary-approved" +task_typing = false +"# + ) + } + + #[test] + fn vgr_route_builds_and_rejects_unsafe_configuration() -> RunnerResult<()> { + let runner = runner_from_toml(&vgr_config())?; + assert!(runner.route("switchyard/vgr").is_some()); + + let bad_approval = + vgr_config().replace("prospective-validation-and-canary-approved", "approved"); + assert!(error_message(&bad_approval).contains("approval attestation")); + + let missing_target = + vgr_config().replace("local_target = \"weak\"", "local_target = \"missing\""); + assert!(error_message(&missing_target).contains("unknown target missing")); + Ok(()) + } + fn with_subagent_llm_classifier(config: &str, route: &str, extra: &str) -> String { let mut configured = config.to_string(); configured.push_str(&format!("\n[routes.{route}.subagents]\n")); diff --git a/crates/switchyard-runner/src/lib.rs b/crates/switchyard-runner/src/lib.rs index a488b31a7..572d0bbbf 100644 --- a/crates/switchyard-runner/src/lib.rs +++ b/crates/switchyard-runner/src/lib.rs @@ -13,6 +13,7 @@ mod runner; pub use algorithm::{ AdvisorTriggerConfig, AlgorithmConfigError, AlgorithmSpec, CategoryModelConfig, ClassifierMode, ClassifierPolicyConfig, LlmClassifierRouteConfig, StageClassifierConfig, SubagentRouteConfig, + VgrModeConfig, VgrRouteConfig, }; pub use failure::{RouteErrorKind, RouteErrorPhase, RouteErrorSummary, stream_error_summary}; // Re-exported because `Route::new` takes it, so a host wiring routes does not need a libsy dep. diff --git a/crates/switchyard-runner/src/runner.rs b/crates/switchyard-runner/src/runner.rs index 249e31271..03864a606 100644 --- a/crates/switchyard-runner/src/runner.rs +++ b/crates/switchyard-runner/src/runner.rs @@ -5,9 +5,12 @@ use std::collections::BTreeMap; use std::path::Path; +use std::sync::Arc; +use std::time::Duration; use libsy::RoutingOutcome; use serde_json::Value; +use switchyard_llm_client::TranslatingLlmClient; use switchyard_protocol::{ModelId, WireFormat}; use crate::config; @@ -18,6 +21,7 @@ pub struct Runner { routes: Vec<(ModelId, Route)>, fallback_base_url: Option, provider_api_keys: Vec, + model_properties_probes: BTreeMap, } /// Borrowed model metadata returned while listing routes. @@ -43,6 +47,33 @@ pub struct DecisionTarget { pub extra_body: BTreeMap, } +pub(crate) struct ModelPropertiesProbe { + model: ModelId, + format: WireFormat, + client: Arc, +} + +impl ModelPropertiesProbe { + pub(crate) fn new( + model: ModelId, + format: WireFormat, + client: Arc, + ) -> Self { + Self { + model, + format, + client, + } + } + + async fn get(&self, timeout: Duration) -> Option { + self.client + .get_model_properties(&self.model, self.format, timeout) + .await + .ok() + } +} + impl Runner { /// Loads and validates a version-1 deployment TOML file. pub fn load(path: impl AsRef) -> Result { @@ -63,6 +94,7 @@ impl Runner { routes, fallback_base_url: None, provider_api_keys: Vec::new(), + model_properties_probes: BTreeMap::new(), } } @@ -83,6 +115,14 @@ impl Runner { self } + pub(crate) fn with_model_properties_probes( + mut self, + probes: BTreeMap, + ) -> Self { + self.model_properties_probes = probes; + self + } + /// Returns the route registered for a model. pub fn route(&self, model: &str) -> Option<&Route> { self.routes @@ -100,6 +140,11 @@ impl Runner { }) } + /// Reads one route's configured local backend properties without exposing its credentials. + pub async fn model_properties(&self, model: &str, timeout: Duration) -> Option { + self.model_properties_probes.get(model)?.get(timeout).await + } + /// Returns the validated API root used for unmatched HTTP requests. pub fn fallback_base_url(&self) -> Option<&str> { self.fallback_base_url.as_deref() diff --git a/crates/switchyard-server/src/lib.rs b/crates/switchyard-server/src/lib.rs index fe5bdef83..8025faeab 100644 --- a/crates/switchyard-server/src/lib.rs +++ b/crates/switchyard-server/src/lib.rs @@ -1452,12 +1452,28 @@ fn error_response( } async fn models(State(state): State) -> Json { - Json(model_list_payload( - state - .runner - .models() - .map(|model| (model.id.as_str(), model.capabilities)), - )) + let mut entries = Vec::new(); + for model in state.runner.models() { + let mut capabilities = model.capabilities; + // A VGR route serves its local tier's context, which llama.cpp reports live. + if model.algorithm == "vgr" + && let Some(context) = discover_context_window(&state.runner, model.id).await + { + capabilities.context_window = Some(context); + } + entries.push((model.id.as_str(), capabilities)); + } + Json(model_list_payload(entries)) +} + +async fn discover_context_window(runner: &Runner, model: &ModelId) -> Option { + let body = runner + .model_properties(model.as_str(), Duration::from_secs(2)) + .await?; + let context = body + .pointer("/default_generation_settings/n_ctx")? + .as_u64()?; + u32::try_from(context).ok().filter(|context| *context > 0) } async fn get_stats(State(state): State) -> Json { diff --git a/crates/switchyard-server/tests/server.rs b/crates/switchyard-server/tests/server.rs index cd21d3c91..01ba9e01a 100644 --- a/crates/switchyard-server/tests/server.rs +++ b/crates/switchyard-server/tests/server.rs @@ -14,7 +14,7 @@ use axum::extract::{DefaultBodyLimit, Path, State}; use axum::http::{HeaderMap, HeaderValue, Request as HttpRequest, StatusCode, Uri}; use axum::response::sse::{Event, Sse}; use axum::response::{IntoResponse, Response as HttpResponse}; -use axum::routing::post; +use axum::routing::{get, post}; use axum::{Json, Router}; use http_body_util::BodyExt; use libsy::{Algorithm, Random}; @@ -50,6 +50,7 @@ impl MockUpstream { async fn start() -> TestResult { let calls = Arc::new(Mutex::new(Vec::new())); let app = Router::new() + .route("/props", get(upstream_props)) .route("/v1/chat/completions", post(upstream_chat)) .route("/buffered/responses", post(upstream_buffered_responses)) .route( @@ -118,6 +119,10 @@ fn has_system_prompt(call: &Value, expected: &str) -> bool { }) } +async fn upstream_props() -> Json { + Json(json!({"default_generation_settings": {"n_ctx": 98_304}})) +} + async fn upstream_chat( State(calls): State>>>, Json(body): Json, @@ -414,6 +419,10 @@ async fn upstream_chat( r#"{"crux":"bounded task","primary_rule":"SUP-1","capability_boundary":"supported","p_solve":0.1,"unexpected":true}"#.to_string() } else if model == "model/classifier" { r#"{"crux":"bounded task","primary_rule":"SUP-1","capability_boundary":"supported","p_solve":0.9}"#.to_string() + } else if model == "model/vgr-judge" { + "yes".to_string() + } else if model == "model/vgr-reject" { + "no".to_string() } else { "ok".to_string() }; @@ -1444,6 +1453,66 @@ impl Response { } } +#[tokio::test] +async fn vgr_serves_verified_attempts_locally_and_escalates_rejected_ones() -> TestResult { + let upstream = MockUpstream::start().await?; + for (judge, served) in [ + ("model/vgr-judge", "model/vgr-local"), + ("model/vgr-reject", "model/vgr-cloud"), + ] { + let app = build_switchyard_router(load_test_config(&format!( + r#" +schema_version = 1 + +[llm_clients.primary] +format = "openai_chat" +base_url = "{}" + +[targets.local] +id = "model/vgr-local" +llm_client = "primary" + +[targets.cloud] +id = "model/vgr-cloud" +llm_client = "primary" + +[targets.judge] +id = "{judge}" +llm_client = "primary" + +[routes.vgr] +id = "switchyard/vgr" +type = "vgr" +local_target = "local" +cloud_target = "cloud" +judge_target = "judge" +mode = "active" +active_approval = "prospective-validation-and-canary-approved" +task_typing = false +"#, + upstream.base_url + ))?); + let response = send( + &app, + "POST", + "/v1/chat/completions", + Some(json!({ + "model": "switchyard/vgr", + "messages": [{"role": "user", "content": "answer the question"}] + })), + ) + .await?; + assert_eq!(response.status, StatusCode::OK); + assert_eq!( + response.headers.get("x-model-router-selected-model"), + Some(&HeaderValue::from_str(served)?) + ); + let models = send(&app, "GET", "/v1/models", None).await?.json()?; + assert_eq!(models["data"][0]["context_length"], 98_304); + } + Ok(()) +} + fn metric_line<'a>(metrics: &'a str, name: &str, labels: &[(&str, &str)]) -> Option<&'a str> { metrics.lines().find(|line| { line.starts_with(name) diff --git a/docs/routing_algorithms/overview.md b/docs/routing_algorithms/overview.md index 1a1eac7f0..3878d7d3c 100644 --- a/docs/routing_algorithms/overview.md +++ b/docs/routing_algorithms/overview.md @@ -47,6 +47,7 @@ These options remain available when you need a different routing policy. | [Advisor Gate](advisor_gate_routing.md) | Keep one executor model and have a stronger advisor review its plans and completion claims. | `advisor` | | [Sub-Agent-Aware Routing](subagent_routing.md) | Delegated sub-agents should use a separate routing policy from the parent agent. | `passthrough`, `stage_router`, or `composite` with `subagents` | | [Random Routing](random_routing.md) | You need a fixed traffic split for A/B tests, baselines, or cost experiments. | `random` | +| [Verification-Gated Routing](vgr_routing.md) | Serve a local attempt only after bounded verification, and send the request to the capable model otherwise. | `vgr` | | [Fixed Model](#direct-model-routes) | Send every request to one target without a routing decision. | `passthrough` | ### Experimental diff --git a/docs/routing_algorithms/vgr_routing.md b/docs/routing_algorithms/vgr_routing.md new file mode 100644 index 000000000..ee8729e3f --- /dev/null +++ b/docs/routing_algorithms/vgr_routing.md @@ -0,0 +1,51 @@ +# Verification-Gated Routing + +Verification-Gated Routing (VGR) generates a candidate on a local tier, gathers +bounded evidence about that exact candidate, and serves it only when the policy +licenses a local commit. All other outcomes use the cloud tier. + +Tool-calling turns continue on the local tier without being mistaken for a +terminal answer. Missing context, unavailable verifiers, malformed verdicts, +and expired decision deadlines fail closed to cloud. + +## Configure a route + +Declare local, cloud, and optional verifier targets using the normal deployment +schema, then reference their target names: + +```toml +[routes.assistant] +id = "assistant" +type = "vgr" +local_target = "local" +cloud_target = "cloud" +judge_target = "local" +cloud_judge_target = "cloud" +mode = "shadow" +deadline_seconds = 30 +task_typing = true +breaker_threshold = 5 +breaker_cooldown_seconds = 30 +confirmed_recovery_min_clean_tail = 1 +``` + +`judge_target` defaults to `local_target`. `cloud_judge_target` is optional. +The local and cloud targets must resolve to distinct model IDs. When the local +backend reports its live context capacity, VGR republishes it through +`/v1/models`. `confirmed_recovery_min_clean_tail`, when set, lets an agentic +run that recovered from tool errors commit locally once that many trailing tool +results are clean and the cloud judge confirms the evidence. + +## Serving modes + +- `off` skips candidate generation and serves cloud. This is the default. +- `shadow` computes a decision but serves cloud. +- `evaluate` serves the policy decision for isolated evaluation. +- `active` serves the policy decision in production and requires + `active_approval = "prospective-validation-and-canary-approved"`. + +VGR buffers a local candidate before releasing it. A streamed candidate that is +licensed is replayed as a stream; a rejected candidate is not sent to the +client. + +`x-model-router-selected-model` on each response names the tier that served it. diff --git a/mkdocs.yml b/mkdocs.yml index 5cd2a986c..2c7b9dfb5 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -37,6 +37,7 @@ nav: - Composite Routing: routing_algorithms/composite_routing.md - Escalation-Router Routing: routing_algorithms/escalation_router_routing.md - Advisor-Gate Routing: routing_algorithms/advisor_gate_routing.md + - Verification-Gated Routing: routing_algorithms/vgr_routing.md - Operations: - Context-Window Handling: operations/context_window.md - Soak Testing: operations/soak_test.md