diff --git a/crates/switchyard-server/tests/server.rs b/crates/switchyard-server/tests/server.rs index cd21d3c91..3612ce9da 100644 --- a/crates/switchyard-server/tests/server.rs +++ b/crates/switchyard-server/tests/server.rs @@ -342,6 +342,23 @@ async fn upstream_chat( .into_response(); } + // A generation that did not complete (`abort` from vLLM or SGLang, `error` from OpenRouter, + // `repetition` from vLLM). + if let Some(reason @ ("abort" | "error" | "repetition")) = model.strip_prefix("model/finish-") { + return Json(json!({ + "id": "chatcmpl-finish", + "object": "chat.completion", + "model": model, + "choices": [{ + "index": 0, + "message": {"role": "assistant", "content": "partial"}, + "finish_reason": reason + }], + "usage": {"prompt_tokens": 3, "completion_tokens": 1, "total_tokens": 4} + })) + .into_response(); + } + // Buffered tool call, the non-streaming counterpart of the branch above. if prompt == "mcp-tool-call" { let called = body["tool_choice"]["function"]["name"] @@ -1134,6 +1151,85 @@ async fn failed_responses_return_errors_and_try_fallback_across_endpoints() -> T Ok(()) } +// A Chat backend that ends a generation with `abort`, `error` or `repetition` did +// not complete it: every client path gets a 502, and a fallback route moves on. +#[tokio::test] +async fn chat_failed_finish_reasons_return_errors_and_try_fallback() -> TestResult { + let upstream = MockUpstream::start().await?; + let requests = [ + ( + "/v1/chat/completions", + json!({ + "model": ROUTE_MODEL, "messages": [{"role": "user", "content": "hello"}] + }), + ), + ( + "/v1/messages", + json!({ + "model": ROUTE_MODEL, "max_tokens": 16, + "messages": [{"role": "user", "content": "hello"}] + }), + ), + ( + "/v1/responses", + json!({"model": ROUTE_MODEL, "input": "hello"}), + ), + ]; + for reason in ["abort", "error", "repetition"] { + let model = format!("model/finish-{reason}"); + let message = format!("provider finished with finish_reason \"{reason}\""); + for fallback in [false, true] { + let route = if fallback { + "type = \"random\"\ntargets = [\"first\", \"second\"]\nweights = [1000, 1]\nseed = 17" + } else { + "type = \"passthrough\"\ntarget = \"first\"" + }; + let app = build_switchyard_router(load_test_config(&format!( + r#" +schema_version = 1 +[llm_clients.mock] +format = "openai_chat" +base_url = "{base_url}" +max_retries = 0 +[targets] +first = {{ id = "{model}", llm_client = "mock" }} +second = {{ id = "model/fallback", llm_client = "mock" }} +[routes.chat] +id = "{ROUTE_MODEL}" +{route} +"#, + base_url = upstream.base_url, + ))?); + for (path, body) in &requests { + let previous_calls = upstream.models().await.len(); + let response = send(&app, "POST", path, Some(body.clone())).await?; + let calls = upstream.models().await[previous_calls..].to_vec(); + if fallback { + assert_eq!(response.status, StatusCode::OK, "{model}: {path}"); + assert_eq!( + response.json()?["model"], + "model/fallback", + "{model}: {path}" + ); + assert_eq!(calls, [model.as_str(), "model/fallback"], "{model}: {path}"); + continue; + } + assert_eq!(response.status, StatusCode::BAD_GATEWAY, "{model}: {path}"); + let expected = if *path == "/v1/messages" { + json!({"type": "error", "error": {"type": "api_error", "message": message}}) + } else { + json!({"error": { + "type": "upstream_error", "code": "upstream_error", "message": message + }}) + }; + assert_eq!(response.json()?, expected, "{model}: {path}"); + assert_eq!(calls, [model.as_str()], "{model}: {path}"); + } + } + } + Ok(()) +} + #[tokio::test] async fn stats_reset_returns_confirmation_and_clears_all_stats() -> TestResult { let (_upstream, app) = test_app(&[(ROUTE_MODEL, &["model/a"])]).await?; diff --git a/crates/switchyard-translation/src/codecs/openai_chat/buffered.rs b/crates/switchyard-translation/src/codecs/openai_chat/buffered.rs index d90423518..c522e3aef 100644 --- a/crates/switchyard-translation/src/codecs/openai_chat/buffered.rs +++ b/crates/switchyard-translation/src/codecs/openai_chat/buffered.rs @@ -269,6 +269,18 @@ impl FormatCodec for OpenAiChatCodec { _policy: &TranslationPolicy, ) -> Result { let object = object(body, "$")?; + // `abort`, `error` and `repetition` mean the generation did not complete. + if let Some(reason @ ("abort" | "error" | "repetition")) = object + .get("choices") + .and_then(Value::as_array) + .and_then(|choices| choices.first()) + .and_then(|choice| choice.get("finish_reason")) + .and_then(Value::as_str) + { + return Err(TranslationError::UpstreamFailure { + error: json!({ "message": format!("provider finished with finish_reason \"{reason}\"") }), + }); + } let mut response = AggLlmResponse { id: object .get("id") diff --git a/crates/switchyard-translation/tests/response_translation.rs b/crates/switchyard-translation/tests/response_translation.rs index 4280fa3f1..20e6dfb30 100644 --- a/crates/switchyard-translation/tests/response_translation.rs +++ b/crates/switchyard-translation/tests/response_translation.rs @@ -1057,6 +1057,31 @@ fn failed_responses_return_upstream_failure_with_provider_message() -> TestResul Ok(()) } +// Verifies Chat `abort`, `error` and `repetition` finish reasons fail the turn. +#[test] +fn failed_finish_reasons_return_upstream_failure() -> TestResult { + let engine = TranslationEngine::default(); + for reason in ["abort", "error", "repetition"] { + let body = json!({ + "id": "chatcmpl-test", "object": "chat.completion", "model": "gpt-4o", + "choices": [{"index": 0, "finish_reason": reason, + "message": {"role": "assistant", "content": "partial"}}] + }); + let error = engine + .translate_response( + WireFormat::OpenAiChat, + WireFormat::AnthropicMessages, + &body, + &TranslationPolicy::default(), + ) + .err() + .ok_or_else(|| format!("accepted finish_reason {reason}"))?; + assert_eq!(error.kind(), "UpstreamFailure"); + assert!(error.to_string().contains(reason), "{error}"); + } + Ok(()) +} + // Verifies a moderation stop stays distinguishable from a normal turn in both // directions, and that a named refusal category survives re-encoding. #[test]