feat: Failure cooldown after exhausted transient failures

crates/libsy-llm-client/Cargo.toml

@@ -34,6 +34,7 @@ tracing.workspace = true
 tracing-opentelemetry.workspace = true
 
 [dev-dependencies]
+tokio = { workspace = true, features = ["test-util"] }
 # SDK + in-memory exporter to assert what the observability layer records.
 opentelemetry_sdk = { version = "0.32", features = ["metrics", "testing", "trace"] }
 thiserror.workspace = true

crates/libsy-llm-client/README.md

@@ -73,6 +73,7 @@ fn build_client() -> switchyard_llm_client::Result<TranslatingLlmClient> {
         extra_body: BTreeMap::new(),
         reasoning_effort: None,
         max_retries: 2,
+        failure_cooldown: std::time::Duration::from_secs(5),
         timeout: None,
     };
 
@@ -258,6 +259,12 @@ fn build_multi_format_client(
   transport failures, timeouts, HTTP 408/429, and 5xx responses. Buffered body
   transport failures are retried; streaming body failures are not replayed after
   the response has been returned.
+- `HttpBackendConfig::failure_cooldown` skips completion calls to a backend briefly
+  after an exhausted transient failure. Zero disables it. State is shared per model
+  within the client. Calls resume when the cooldown expires. Deployment TOML defaults
+  `failure_cooldown_ms` to `5000` (5 seconds); set it to `0` to disable cooldown.
+  With `forward_auth`, HTTP 429 keeps its request-local retries and fallback while
+  leaving shared cooldown state unchanged.
 - `HttpBackendConfig::timeout` bounds one complete response, including retries,
   retry delays, and every stream read. Expiry returns `LlmClientError::Timeout`,
   either from the call or from the returned stream, which then ends. `None` leaves

crates/libsy-llm-client/src/backend.rs

@@ -72,6 +72,8 @@ pub struct HttpBackendConfig {
     pub reasoning_effort: Option<String>,
     /// Additional attempts after the initial upstream request.
     pub max_retries: u32,
+    /// Cooldown after an exhausted transient completion failure. Zero disables it.
+    pub failure_cooldown: Duration,
     /// Deadline for one complete response, including retries, retry delays, and stream reads.
     /// `None` leaves the wait unbounded.
     pub timeout: Option<Duration>,
@@ -88,6 +90,7 @@ impl fmt::Debug for HttpBackendConfig {
             .field("omit_body_fields", &self.omit_body_fields)
             .field("reasoning_effort", &self.reasoning_effort)
             .field("max_retries", &self.max_retries)
+            .field("failure_cooldown", &self.failure_cooldown)
             .field("timeout", &self.timeout)
             .finish()
     }
@@ -319,6 +322,11 @@ impl Backend {
         self.config().max_retries
     }
 
+    /// Cooldown after an exhausted transient completion failure.
+    pub fn failure_cooldown(&self) -> Duration {
+        self.config().failure_cooldown
+    }
+
     /// Deadline for all attempts and the complete response; `None` leaves the wait unbounded.
     pub fn timeout(&self) -> Option<Duration> {
         self.config().timeout
@@ -431,6 +439,7 @@ mod tests {
             omit_body_fields: BTreeSet::new(),
             reasoning_effort: None,
             max_retries: 0,
+            failure_cooldown: Duration::ZERO,
             timeout: None,
         }
     }

crates/libsy-llm-client/src/client.rs

@@ -6,6 +6,7 @@
 
 use std::collections::{BTreeMap, BTreeSet, HashMap};
 use std::future::ready;
+use std::sync::atomic::{AtomicU64, Ordering};
 use std::time::{Duration, SystemTime};
 
 use async_trait::async_trait;
@@ -22,6 +23,7 @@ use switchyard_translation::{
     TranslationError, WireFormat, decode_aggregated_response, decode_request, decode_stream,
     encode_aggregated_response_with_extensions, encode_request, encode_stream_with_extensions,
 };
+use tokio::time::Instant;
 use tracing::Instrument;
 
 use crate::backend::{Backend, openai_url};
@@ -126,19 +128,24 @@ pub struct TranslatingLlmClient {
     model_to_config: HashMap<ModelId, ModelConfig>,
     client: reqwest::Client,
     forward_auth_client: reqwest::Client,
+    cooldown_epoch: Instant,
+    // Milliseconds since cooldown_epoch. Each atomic is independent, so relaxed ordering suffices.
+    unavailable_until: HashMap<ModelId, AtomicU64>,
 }
 
 impl TranslatingLlmClient {
     /// Builds a client over the given [`ModelConfig`]s, with a fresh shared HTTP
     /// client and the built-in translation codecs.
     pub fn new(model_configs: &[ModelConfig]) -> Result<Self> {
+        // Pre-populate so concurrent calls update only atomics and keep map access lock-free.
+        let mut unavailable_until = HashMap::new();
         for config in model_configs {
-            config
-                .default_backend
-                .validate_configured_headers(&config.model_name)?;
-            for backend in config.other_backends.iter().flatten() {
+            for backend in std::iter::once(&config.default_backend)
+                .chain(config.other_backends.iter().flatten())
+            {
                 backend.validate_configured_headers(&config.model_name)?;
             }
+            unavailable_until.insert(config.model_name.clone(), AtomicU64::new(0));
         }
         let build_client = |builder: reqwest::ClientBuilder| {
             builder.build().map_err(|error| LlmClientError::Transport {
@@ -159,6 +166,8 @@ impl TranslatingLlmClient {
             model_to_config,
             client,
             forward_auth_client,
+            cooldown_epoch: Instant::now(),
+            unavailable_until,
         })
     }
 
@@ -245,6 +254,20 @@ impl TranslatingLlmClient {
         model: &ModelId,
         endpoint: UpstreamEndpoint,
     ) -> Result<EncodedResponse> {
+        let cooldown = backend.failure_cooldown();
+        let unavailable_until =
+            if matches!(endpoint, UpstreamEndpoint::Completion) && !cooldown.is_zero() {
+                self.unavailable_until.get(model)
+            } else {
+                None
+            };
+        if let Some(until) = unavailable_until
+            && until.load(Ordering::Relaxed) > duration_millis(self.cooldown_epoch.elapsed())
+        {
+            tracing::debug!(model = %model, wire_format = %wire_format, "skipping backend during failure cooldown");
+            return Err(LlmClientError::TemporarilyUnavailable);
+        }
+
         let mut body = encode_request(&llm_request, wire_format)
             .map_err(|error| LlmClientError::RequestEncoding(error.to_string()))?;
         // `encode_request` round-trips a preserved same-format body verbatim,
@@ -276,8 +299,21 @@ impl TranslatingLlmClient {
         let url = endpoint.url(backend);
         record_gen_ai_request(&url, model, streaming);
 
-        self.send_with_retries(&url, backend, &body, metadata, model, streaming)
-            .await
+        let result = self
+            .send_with_retries(&url, backend, &body, metadata, model, streaming)
+            .await;
+        // Forwarded credentials can hit a user's quota while the backend remains healthy.
+        if let Some(until) = unavailable_until
+            && let Err(error) = &result
+            && is_transient_failure(error)
+            && !(backend.is_forwarding_auth()
+                && matches!(error, LlmClientError::UpstreamHttp { status, .. } if *status == StatusCode::TOO_MANY_REQUESTS))
+        {
+            let deadline = duration_millis(self.cooldown_epoch.elapsed().saturating_add(cooldown));
+            until.fetch_max(deadline, Ordering::Relaxed);
+        }
+        // Let cooldown expire naturally; a successful in-flight call may overlap a newer failure.
+        result
     }
 
     // Sends the encoded body, retrying retryable failures within the backend's retry budget.
@@ -733,13 +769,17 @@ fn deadline_error(timeout: Duration) -> LlmClientError {
 
 impl AttemptFailure {
     fn is_retryable(&self) -> bool {
-        match &self.error {
-            LlmClientError::Transport { .. } | LlmClientError::Timeout { .. } => true,
-            LlmClientError::UpstreamHttp { status, .. } => {
-                metrics::is_retryable_http_status(status.as_u16())
-            }
-            _ => false,
+        is_transient_failure(&self.error)
+    }
+}
+
+fn is_transient_failure(error: &LlmClientError) -> bool {
+    match error {
+        LlmClientError::Transport { .. } | LlmClientError::Timeout { .. } => true,
+        LlmClientError::UpstreamHttp { status, .. } => {
+            metrics::is_retryable_http_status(status.as_u16())
         }
+        _ => false,
     }
 }
 
@@ -1307,6 +1347,7 @@ mod tests {
             omit_body_fields: BTreeSet::new(),
             reasoning_effort: None,
             max_retries: 0,
+            failure_cooldown: Duration::ZERO,
             timeout: None,
         }
     }
@@ -2399,6 +2440,51 @@ mod tests {
         Ok(())
     }
 
+    #[tokio::test]
+    async fn rate_limit_cooldown_depends_on_shared_credentials()
+    -> std::result::Result<(), Box<dyn Error + Sync + Send + 'static>> {
+        for forward_auth in [true, false] {
+            let server = MockServer::start().await;
+            Mock::given(method("POST"))
+                .respond_with(ResponseTemplate::new(429).insert_header("retry-after", "0"))
+                .expect(if forward_auth { 6 } else { 3 })
+                .mount(&server)
+                .await;
+            let backend = HttpBackendConfig {
+                forward_auth,
+                failure_cooldown: Duration::from_secs(60),
+                ..config_with_retries(&format!("{}/v1", server.uri()), 2)
+            };
+            let client = TranslatingLlmClient::new(&[ModelConfig::new(
+                "gpt",
+                Backend::OpenAiChat(backend),
+                None,
+            )])?;
+            for credential in ["Bearer first-user", "Bearer second-user"] {
+                let mut headers = http::HeaderMap::new();
+                headers.insert("authorization", http::HeaderValue::from_static(credential));
+                let result = client
+                    .call_rewrite_model(request_with_headers("gpt", headers), None)
+                    .await;
+                if !forward_auth && credential == "Bearer second-user" {
+                    assert!(matches!(
+                        result,
+                        Err(LlmClientError::TemporarilyUnavailable)
+                    ));
+                } else {
+                    assert!(matches!(
+                        result,
+                        Err(LlmClientError::UpstreamHttp {
+                            status: StatusCode::TOO_MANY_REQUESTS,
+                            ..
+                        })
+                    ));
+                }
+            }
+        }
+        Ok(())
+    }
+
     #[tokio::test]
     async fn retry_exhaustion_returns_the_final_upstream_error()
     -> std::result::Result<(), Box<dyn Error + Sync + Send + 'static>> {

crates/libsy-llm-client/src/observability.rs

@@ -201,6 +201,7 @@ fn llm_client_error_type(error: &LlmClientError) -> Cow<'static, str> {
         LlmClientError::ResponseTranslation(_) => Cow::Borrowed("response_translation"),
         LlmClientError::Configuration { .. } => Cow::Borrowed("configuration"),
         LlmClientError::Transport { .. } => Cow::Borrowed("transport"),
+        LlmClientError::TemporarilyUnavailable => Cow::Borrowed("temporarily_unavailable"),
         LlmClientError::Timeout { .. } => Cow::Borrowed("timeout"),
         LlmClientError::ContextWindowExceeded { .. } => Cow::Borrowed("context_window_exceeded"),
         LlmClientError::UpstreamHttp { status, .. } => Cow::Owned(status.as_str().to_owned()),

crates/libsy-llm-client/src/run.rs

@@ -371,7 +371,9 @@ fn fallback_reason(error: &LibsyError) -> Option<RoutingFallbackReason> {
     };
     match source {
         LlmClientError::ContextWindowExceeded { .. } => Some(RoutingFallbackReason::ContextWindow),
-        LlmClientError::Transport { .. } => Some(RoutingFallbackReason::Unavailable),
+        LlmClientError::Transport { .. } | LlmClientError::TemporarilyUnavailable => {
+            Some(RoutingFallbackReason::Unavailable)
+        }
         // A policy denial can be specific to one provider. Preserve its HTTP
         // error, but allow another candidate to serve the request.
         LlmClientError::UpstreamHttp { status, body }
@@ -1213,6 +1215,7 @@ mod tests {
                         omit_body_fields: BTreeSet::new(),
                         reasoning_effort: None,
                         max_retries: 0,
+                        failure_cooldown: std::time::Duration::ZERO,
                         timeout: None,
                     };
                     let backend = if responses {
@@ -1359,6 +1362,7 @@ mod tests {
                     omit_body_fields: BTreeSet::new(),
                     reasoning_effort: None,
                     max_retries: 0,
+                    failure_cooldown: std::time::Duration::ZERO,
                     timeout: None,
                 }),
                 None,
@@ -1991,7 +1995,8 @@ mod tests {
     }
 
     #[tokio::test]
-    async fn retry_budget_is_exhausted_before_falling_through() -> Result<()> {
+    async fn retry_exhaustion_falls_back_and_optional_cooldown_skips_failed_backend() -> Result<()>
+    {
         let server = MockServer::start().await;
         let calls = Arc::new(Mutex::new(Vec::new()));
         let observed_calls = Arc::clone(&calls);
@@ -2021,37 +2026,60 @@ mod tests {
             .mount(&server)
             .await;
 
-        let backend = || {
-            Backend::OpenAiChat(HttpBackendConfig {
-                base_url: format!("{}/v1", server.uri()),
-                api_key: None,
-                forward_auth: false,
-                extra_headers: BTreeMap::new(),
-                extra_body: BTreeMap::new(),
-                omit_body_fields: BTreeSet::new(),
-                reasoning_effort: None,
-                max_retries: 2,
-                timeout: None,
-            })
-        };
-        let client = Arc::new(
+        let client_with_cooldown = |cooldown| {
+            let backend = || {
+                Backend::OpenAiChat(HttpBackendConfig {
+                    base_url: format!("{}/v1", server.uri()),
+                    api_key: None,
+                    forward_auth: false,
+                    extra_headers: BTreeMap::new(),
+                    extra_body: BTreeMap::new(),
+                    omit_body_fields: BTreeSet::new(),
+                    reasoning_effort: None,
+                    max_retries: 2,
+                    failure_cooldown: cooldown,
+                    timeout: None,
+                })
+            };
             TranslatingLlmClient::new(&[
                 ModelConfig::new("weak", backend(), None),
                 ModelConfig::new("strong", backend(), None),
             ])
-            .map_err(|error| LibsyError::external("building test client", error))?,
+            .map(Arc::new)
+            .map_err(|error| LibsyError::external("building test client", error))
+        };
+        let call = async |client: Arc<TranslatingLlmClient>| {
+            run(
+                Arc::new(CandidateAlgorithm {}),
+                ClientRouter::single(client),
+                request(),
+                to_category_map(&["weak", "strong"]),
+                None,
+            )
+            .await?;
+            Ok::<_, LibsyError>(std::mem::take(&mut *calls.lock()))
+        };
+
+        let client = client_with_cooldown(std::time::Duration::ZERO)?;
+        assert_eq!(
+            call(client.clone()).await?,
+            ["weak", "weak", "weak", "strong"]
         );
-        let algorithm = Arc::new(CandidateAlgorithm {});
-        run(
-            algorithm,
-            ClientRouter::single(client),
-            request(),
-            to_category_map(&["weak", "strong"]),
-            None,
-        )
-        .await?;
+        assert_eq!(call(client).await?, ["weak", "weak", "weak", "strong"]);
+
+        let cooldown = std::time::Duration::from_secs(60);
+        let client = client_with_cooldown(cooldown)?;
+        assert_eq!(
+            call(client.clone()).await?,
+            ["weak", "weak", "weak", "strong"]
+        );
+        assert_eq!(call(client.clone()).await?, ["strong"]);
+
+        tokio::time::pause();
+        tokio::time::advance(cooldown).await;
+        tokio::time::resume();
 
-        assert_eq!(&*calls.lock(), &["weak", "weak", "weak", "strong"]);
+        assert_eq!(call(client).await?, ["weak", "weak", "weak", "strong"]);
         Ok(())
     }
 
@@ -2130,6 +2158,7 @@ mod tests {
                 omit_body_fields: BTreeSet::new(),
                 reasoning_effort: None,
                 max_retries: 0,
+                failure_cooldown: std::time::Duration::ZERO,
                 timeout: None,
             })
         };

crates/protocol/src/client.rs

@@ -63,6 +63,10 @@ pub enum LlmClientError {
     #[error("Responses state ID is already recorded for another model; its owner was not changed")]
     ResponseStateConflict,
 
+    /// The selected backend is temporarily unavailable. Another candidate may serve the request.
+    #[error("backend temporarily unavailable")]
+    TemporarilyUnavailable,
+
     /// The upstream could not be reached or the request could not be sent.
     #[error("upstream transport error: {source}")]
     Transport {