diff --git a/crates/tw-api/msg-codes.txt b/crates/tw-api/msg-codes.txt index 31de383a..9a24df93 100644 --- a/crates/tw-api/msg-codes.txt +++ b/crates/tw-api/msg-codes.txt @@ -90,6 +90,7 @@ config.manual_model_duplicate config.manual_model_padded config.manual_model_too_long config.manual_model_wildcard +config.min_speed_range config.model_spec_blank_model config.model_spec_nothing_set config.model_spec_wildcard @@ -458,6 +459,7 @@ gw.upstream.stream_error gw.upstream.stream_exception gw.upstream.stream_opening_error gw.upstream.timeout +gw.upstream.too_slow gw.upstream.unreachable gw.upstream.window_full gw.wait.mixed @@ -654,6 +656,7 @@ t.broke test t.limited test t.plugin_threw test t.prefs test +t.slow test t.upstream test t.x test yaml.anchor_or_alias diff --git a/crates/tw-api/src/lib.rs b/crates/tw-api/src/lib.rs index efbb33ec..e0bb0f04 100644 --- a/crates/tw-api/src/lib.rs +++ b/crates/tw-api/src/lib.rs @@ -1131,7 +1131,18 @@ pub const MSG_CODES: &str = include_str!("../msg-codes.txt"); /// `security.unknown_event`。新端点 `GET /security/counts`([`SecurityRuleHitsQuery`] → /// [`SecurityRuleHits`] 的列表):一项防护从某个时刻起每条规则命中了几次,各条记录的 `count` /// 加起来,规则表的组头用 -pub const CONTROL_API_VERSION: u32 = 57; +/// +/// **58 起上游可以设最低生成速度**(配置的 `providers[].min_speed`):[`ProviderView`] 多了 +/// `min_speed`([`MinSpeedView`],不管是空),[`ProviderInput`] 多了 `min_speed` +/// ([`MinSpeedInput`],不给就是不管)。流式回答从第一个 token 起持续低于它(最近 `window_secs` +/// 秒的平均速度,中途 5 秒以上的静默不算),回答以一条 503 的错误结束(Anthropic 的 +/// `overloaded_error`、Responses 的 `response.failed` / `server_error`、Gemini 的 `UNAVAILABLE`), +/// 请求失败,来源 `upstream`,码 `gw.upstream.too_slow`(`upstream`、`rate`、`min`、`secs`),这家 +/// 记一次失败。[`Event::RequestFailed`] 多了 `tokens_per_sec`:只有这种失败带着,是结束那一刻的 +/// 速度,请求记录里照记。新码:`gw.upstream.too_slow`、`config.min_speed_range`(`upstream`、 +/// `field`、`value`、`min`、`max`);一键修复修得了它:速度越界删掉 `min_speed`,判定时长越界改回 +/// 20。请求库 SCHEMA 不变 +pub const CONTROL_API_VERSION: u32 = 58; /// `/live` 的速度看最近多久(见 [`LiveView::tokens_per_sec`]),毫秒 pub const LIVE_RATE_WINDOW_MS: u64 = 3_000; @@ -1435,6 +1446,11 @@ pub enum Event { /// 失败之前上游在回答里写的模型名(见 `RequestFinished::answered_model`) #[serde(default, skip_serializing_if = "Option::is_none")] answered_model: Option, + /// 生成速度,token/秒。**失败的请求一般没有**,只有因为生成太慢被结束的 + /// (`gw.upstream.too_slow`,见 [`MinSpeedView`])有:那是结束那一刻判定窗口里的速度, + /// 估出来的,和 [`RunningView::tokens_per_sec`] 同一个估法 + #[serde(default, skip_serializing_if = "Option::is_none")] + tokens_per_sec: Option, }, /// 客户端没等到响应结束就走了(Claude Code 里按一下 Esc)。 /// @@ -2711,6 +2727,9 @@ pub struct ProviderView { /// 同时最多发给这家几个请求。不限是空 #[serde(default, skip_serializing_if = "Option::is_none")] pub max_concurrent: Option, + /// 最低生成速度(配置里的 `min_speed`)。不管是空 + #[serde(default, skip_serializing_if = "Option::is_none")] + pub min_speed: Option, /// 用量上限,按配置里的顺序,各带此刻用了多少(发出去还没结束的按估算算)。没设的是空的 pub limits: Vec, /// 配置里的 `balance:`,原样:编辑对话框保存时交回去([`ProviderInput::balance`]) @@ -3691,6 +3710,9 @@ pub struct ProviderInput { /// 同时最多发给这家几个请求,1 到 1000。不给就是不限 #[serde(default, skip_serializing_if = "Option::is_none")] pub max_concurrent: Option, + /// 最低生成速度。不给就是不管 + #[serde(default, skip_serializing_if = "Option::is_none")] + pub min_speed: Option, /// 用量上限,整份替换。不给 = 一条都没有 #[serde(default)] pub limits: Vec, @@ -3702,6 +3724,31 @@ pub struct ProviderInput { pub balance: Option, } +/// 一家上游的最低生成速度(配置的 `providers[].min_speed`)。 +/// +/// 流式回答从第一个 token 起,每秒看一次最近 `window_secs` 秒的平均生成速度(和 +/// [`RunningView::tokens_per_sec`] 同一个估法;中途连续 5 秒以上没有内容的那一段不算),低于 +/// `tokens_per_sec` 就结束回答:客户端收到一条 503 的错误(可以重试),请求失败,码是 +/// `gw.upstream.too_slow`,这家记一次失败。非流式的回答、WebSocket 上的不管 +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +#[cfg_attr(feature = "ts", derive(ts_rs::TS))] +pub struct MinSpeedView { + /// 每秒至少多少 token + pub tokens_per_sec: u32, + /// 按最近多少秒的平均速度判断。配置里没写的是 20 + pub window_secs: u32, +} + +/// 保存上游时交的最低生成速度([`ProviderInput::min_speed`])。`tokens_per_sec` 是 1 到 1000, +/// `window_secs` 是 5 到 120(界面显式填好默认的 20);越界的保存被拒,码是 +/// `config.min_speed_range` +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +#[cfg_attr(feature = "ts", derive(ts_rs::TS))] +pub struct MinSpeedInput { + pub tokens_per_sec: u32, + pub window_secs: u32, +} + /// 按接口地址自动识别的结果。给编辑中、还没保存的上游显示「自动识别」 /// 会选成什么。 #[derive(Debug, Clone, Serialize, Deserialize)] @@ -7691,6 +7738,7 @@ mod tests { answered_model: None, }, Event::RequestFailed { + tokens_per_sec: None, id: 7, model: String::new(), source: FailureSource::Upstream, @@ -7842,6 +7890,7 @@ mod tests { )); let silent = Event::RequestFailed { + tokens_per_sec: None, id: 2, model: String::new(), source: FailureSource::Upstream, diff --git a/crates/tw-config/src/lib.rs b/crates/tw-config/src/lib.rs index f6a0b02f..1194db73 100644 --- a/crates/tw-config/src/lib.rs +++ b/crates/tw-config/src/lib.rs @@ -200,6 +200,7 @@ impl Default for Provider { pricing: None, model_specs: std::collections::BTreeMap::new(), max_concurrent: None, + min_speed: None, limits: Vec::new(), disabled: false, balance: BalanceSetting::Auto, @@ -850,6 +851,10 @@ pub struct Provider { /// 这家的对话等它空出来,别的请求换下一家(见 `tw_gateway::slots`) #[serde(default, skip_serializing_if = "Option::is_none")] pub max_concurrent: Option, + /// 最低生成速度:流式回答在第一个 token 之后持续低于它,就以一条可重试的错误结束回答(见 + /// `tw_gateway::slow`)。不写就是不管 + #[serde(default, skip_serializing_if = "Option::is_none")] + pub min_speed: Option, /// 用量上限:每 N 分钟、每 N 小时、每天、每周、每月最多发给这家多少个请求、多少 token、 /// 花多少钱。**每一条都要过**,不写就不限。写法和密钥的一样([`limits`])。 /// @@ -958,6 +963,39 @@ impl From for BalanceSetting { /// 一家上游的并发上限最多写多少 pub const MAX_PROVIDER_CONCURRENCY: u32 = 1000; +/// 一家上游的最低生成速度(`providers[].min_speed`)。 +/// +/// **只管流式回答的生成阶段**:从第一个 token 起,每秒看一次最近 `window_secs` 秒的平均速度, +/// 低于 `tokens_per_sec` 就结束这个回答,客户端收到一条可以重试的错误(503)。速度和流量页、 +/// 菜单栏上的实时速度是同一个估法;中途连续几秒没有内容的那一段不算进去,由无响应超时管。 +/// +/// ```yaml +/// providers: +/// - name: local-vllm +/// min_speed: { tokens_per_sec: 15, window_secs: 20 } +/// ``` +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct MinSpeed { + /// 每秒至少多少 token,1 到 [`MIN_SPEED_MAX_RATE`] + pub tokens_per_sec: u32, + /// 按最近多少秒的平均速度判断,[`MIN_SPEED_WINDOW_SECS`] 之内。不写是 + /// [`DEFAULT_MIN_SPEED_WINDOW_SECS`];**写回配置时照写**,等于默认值也写,一眼看得出按多久判断 + #[serde(default = "d_min_speed_window")] + pub window_secs: u32, +} + +/// 最低生成速度最多写多少 token/秒 +pub const MIN_SPEED_MAX_RATE: u32 = 1000; +/// 最低生成速度的判定时长能写的范围,秒 +pub const MIN_SPEED_WINDOW_SECS: (u32, u32) = (5, 120); +/// 最低生成速度的判定时长不写时是多少秒 +pub const DEFAULT_MIN_SPEED_WINDOW_SECS: u32 = 20; + +fn d_min_speed_window() -> u32 { + DEFAULT_MIN_SPEED_WINDOW_SECS +} + /// 手动添加的一个模型 ID 最多多少个字符。最长的真名字是 Bedrock 应用推理配置的 ARN, /// 一百来个字符 pub const MANUAL_MODEL_MAX: usize = 256; diff --git a/crates/tw-config/src/repair.rs b/crates/tw-config/src/repair.rs index d5667819..1c79d48d 100644 --- a/crates/tw-config/src/repair.rs +++ b/crates/tw-config/src/repair.rs @@ -5,6 +5,10 @@ //! —— 而它们的修法不需要猜用户想写什么。语法错(要看标点)、缺字段(没有默认值可回)、语义错 //! (整份配置的事)都不碰。 //! +//! 语义错里只修一种:上游的最低生成速度越界(`config.min_speed_range`)。速度越界删掉整项 +//! (不管),判定时长越界改回默认的 20 秒 —— 改值而不是删键,因为它常写在行内映射里 +//! (`min_speed: { tokens_per_sec: 15, window_secs: 3 }`),行内的键删不了。 +//! //! **修完整份配置读得进来才算数**:删掉一处之后冒出别的错(比如那个字段没有默认值,删了就 //! 成了缺字段),整个修复就不给。半修好的配置还是读不进来,给了只会让人以为好了。 //! @@ -61,16 +65,25 @@ pub fn repair(text: &str) -> Option { } Err(r) => r, }; - let (kind, field) = fixable(&r)?; + let Action { + kind, + field, + set, + shown, + } = fixable(&r, &out)?; let path = steps(&field)?; - // 行号和原值都按**修之前的原文**找:前面删过行,`r.line` 已经对不上了 - let before = tw_yaml::find(text, &path).ok(); + // 行号和原值都按**修之前的原文**找:前面删过行,`r.line` 已经对不上了。删掉整项的, + // 原值给人看的是越界的那一个(`shown`) + let before = tw_yaml::find(text, &steps(shown.as_deref().unwrap_or(&field))?).ok(); let line = before .as_ref() .map(|f| text[..f.bytes.start].matches('\n').count() + 1) .or(r.line); let value = before.map(|f| masked(&path, &f.value)); - out = tw_yaml::remove_key(&out, &path).ok()?; + out = match set { + Some(v) => tw_yaml::set(&out, &path, &v).ok()?, + None => tw_yaml::remove_key(&out, &path).ok()?, + }; found.push(Fix { kind, field, @@ -97,14 +110,57 @@ fn masked(path: &[Step], v: &str) -> String { } } -/// 这一处能不能修,能修的话是哪一种、哪个字段 -fn fixable(r: &Rejected) -> Option<(FixKind, String)> { +/// 一处怎么修 +struct Action { + kind: FixKind, + /// 改的是哪个字段 + field: String, + /// 改成什么。`None` 是删掉 + set: Option, + /// 原值和行号从哪个字段读。`None` 就是 `field` + shown: Option, +} + +/// 这一处能不能修,能修的话怎么修。`text` 是此刻的原文 +fn fixable(r: &Rejected, text: &str) -> Option { let kind = match r.message.code.as_str() { "config.unknown_variant" => FixKind::UnknownValue, "config.unknown_field" => FixKind::UnknownField, + "config.min_speed_range" => return min_speed(r, text), _ => return None, }; - Some((kind, r.message.args.get("field")?.clone())) + Some(Action { + kind, + field: r.message.args.get("field")?.clone(), + set: None, + shown: None, + }) +} + +/// 最低生成速度越界:速度不对删掉整项,判定时长不对改回默认值。上游按名字找它在第几个 +/// (到了校验这一步,原文是读得进来的) +fn min_speed(r: &Rejected, text: &str) -> Option { + let cfg: Config = serde_yaml_ng::from_str(text).ok()?; + let upstream = r.message.args.get("upstream")?; + let i = cfg.providers.iter().position(|p| &p.name == upstream)?; + let at = format!("providers[{i}].min_speed"); + Some(match r.message.args.get("field")?.as_str() { + "tokens_per_sec" => Action { + kind: FixKind::UnknownValue, + shown: Some(format!("{at}.tokens_per_sec")), + field: at, + set: None, + }, + "window_secs" => Action { + kind: FixKind::UnknownValue, + field: format!("{at}.window_secs"), + set: Some(tw_yaml::Scalar::Int(i64::from( + crate::DEFAULT_MIN_SPEED_WINDOW_SECS, + ))), + shown: None, + }, + _ => return None, + }) } /// 取值改回默认值的那几处,查出默认值是什么,给人看 @@ -355,6 +411,59 @@ mod tests { assert!(!shown.contains("abcdefghijklmnopqrstuvwxyz"), "{shown}"); } + /// 最低生成速度越界:速度不对删掉整项,判定时长不对改回 20(行内映射里也改得了) + #[test] + fn a_minimum_speed_out_of_range_is_dropped_or_its_window_reset() { + let ups = |min_speed: &str| { + format!( + "{GOOD}providers:\n - name: a\n base_url: https://x\n - name: b\n base_url: https://y\n{min_speed}" + ) + }; + let bad = ups(" min_speed: { tokens_per_sec: 0, window_secs: 20 }\n"); + let r = repair(&bad).expect("该修得了"); + assert_eq!( + r.fixes, + vec![Fix { + kind: FixKind::UnknownValue, + field: "providers[1].min_speed".into(), + line: Some(13), + value: Some("0".into()), + now: None, + }] + ); + assert_eq!(r.text, ups("")); + assert!(try_parse(&r.text).unwrap().providers[1].min_speed.is_none()); + + let bad = ups(" min_speed: { tokens_per_sec: 15, window_secs: 3 }\n"); + let r = repair(&bad).expect("该修得了"); + assert_eq!( + r.fixes, + vec![Fix { + kind: FixKind::UnknownValue, + field: "providers[1].min_speed.window_secs".into(), + line: Some(13), + value: Some("3".into()), + now: Some("20".into()), + }] + ); + assert_eq!( + r.text, + ups(" min_speed: { tokens_per_sec: 15, window_secs: 20 }\n") + ); + + // 块式的照样改 + let bad = ups(" min_speed:\n tokens_per_sec: 15\n window_secs: 999\n"); + let r = repair(&bad).expect("该修得了"); + assert_eq!(r.fixes[0].line, Some(15)); + assert_eq!( + try_parse(&r.text).unwrap().providers[1].min_speed, + Some(crate::MinSpeed { + tokens_per_sec: 15, + window_secs: 20 + }) + ); + } + #[test] fn paths_with_indices_are_understood_and_odd_ones_are_left_alone() { assert_eq!( diff --git a/crates/tw-config/src/validate.rs b/crates/tw-config/src/validate.rs index 33721d6a..8101efca 100644 --- a/crates/tw-config/src/validate.rs +++ b/crates/tw-config/src/validate.rs @@ -43,6 +43,15 @@ pub enum ValidationError { #[error("{}", self.msg())] ProviderConcurrency { name: String, value: u32 }, #[error("{}", self.msg())] + MinSpeedRange { + name: String, + /// `tokens_per_sec` / `window_secs` + field: &'static str, + value: u32, + min: u32, + max: u32, + }, + #[error("{}", self.msg())] Routing(#[from] tw_engine::RouteError), #[error("{}", self.msg())] NameCollision(String), @@ -298,6 +307,18 @@ impl ValidationError { "upstream `{upstream}` has max_concurrent: {value}; it has to be between 1 and \ {max}. Leave max_concurrent out for no limit" ), + MinSpeedRange { + name, + field, + value, + min, + max, + } => msg!( + "config.min_speed_range", upstream = name, field = field, value = value, + min = min, max = max => + "upstream `{upstream}` has min_speed.{field}: {value}; it has to be between \ + {min} and {max}" + ), // 路由那几句本身就说清了是哪条规则、哪个组,前面不用再垫一句 Routing(e) => e.msg(), NameCollision(name) => msg!( @@ -676,6 +697,33 @@ pub fn validate(cfg: &Config) -> Result<(), ValidationError> { value: n, }); } + // 最低生成速度:0 等于不管(那是不写),上千的话正常的回答也会被掐断;判定时长太短 + // 是拿几秒的波动下结论,太长等于不管。**速度先查**:一键修复按它删掉整项,判定时长 + // 跟着没了 + if let Some(m) = p.min_speed { + let (lo, hi) = crate::MIN_SPEED_WINDOW_SECS; + let out = if !(1..=crate::MIN_SPEED_MAX_RATE).contains(&m.tokens_per_sec) { + Some(( + "tokens_per_sec", + m.tokens_per_sec, + 1, + crate::MIN_SPEED_MAX_RATE, + )) + } else if !(lo..=hi).contains(&m.window_secs) { + Some(("window_secs", m.window_secs, lo, hi)) + } else { + None + }; + if let Some((field, value, min, max)) = out { + return Err(ValidationError::MinSpeedRange { + name: p.name.clone(), + field, + value, + min, + max, + }); + } + } // **空范围不是「全部」,也不是一个合理的「停用」。**两种读法各有 // 人会当真,而停用有自己的开关 if let Some(only) = &p.models_only { @@ -1505,6 +1553,61 @@ mod tests { } } + #[test] + fn a_minimum_speed_is_1_to_1000_tokens_per_second_over_5_to_120_seconds() { + let with = |m: Option<(u32, u32)>| { + let mut prov = p("relay", "https://relay.example"); + prov.min_speed = m.map(|(tokens_per_sec, window_secs)| crate::MinSpeed { + tokens_per_sec, + window_secs, + }); + validate(&cfg(vec![c("a", "tw-a")], vec![prov])) + }; + assert!(with(None).is_ok(), "不写是不管"); + for ok in [(1, 5), (1000, 120), (15, 20)] { + assert!(with(Some(ok)).is_ok(), "{ok:?}"); + } + for (bad, field) in [ + ((0, 20), "tokens_per_sec"), + ((1001, 20), "tokens_per_sec"), + ((15, 4), "window_secs"), + ((15, 121), "window_secs"), + // 两样都不对:先说速度(一键修复按它删掉整项) + ((0, 0), "tokens_per_sec"), + ] { + let e = with(Some(bad)).unwrap_err(); + let m = e.msg(); + assert_eq!(m.code, "config.min_speed_range", "{e}"); + assert_eq!(m.arg("upstream"), "relay"); + assert_eq!(m.arg("field"), field, "{bad:?}"); + assert!(e.to_string().contains(&format!("min_speed.{field}")), "{e}"); + } + } + + #[test] + fn a_minimum_speed_reads_with_a_default_window_and_writes_it_back_explicitly() { + let prov: crate::Provider = serde_yaml_ng::from_str( + "name: local\nbase_url: http://127.0.0.1:8000\nmin_speed: { tokens_per_sec: 15 }\n", + ) + .unwrap(); + assert_eq!( + prov.min_speed, + Some(crate::MinSpeed { + tokens_per_sec: 15, + window_secs: crate::DEFAULT_MIN_SPEED_WINDOW_SECS + }) + ); + let out = serde_yaml_ng::to_string(&prov).unwrap(); + assert!(out.contains("window_secs: 20"), "{out}"); + // 拼错的字段照样报出来 + assert!( + serde_yaml_ng::from_str::( + "name: local\nbase_url: http://x\nmin_speed: { tokens_per_sec: 15, window: 20 }\n", + ) + .is_err() + ); + } + #[test] fn an_empty_model_scope_is_refused_and_points_at_disabling_instead() { let mut prov = p("relay", "https://relay.example"); @@ -2329,6 +2432,13 @@ mod msg_codes { name: "a".into(), value: 0, }, + MinSpeedRange { + name: "a".into(), + field: "window_secs", + value: 1, + min: 5, + max: 120, + }, NameCollision("a".into()), BadCidr { entry: "x".into() }, UnknownPriceSheet { diff --git a/crates/tw-config/tests/manual/schema.rs b/crates/tw-config/tests/manual/schema.rs index 164e2dba..5f52abf0 100644 --- a/crates/tw-config/tests/manual/schema.rs +++ b/crates/tw-config/tests/manual/schema.rs @@ -731,6 +731,15 @@ pub fn sections() -> Vec
{ "同时发给这家的请求最多几个,取值 1 到 1000。满了的时候,留在这家的对话等空位,别的请求换下一家;等多久见 `failover.wait_secs`。不写:不限。", ), ), + row( + "min_speed", + Kind::Obj("providers[].min_speed"), + Def::Unset, + t( + "End a streamed answer that generates too slowly: from the first token on, once a second, when the average speed over the last `window_secs` is below `tokens_per_sec`, the answer ends with an error clients retry (503, `overloaded_error` for Anthropic clients), and the upstream counts one failure toward `failover.failures_to_pause`. Pauses of 5 seconds or more partway through are left out of the average; the idle timeout deals with them. Answers that are not streamed are not judged. Unset: never.", + "流式回答生成得太慢时结束它:从第一个 token 起每秒看一次,最近 `window_secs` 秒的平均速度低于 `tokens_per_sec` 时,回答以一条客户端会重试的错误结束(503,Anthropic 格式的客户端收到 `overloaded_error`),这家记一次失败,计入 `failover.failures_to_pause`。中途 5 秒以上的停顿不算进平均速度,由无响应超时处理。不是流式的回答不管。不写:不管。", + ), + ), row( "limits", Kind::Objs("providers[].limits[]"), @@ -812,6 +821,30 @@ pub fn sections() -> Vec
{ ), ], }, + Section { + path: "providers[].min_speed", + ty: checked!(MinSpeed, "{tokens_per_sec: 15}"), + rows: vec![ + row( + "tokens_per_sec", + Kind::Int, + Def::Required, + t( + "The lowest acceptable speed, in tokens per second, counted the way the live speed in the app is. From 1 to 1000.", + "最低生成速度,单位 token/秒,和应用里显示的实时速度同一个算法。取值 1 到 1000。", + ), + ), + row( + "window_secs", + Kind::Int, + Def::Is("20"), + t( + "How many seconds the average is taken over. The first one is taken this long after the first token. From 5 to 120.", + "按最近多少秒的平均速度判断。第一次判断在第一个 token 之后这么久。取值 5 到 120。", + ), + ), + ], + }, Section { path: "providers[].limits[]", ty: checked!(UsageLimit, "{per: day}"), diff --git a/crates/tw-control/src/lib.rs b/crates/tw-control/src/lib.rs index b5a50a71..f0801ccc 100644 --- a/crates/tw-control/src/lib.rs +++ b/crates/tw-control/src/lib.rs @@ -530,6 +530,10 @@ fn provider_view( .collect(), pricing: p.pricing.clone(), max_concurrent: p.max_concurrent, + min_speed: p.min_speed.map(|m| tw_api::MinSpeedView { + tokens_per_sec: m.tokens_per_sec, + window_secs: m.window_secs, + }), limits: s.gateway.upstream_limits.view(&p.name, &p.limits), balance_setting: p.balance.into(), balance: s.gateway.balance_of(p), diff --git a/crates/tw-control/src/resources.rs b/crates/tw-control/src/resources.rs index 7430a5a8..efebc619 100644 --- a/crates/tw-control/src/resources.rs +++ b/crates/tw-control/src/resources.rs @@ -561,6 +561,11 @@ fn to_provider( // 沿用原来的,改名时跟着这一项走 model_specs: existing.map(|e| e.model_specs.clone()).unwrap_or_default(), max_concurrent: input.max_concurrent, + // 越界的由配置校验说(`config.min_speed_range`),和手写的同一套 + min_speed: input.min_speed.map(|m| tw_config::MinSpeed { + tokens_per_sec: m.tokens_per_sec, + window_secs: m.window_secs, + }), // 写得对不对由配置校验说,和手写的同一套 limits: input.limits.iter().map(crate::keys::limit).collect(), disabled: input.disabled, diff --git a/crates/tw-control/tests/handover.rs b/crates/tw-control/tests/handover.rs index 73501c2a..a0e71a58 100644 --- a/crates/tw-control/tests/handover.rs +++ b/crates/tw-control/tests/handover.rs @@ -146,7 +146,7 @@ async fn status_reports_the_handover_phase_and_the_draining_count() { let st = status(&s).await; assert_eq!(st.handover, tw_api::HandoverState::Draining); assert_eq!(st.draining, 3); - assert_eq!(st.api_version, 57); + assert_eq!(st.api_version, 58); } /// arm 之后(以及接替者正在接管时)不开始新的账号登录:回调端口不交接。(平常、排空中的 diff --git a/crates/tw-control/tests/key_limits.rs b/crates/tw-control/tests/key_limits.rs index 056c5cf3..bc0a6ade 100644 --- a/crates/tw-control/tests/key_limits.rs +++ b/crates/tw-control/tests/key_limits.rs @@ -426,6 +426,7 @@ fn refused( } } evs.push(Event::RequestFailed { + tokens_per_sec: None, id, model: String::new(), source, diff --git a/crates/tw-control/tests/resources.rs b/crates/tw-control/tests/resources.rs index 8b078171..b8fc68db 100644 --- a/crates/tw-control/tests/resources.rs +++ b/crates/tw-control/tests/resources.rs @@ -412,6 +412,62 @@ async fn an_upstreams_concurrency_limit_is_saved_shown_and_checked() { ); } +/// 最低生成速度:存进去(判定时长照写)、概览原样给回来;越界的存不进去;去掉就是不管 +#[tokio::test] +async fn an_upstreams_minimum_speed_is_saved_shown_and_checked() { + let b = bed(BASE); + let save = |m: serde_json::Value| { + let app = b.app.clone(); + async move { + call( + &app, + "PUT", + "/providers/官方", + serde_json::json!({ "provider": official(serde_json::json!({ "min_speed": m })) }), + ) + .await + } + }; + let (st, body) = save(serde_json::json!({"tokens_per_sec": 15, "window_secs": 20})).await; + assert_eq!(st, StatusCode::OK, "{body}"); + assert_eq!( + b.parsed().providers[0].min_speed, + Some(tw_config::MinSpeed { + tokens_per_sec: 15, + window_secs: 20 + }) + ); + assert!(b.file().contains("window_secs: 20"), "{}", b.file()); + let (_, body) = call(&b.app, "GET", "/overview", serde_json::Value::Null).await; + assert_eq!( + json(&body)["providers"][0]["min_speed"], + serde_json::json!({"tokens_per_sec": 15, "window_secs": 20}), + "{body}" + ); + + let before = b.file(); + for bad in [ + serde_json::json!({"tokens_per_sec": 0, "window_secs": 20}), + serde_json::json!({"tokens_per_sec": 1001, "window_secs": 20}), + serde_json::json!({"tokens_per_sec": 15, "window_secs": 4}), + serde_json::json!({"tokens_per_sec": 15, "window_secs": 121}), + ] { + let (st, body) = save(bad.clone()).await; + assert_eq!(st, StatusCode::BAD_REQUEST, "{bad}:{body}"); + assert!(body.contains("config.min_speed_range"), "{body}"); + } + assert_eq!(b.file(), before, "拒绝了的保存不该动配置"); + + let (st, body) = save(serde_json::Value::Null).await; + assert_eq!(st, StatusCode::OK, "{body}"); + assert!(!b.file().contains("min_speed"), "{}", b.file()); + let (_, body) = call(&b.app, "GET", "/overview", serde_json::Value::Null).await; + assert!( + json(&body)["providers"][0].get("min_speed").is_none(), + "{body}" + ); +} + #[tokio::test] async fn billing_is_per_token_or_free_and_nothing_else() { // 订阅账号也按价目表算费用:「订阅」「未知」不再是计费方式,存不进去 diff --git a/crates/tw-dialect/src/anthropic/stream.rs b/crates/tw-dialect/src/anthropic/stream.rs index dc141732..5bd30600 100644 --- a/crates/tw-dialect/src/anthropic/stream.rs +++ b/crates/tw-dialect/src/anthropic/stream.rs @@ -237,14 +237,17 @@ impl Writer { Event::BlockStop { index } => self.stop_block(*index, &mut out), Event::Usage(u) => self.usage.merge(u), Event::Stop(s) => self.stop = Some(s.clone()), - Event::Error { message } => { - self.failed = true; - out.push_str(&error_frame(500, message)); - } + Event::Error { message } => out.push_str(&self.fail(500, message)), } out } + /// 中途出错:一条状态码 `status` 的错误帧(见 [`error_frame`]),之后不再写收尾 + pub(crate) fn fail(&mut self, status: u16, message: &str) -> String { + self.failed = true; + error_frame(status, message) + } + fn stop_block(&mut self, index: usize, out: &mut String) { let Some(pos) = self.open.iter().position(|i| *i == index) else { return; diff --git a/crates/tw-dialect/src/chat/stream.rs b/crates/tw-dialect/src/chat/stream.rs index 2130f871..9cb07190 100644 --- a/crates/tw-dialect/src/chat/stream.rs +++ b/crates/tw-dialect/src/chat/stream.rs @@ -255,14 +255,17 @@ impl Writer { Event::BlockStop { .. } => {} Event::Usage(u) => self.usage.get_or_insert_default().merge(u), Event::Stop(s) => self.stop = Some(s.clone()), - Event::Error { message } => { - self.done = true; - out.push_str(&error_frame(500, message)); - } + Event::Error { message } => out.push_str(&self.fail(500, message)), } out } + /// 中途出错:一条状态码 `status` 的错误帧,之后不再写收尾 + pub(crate) fn fail(&mut self, status: u16, message: &str) -> String { + self.done = true; + error_frame(status, message) + } + /// 流结束。**幂等** pub fn finish(&mut self) -> String { let mut out = String::new(); diff --git a/crates/tw-dialect/src/convert.rs b/crates/tw-dialect/src/convert.rs index 37743f60..fc047335 100644 --- a/crates/tw-dialect/src/convert.rs +++ b/crates/tw-dialect/src/convert.rs @@ -566,6 +566,19 @@ impl Writer { } } + /// 中途出错,状态码 `status`。Bedrock 的异常帧不带状态码 + fn fail(&mut self, status: u16, message: &str) -> String { + match self { + Writer::Anthropic(w) => w.fail(status, message), + Writer::Chat(w) => w.fail(status, message), + Writer::Responses(w) => w.fail(status, message), + Writer::Gemini(w) => w.fail(status, message), + Writer::Bedrock(w) => w.event(&Event::Error { + message: message.to_string(), + }), + } + } + fn finish(&mut self) -> String { match self { Writer::Anthropic(w) => w.finish(), @@ -794,13 +807,17 @@ impl StreamConverter { /// 流在中途出了错(上游断开、被策略切断):按客户端的格式写一个错误收尾。之后 /// [`StreamConverter::finish`] 什么都不再写 pub fn fail(&mut self, message: &str) -> Vec { + self.fail_with(500, message) + } + + /// 同 [`StreamConverter::fail`],错误按状态码 `status` 写:Anthropic 的 503 是 + /// `overloaded_error`、Gemini 的是 `UNAVAILABLE`,客户端认得出该不该重试 + pub fn fail_with(&mut self, status: u16, message: &str) -> Vec { if self.finished { return Vec::new(); } self.finished = true; - let mut out = self.writer.event(&Event::Error { - message: message.to_string(), - }); + let mut out = self.writer.fail(status, message); out.push_str(&self.writer.finish()); out.into_bytes() } @@ -1361,6 +1378,32 @@ mod tests { assert!(c.finish().is_empty()); } + /// 带状态码的收尾:503 在每种客户端格式里都是「过载、可以重试」那一种 + #[test] + fn a_failure_with_a_status_is_written_in_that_status() { + let ended = |client| { + let mut c = Session::for_test(client, Dialect::Chat).stream(); + let out = String::from_utf8(c.fail_with(503, "too slow")).unwrap(); + assert!(c.finish().is_empty()); + out + }; + let out = ended(Dialect::Anthropic); + assert!(out.contains("event: error"), "{out}"); + assert!(out.contains("\"overloaded_error\""), "{out}"); + let out = ended(Dialect::Responses); + assert!(out.contains("response.failed"), "{out}"); + assert!(out.contains("\"server_error\""), "{out}"); + let out = ended(Dialect::Gemini); + assert!(out.contains("\"UNAVAILABLE\""), "{out}"); + let out = ended(Dialect::Chat); + assert!(out.contains("\"error\""), "{out}"); + assert!(out.contains("too slow"), "{out}"); + // 不带状态码的照旧是 500 + let mut c = Session::for_test(Dialect::Anthropic, Dialect::Chat).stream(); + let out = String::from_utf8(c.fail("broke")).unwrap(); + assert!(out.contains("\"api_error\""), "{out}"); + } + #[test] fn a_body_that_is_not_json_is_refused_in_words() { let e = prepare( diff --git a/crates/tw-dialect/src/gemini/stream.rs b/crates/tw-dialect/src/gemini/stream.rs index e1d39326..1b8def04 100644 --- a/crates/tw-dialect/src/gemini/stream.rs +++ b/crates/tw-dialect/src/gemini/stream.rs @@ -320,14 +320,19 @@ impl Writer { } Event::Usage(u) => self.usage.get_or_insert_default().merge(u), Event::Stop(s) => self.stop = Some(s.clone()), - Event::Error { message } => { - self.write(&super::response::error_body(500, message), &mut out); - self.close(&mut out); - } + Event::Error { message } => out.push_str(&self.fail(500, message)), } out } + /// 中途出错:一个状态码 `status` 的错误元素,然后收尾 + pub(crate) fn fail(&mut self, status: u16, message: &str) -> String { + let mut out = String::new(); + self.write(&super::response::error_body(status, message), &mut out); + self.close(&mut out); + out + } + fn close(&mut self, out: &mut String) { self.done = true; if !self.sse { diff --git a/crates/tw-dialect/src/responses/stream.rs b/crates/tw-dialect/src/responses/stream.rs index 6290a86c..b44c8e6c 100644 --- a/crates/tw-dialect/src/responses/stream.rs +++ b/crates/tw-dialect/src/responses/stream.rs @@ -487,17 +487,22 @@ impl Writer { Event::BlockStop { index } => self.stop_item(*index, &mut out), Event::Usage(u) => self.usage.get_or_insert_default().merge(u), Event::Stop(s) => self.stop = Some(s.clone()), - Event::Error { message } => { - self.start(&mut out); - self.finished = true; - let mut r = envelope(&self.id, self.created, &self.model, "failed"); - r["error"] = failure(500, message); - self.emit("response.failed", json!({ "response": r }), &mut out); - } + Event::Error { message } => out.push_str(&self.fail(500, message)), } out } + /// 中途出错:`response.failed`,错误码按状态码 `status` 定(见 [`failure`]) + pub(crate) fn fail(&mut self, status: u16, message: &str) -> String { + let mut out = String::new(); + self.start(&mut out); + self.finished = true; + let mut r = envelope(&self.id, self.created, &self.model, "failed"); + r["error"] = failure(status, message); + self.emit("response.failed", json!({ "response": r }), &mut out); + out + } + fn delta(&mut self, index: usize, delta: &Delta, out: &mut String) { let Some(o) = self.items.get_mut(&index) else { return; diff --git a/crates/tw-gateway/src/ending.rs b/crates/tw-gateway/src/ending.rs index 7f68a57b..cb5b8cd8 100644 --- a/crates/tw-gateway/src/ending.rs +++ b/crates/tw-gateway/src/ending.rs @@ -95,6 +95,11 @@ pub struct Ending { /// 这个请求的报文记录(见 [`crate::content`])。有它时,回答那一跳的回答攒在它身上 /// (`tap` 不用):实时内容和落盘看的是同一份 capture: Option, + /// 回答因为生成太慢被结束了(见 [`crate::slow`]):结束那一刻窗口里的速度。失败的结局带着它 + slow: Option, + /// 网关估的输入 token(和开始事件的 `input_estimate` 同一个数)。生成太慢被结束、上游 + /// 还没报用量时按它记(见 [`Ending::too_slow`]) + input_estimate: Option, /// 报过了。**只能报一次** told: bool, } @@ -232,6 +237,10 @@ struct Stream { pace: std::sync::Arc, provider: String, ratio: f64, + /// 最低生成速度的判定窗口(见 [`crate::slow`]):这一家设了才有 + gauge: Option, + /// 这一块里来了内容(有字的正文、推理、工具参数,或者一个工具调用开了头) + fresh: bool, } impl Stream { @@ -241,7 +250,10 @@ impl Stream { ir::Event::Delta { delta: ir::Delta::Text(t) | ir::Delta::ToolInput(t), .. - } => self.text.add(t), + } => { + self.text.add(t); + self.fresh |= !t.is_empty(); + } ir::Event::Delta { delta: ir::Delta::Thinking(t), .. @@ -250,8 +262,13 @@ impl Stream { if !t.is_empty() { self.thought_text = true; self.thought_pending = false; + self.fresh = true; } } + ir::Event::BlockStart { + kind: ir::BlockKind::ToolCall { .. }, + .. + } => self.fresh = true, ir::Event::BlockStart { kind: ir::BlockKind::Thinking, .. @@ -269,15 +286,24 @@ impl Stream { /// 这一块读完了:估到目前为止一共多少,多出来的记到速度表上。开口那一刻哪怕估出 0 个 /// 也记一笔 —— 速度从那一刻算起 + /// + /// 判定窗口(有的话)记的是同一个数,只在这一块来了内容时记(长静默按它认) fn tally(&mut self) { if !self.opened { return; } let estimated = (self.text.tokens() as f64 * self.ratio).round() as u64; let total = estimated.max(self.reported); - let Some(m) = &self.meter else { return }; + let grew = total.saturating_sub(self.added); + if let Some(g) = self.gauge.as_mut() + && std::mem::take(&mut self.fresh) + { + g.content(Instant::now(), grew); + } if total > self.added || self.added == 0 { - m.add(total.saturating_sub(self.added)); + if let Some(m) = &self.meter { + m.add(grew); + } self.added = total.max(self.added); } } @@ -350,6 +376,8 @@ impl Ending { abort: None, delivered: false, capture: None, + slow: None, + input_estimate: None, told: false, } } @@ -387,6 +415,8 @@ impl Ending { pace: pace.clone(), provider: provider.to_string(), ratio: pace.ratio(provider, &self.model), + gauge: None, + fresh: false, }); self.watch = Some(Watch { provider: provider.to_string(), @@ -470,6 +500,33 @@ impl Ending { } } + /// 这一家设了最低生成速度(见 [`crate::slow`]):从第一个 token 起记判定窗口。**在 + /// [`Ending::streaming`] 之后才有用**,没有流的什么都不记 + pub fn gauge(&mut self, g: crate::slow::Gauge) { + if let Some(s) = self.stream.as_mut() { + s.gauge = Some(g); + } + } + + /// 此刻判定窗口里的速度(见 [`crate::slow::Gauge::rate`])。没有判定窗口、说不出的是 `None` + pub fn pace_at(&mut self, now: Instant) -> Option { + self.stream.as_mut()?.gauge.as_mut()?.rate(now) + } + + /// 网关估的输入 token(开始事件的 `input_estimate`):生成太慢被结束时,上游没报输入就按它记 + pub fn input_estimated(&mut self, tokens: u64) { + self.input_estimate = Some(tokens); + } + + /// 回答因为生成太慢被结束了(见 [`crate::slow`]),`rate` 是结束那一刻窗口里的速度:这段对话 + /// 这一轮不再留在这一家;失败的结局带着这个速度,已经生成的输出上游没报全的按估数记 + pub fn too_slow(&mut self, rate: u32) { + self.slow = Some(rate); + if let Some(t) = self.answer.take() { + t.left(); + } + } + /// 回答的那一家答到一半没了声音(无响应超时,见 `server::pipeline::idle`):这段对话这一轮 /// 不再留在它那儿 pub fn went_quiet(&mut self) { @@ -696,6 +753,10 @@ impl Ending { fn fail(&mut self, source: tw_api::FailureSource, message: Msg) { let (usage, answered_model) = self.settle(); + let usage = match self.slow { + Some(_) => self.estimated(usage), + None => usage, + }; let (sent_bytes, received_bytes) = self.traffic_seen(); self.close(if source == tw_api::FailureSource::Aborted { tw_api::LiveOutcome::Aborted @@ -712,9 +773,34 @@ impl Ending { duration_ms: Some(self.duration_ms()), usage: usage.map(view), answered_model, + tokens_per_sec: self.slow, }); } + /// 生成太慢被结束的用量:上游报的为准,没报全的补上估数 —— 输出是流里估出来的(和速度同一个 + /// 估法),输入是网关估的。Anthropic 在流的末尾才报累计输出,Responses、Chat 的用量也在 + /// 末尾,断在中间时手里的数是个占位,或者干脆没有。按估数记的钱是估算(存储层对失败的 + /// 一律这样记) + fn estimated(&self, usage: Option) -> Option { + let output = self.stream.as_ref().map_or(0, |s| s.added); + let input = self.input_estimate.unwrap_or(0); + match usage { + Some(mut u) => { + u.output = u.output.max(output); + if u.prompt_total() == 0 { + u.input = input; + } + Some(u) + } + None if output > 0 || input > 0 => Some(Usage { + input, + output, + ..Usage::default() + }), + None => None, + } + } + /// 收尾:先把响应体交出去,再拿走用量和回答里写的模型名(两样是嗅探器在同一遍里 /// 认的)。三种结局共用,只走一次。 fn settle(&mut self) -> (Option, Option) { @@ -790,6 +876,7 @@ impl Drop for Ending { if self.abort.as_ref().is_some_and(|r| r.thrown()) { self.close(tw_api::LiveOutcome::Aborted); self.bus.emit(tw_api::Event::RequestFailed { + tokens_per_sec: None, id: self.id, model: std::mem::take(&mut self.model), source: tw_api::FailureSource::Aborted, @@ -807,6 +894,7 @@ impl Drop for Ending { if std::thread::panicking() { self.close(tw_api::LiveOutcome::Failed); self.bus.emit(tw_api::Event::RequestFailed { + tokens_per_sec: None, id: self.id, model: std::mem::take(&mut self.model), source: tw_api::FailureSource::Internal, diff --git a/crates/tw-gateway/src/error.rs b/crates/tw-gateway/src/error.rs index 997b13cb..a59aa340 100644 --- a/crates/tw-gateway/src/error.rs +++ b/crates/tw-gateway/src/error.rs @@ -51,6 +51,11 @@ pub enum Source { /// 上游没坏,是没在时限里回话 —— Anthropic 的格式里它是 `timeout_error`,Gemini 是 /// `DEADLINE_EXCEEDED`。对外的词表里算 `upstream` Timeout, + /// 上游生成得比设定的最低速度慢(`providers[].min_speed`,见 [`crate::slow`]),回答被结束了。 + /// **503**:Anthropic 的格式里是 `overloaded_error`,Responses 是 `server_error`,Gemini 是 + /// `UNAVAILABLE` —— 都是客户端认作「上游此刻过载、可以重试」的那一种。504 在 Anthropic 里是 + /// `timeout_error`,更少有客户端重试。对外的词表里算 `upstream` + Slow, /// 在界面上手动中止的(见 [`crate::abort`])。**499**:客户端的 SDK 不重试 4xx /// (408、409、429 除外),不会把用户叫停的请求自己再发一遍;Google 的接口给「操作被 /// 取消」用的也是它。对外的词表里是 `aborted` @@ -74,7 +79,7 @@ impl Source { Source::RateLimited | Source::Busy => "rate_limited", Source::Denied => "denied", Source::NotSupported | Source::TooLarge => "request", - Source::Timeout => "upstream", + Source::Timeout | Source::Slow => "upstream", Source::Aborted => "aborted", } } @@ -90,6 +95,7 @@ impl Source { Source::NotSupported => StatusCode::NOT_IMPLEMENTED, Source::TooLarge => StatusCode::PAYLOAD_TOO_LARGE, Source::Timeout => StatusCode::GATEWAY_TIMEOUT, + Source::Slow => StatusCode::SERVICE_UNAVAILABLE, Source::Aborted => StatusCode::from_u16(499).expect("499 is a valid status code"), } } @@ -201,6 +207,10 @@ impl GatewayError { pub fn timeout(detail: Msg) -> Self { Self::new(Source::Timeout, detail) } + /// 生成太慢,回答被结束了(见 [`crate::slow`]):503 + pub fn slow(detail: Msg) -> Self { + Self::new(Source::Slow, detail) + } /// 手动中止(见 [`crate::abort`])。句子只有一句,码是 [`tw_api::ABORTED`] pub fn aborted() -> Self { Self::new( @@ -211,6 +221,11 @@ impl GatewayError { } impl GatewayError { + /// 这个错误的状态码:响应头上的,和流里错误帧按它写的那一个 + pub fn status(&self) -> StatusCode { + self.source.status() + } + /// 发给客户端的那句话。`[ThinkWatch]` 前缀不是装饰:没有它,用户看到 /// 一个 401 会先去查上游的密钥 —— 而问题在中间这一层。 fn client_message(&self) -> String { @@ -317,6 +332,23 @@ mod tests { ); } + /// 生成太慢:503,客户端按「过载」重试;对外的词表里还是上游的事 + #[tokio::test] + async fn too_slow_is_503_overloaded_and_counts_as_upstream() { + let e = GatewayError::slow(msg!("t.slow" => "too slow")); + assert!(e.sse_frame().contains("overloaded_error")); + let (status, slug, json) = body_of(e).await; + assert_eq!(status, StatusCode::SERVICE_UNAVAILABLE); + assert_eq!(slug, "upstream"); + assert_eq!(json["error"]["type"], "overloaded_error"); + let gemini = GatewayError::slow(msg!("t.slow" => "too slow")).in_dialect(Dialect::Gemini); + assert!(gemini.sse_frame().contains("UNAVAILABLE")); + let responses = + GatewayError::slow(msg!("t.slow" => "too slow")).in_dialect(Dialect::Responses); + let frame = responses.sse_frame(); + assert!(frame.contains("response.failed") && frame.contains("server_error")); + } + #[tokio::test] async fn upstream_failures_are_502_not_500() { // 500 会让人怀疑我们;502 说清楚是上游那边。 diff --git a/crates/tw-gateway/src/lib.rs b/crates/tw-gateway/src/lib.rs index 2a766094..c57114af 100644 --- a/crates/tw-gateway/src/lib.rs +++ b/crates/tw-gateway/src/lib.rs @@ -57,6 +57,7 @@ pub mod sent; pub mod server; pub mod session; pub mod slots; +pub mod slow; mod splice; pub mod state; pub mod throttle; diff --git a/crates/tw-gateway/src/server/pipeline.rs b/crates/tw-gateway/src/server/pipeline.rs index 639c14fd..59f524a6 100644 --- a/crates/tw-gateway/src/server/pipeline.rs +++ b/crates/tw-gateway/src/server/pipeline.rs @@ -381,6 +381,10 @@ async fn answer_with( &served.provider.name, )); } + // 生成太慢被结束、上游还没报输入时按它记(见 `Ending::too_slow`) + if matches!(reading.decoded, Some(Ok(_))) { + ending.input_estimated(reading.facts.input_tokens); + } Ok(relay::respond( state, rt, diff --git a/crates/tw-gateway/src/server/pipeline/relay.rs b/crates/tw-gateway/src/server/pipeline/relay.rs index eef3144f..60b2b5a0 100644 --- a/crates/tw-gateway/src/server/pipeline/relay.rs +++ b/crates/tw-gateway/src/server/pipeline/relay.rs @@ -8,9 +8,9 @@ //! 这些步骤要不要做、怎么做,在响应头到手的那一刻就全定了([`Plan`]); //! 流里每一块怎么处理在 [`Relay`] 上,`respond` 里的流只剩一个循环。 //! -//! 那个循环还看着两件事:上游多久没有内容(无响应超时,见 [`super::idle`]),和这个请求有 -//! 没有被手动中止(见 [`crate::abort`])。哪一件到了都丢掉上游的流,按客户端的格式补一条 -//! 错误收尾。 +//! 那个循环还看着三件事:上游多久没有内容(无响应超时,见 [`super::idle`]),生成得够不够快 +//! (这一家设了最低生成速度时,见 [`crate::slow`]),和这个请求有没有被手动中止(见 +//! [`crate::abort`])。哪一件到了都丢掉上游的流,按客户端的格式补一条错误收尾。 use axum::body::Body; use axum::http::{HeaderMap, StatusCode}; @@ -150,6 +150,15 @@ pub(super) fn respond( sent: sent_at, }); } + // 最低生成速度(见 `crate::slow`):这一家设了、回的是成功的流、客户端收的也是流才看。 + // 客户端要整包的(上游的流收齐了再交)不管:它在结束之前什么都收不到,谈不上快慢 + let slow = provider + .min_speed + .filter(|_| generates && plan.is_sse && status.is_success() && !plan.collect) + .map(|m| crate::slow::Watch::of(&m, state.speed_tick)); + if let Some(w) = &slow { + ending.gauge(w.gauge()); + } // 回的不是 2xx:原样交给客户端,**这个请求照样是失败的** —— 客户端拿到的是上游的 // 错误,不是回答。原因在错误正文里,交完时读(见 `Ending::refused`) if !status.is_success() { @@ -218,7 +227,40 @@ pub(super) fn respond( let mut aborted = false; // 这一家的成败还没记(最后一家不压开头时,见 `hop`):第一段真内容到了记成功 let mut unconfirmed = unconfirmed; + // 下一次看生成速度的时刻(这一家没设最低生成速度时用不到) + let mut pace_at = tokio::time::Instant::now() + pace_every(slow); loop { + // 每秒看一次生成速度:到点了就看(上游一直在发、计时那一支轮不上的时候也看得到) + if let Some(w) = slow + && tokio::time::Instant::now() >= pace_at + { + pace_at = tokio::time::Instant::now() + w.tick; + if let Some(rate) = w.judge(ending.pace_at(std::time::Instant::now())) { + tracing::warn!( + provider = %upstream_name, rate, min = w.min, secs = w.secs, + "ended the response: the upstream generated below its minimum speed" + ); + // 这是用户给这一家定的「不达标」:记一次失败(重发的请求先去别家,连着几次 + // 停用),这段对话这一轮不再留在它那儿 + crate::server::note_health( + &bus, + &health, + &upstream_name, + health.record_failure(&upstream_name), + ); + ending.too_slow(rate); + broke = Some( + GatewayError::slow(crate::slow::too_slow( + &upstream_name, + rate, + w.min, + w.secs, + )) + .in_dialect(dialect), + ); + break; + } + } // 补心跳的那一刻。不补的格式给一个不会用到的时刻,那一支不参与 let ping_at = ping_every.map_or(quiet_by, |every| quiet_since + every); let next = tokio::select! { @@ -247,6 +289,8 @@ pub(super) fn respond( broke = Some(GatewayError::timeout(why).in_dialect(dialect)); break; } + // 到了看生成速度的时候:回到开头去看 + _ = tokio::time::sleep_until(pace_at), if slow.is_some() => continue, _ = tokio::time::sleep_until(ping_at), if ping_every.is_some() => { // 停在一帧中间时这一轮不补,也要重新计时,否则会原地空转 quiet_since = tokio::time::Instant::now(); @@ -369,8 +413,9 @@ pub(super) fn respond( // 防火墙切断是 `denied` —— 后者不是上游坏了,是策略拦的。 // 码保持不变,只在句子前面点明它断在流里 —— 界面认的是码 // 手动中止的不是流断了:原句照报,记录里是那个码 + // 生成太慢也一样:流没断,是网关按设定结束的 let mut why = err.detail.clone(); - if !aborted { + if !aborted && err.source != crate::error::Source::Slow { why.text = format!("the response stream broke: {}", why.text); } ending.failed(err.source.into(), why); @@ -386,6 +431,11 @@ pub(super) fn respond( resp } +/// 看生成速度的间隔:一「秒」。没设最低生成速度的给一个不会用到的数 +fn pace_every(slow: Option) -> std::time::Duration { + slow.map_or(std::time::Duration::from_secs(3600), |w| w.tick) +} + /// 上游说的是哪种格式。转换过的看会话,直通的就是客户端那一种。 fn upstream_dialect( req: &Inbound, @@ -1187,8 +1237,13 @@ impl Relay { fn error_frame(&mut self, err: &GatewayError) -> Option> { if let Some(c) = self.back.as_mut() { - // 转换过的流按客户端的格式收尾 - Some(c.fail(&format!("[ThinkWatch] {}", err.message()))) + // 转换过的流按客户端的格式收尾。生成太慢的按它的状态码写(503:客户端认作过载、会 + // 重试);别的照旧 + let said = format!("[ThinkWatch] {}", err.message()); + Some(match err.source { + crate::error::Source::Slow => c.fail_with(err.status().as_u16(), &said), + _ => c.fail(&said), + }) } else if let (true, Some(s)) = (self.plan.collect, self.session.as_ref()) { // 要收齐的整包一个字节都还没发:按客户端的格式回一个错误体 Some(tw_dialect::convert::error_body( diff --git a/crates/tw-gateway/src/slow.rs b/crates/tw-gateway/src/slow.rs new file mode 100644 index 00000000..d954d125 --- /dev/null +++ b/crates/tw-gateway/src/slow.rs @@ -0,0 +1,254 @@ +//! 生成过慢时中断(配置的 `providers[].min_speed`)。 +//! +//! 自部署的模型偶尔会掉到每秒几个 token:一直有内容,无响应超时(见 +//! `server::pipeline::idle`)永远不到点,用户只能干等或者手动中止。设了最低生成速度的上游, +//! 流式回答从**第一个 token** 起每秒看一次:最近 `window_secs` 秒的平均速度低于它,就以一条 +//! 客户端可以重试的错误(503)结束回答,断开上游,这家记一次失败(见 `relay`)。 +//! +//! 速度怎么算: +//! +//! - **token 和实时速度同一个估法**(`/live`、`RunningView::tokens_per_sec`,见 +//! [`crate::ending`]):正文、推理、工具参数按字数估,乘上这家这个模型校准过的比例 +//! ([`crate::pace`])。界面上看到的数就是判断的依据。 +//! - **长静默不算**:连续 [`SILENCE_SECS`] 秒以上没有内容的那一段,时长和其中的 token(0 个) +//! 都从窗口里扣掉 —— 那是无响应超时的事,回答中途想一会儿、等工具参数这类正常的停顿不该 +//! 被当成生成太慢。扣完之后剩下的不到窗口的一半,这一秒不下结论:几秒的数据说明不了什么。 +//! - **第一个 token 之后不到一个窗口不判断**:开头几秒的波动不算数。第一个 token 之前的 +//! 等待归无响应超时管。 +//! +//! 只管流式的回答(SSE、Bedrock 的 eventstream):整包的回答在结束之前没有可测的速度, +//! WebSocket 上的一轮也不管。 + +use std::collections::VecDeque; +use std::time::{Duration, Instant}; + +use tw_types::msg; + +/// 连续多少秒没有内容算长静默(从窗口里扣掉) +pub const SILENCE_SECS: u32 = 5; + +/// 一条流最近一段时间里生成了多少:每来一段内容记一笔,判断时看最近一个窗口。 +/// +/// **按格记,不按笔记**:一条流一秒几十笔,相隔不到四分之一秒的并成一笔。窗口最长两分钟, +/// 留下的最多几百笔。 +#[derive(Debug)] +pub struct Gauge { + /// 「一秒」有多长。**测试会把它调短**(`AppState::speed_tick`),速度按这个「秒」算 + tick: Duration, + /// 判定窗口 + window: Duration, + /// 长静默的门限 + silence: Duration, + /// 第一段内容到的时刻 + first: Option, + /// 上一段内容到的时刻 + last: Option, + /// (时刻,这一笔估出来的 token),按时间排,只留窗口里的 + samples: VecDeque<(Instant, u64)>, + /// 长静默:(开始,结束),按时间排,只留和窗口有交集的 + gaps: VecDeque<(Instant, Instant)>, +} + +impl Gauge { + pub fn new(window_secs: u32, tick: Duration) -> Self { + Self { + tick, + window: tick.saturating_mul(window_secs), + silence: tick.saturating_mul(SILENCE_SECS), + first: None, + last: None, + samples: VecDeque::new(), + gaps: VecDeque::new(), + } + } + + /// 在 `at` 来了一段内容,估出 `tokens` 个(可以是 0:一个字估不出一个 token) + pub fn content(&mut self, at: Instant, tokens: u64) { + self.first.get_or_insert(at); + if let Some(last) = self.last + && at.saturating_duration_since(last) >= self.silence + { + self.gaps.push_back((last, at)); + } + self.last = Some(at); + match self.samples.back_mut() { + Some((t, n)) if at.saturating_duration_since(*t) < self.tick / 4 => { + *n = n.saturating_add(tokens) + } + _ => self.samples.push_back((at, tokens)), + } + } + + /// 此刻最近一个窗口的平均速度,每「秒」多少 token。说不出的时候是 `None`:还没开口、 + /// 开口不到一个窗口、扣掉长静默之后剩下的不到窗口的一半 + pub fn rate(&mut self, now: Instant) -> Option { + let first = self.first?; + let start = now.checked_sub(self.window)?; + if start < first { + return None; + } + while self.samples.front().is_some_and(|(t, _)| *t <= start) { + self.samples.pop_front(); + } + while self.gaps.front().is_some_and(|(_, end)| *end <= start) { + self.gaps.pop_front(); + } + let tokens: u64 = self.samples.iter().map(|(_, n)| n).sum(); + let overlap = + |(from, to): (Instant, Instant)| to.min(now).saturating_duration_since(from.max(start)); + let mut quiet: Duration = self.gaps.iter().map(|g| overlap(*g)).sum(); + // 到此刻为止已经静了这么久:也是一段长静默,还没结束 + if let Some(last) = self.last + && now.saturating_duration_since(last) >= self.silence + { + quiet += overlap((last, now)); + } + let spoken = self.window.saturating_sub(quiet); + if spoken < self.window / 2 || spoken.is_zero() { + return None; + } + Some(tokens as f64 * self.tick.as_secs_f64() / spoken.as_secs_f64()) + } +} + +/// 这一家设了最低生成速度时,读流的循环怎么看着它(见 `relay`)。 +#[derive(Debug, Clone, Copy)] +pub struct Watch { + /// 每秒至少多少 token + pub min: u32, + /// 判定窗口,秒(配置写的,报给人看的句子用它) + pub secs: u32, + /// 「一秒」有多长 + pub tick: Duration, +} + +impl Watch { + pub fn of(m: &tw_config::MinSpeed, tick: Duration) -> Self { + Self { + min: m.tokens_per_sec, + secs: m.window_secs, + tick, + } + } + + pub fn gauge(&self) -> Gauge { + Gauge::new(self.secs, self.tick) + } + + /// 这个速度够不够:不够的话交回报给人看的那个数(取整,往下) + pub fn judge(&self, rate: Option) -> Option { + let r = rate?; + (r < f64::from(self.min)).then(|| r.max(0.0).floor() as u32) + } +} + +/// 回答因为生成太慢被结束了:请求记录里的原因,也是交给客户端的那句话 +pub fn too_slow(upstream: &str, rate: u32, min: u32, secs: u32) -> tw_types::Msg { + msg!( + "gw.upstream.too_slow", upstream = upstream, rate = rate, min = min, secs = secs => + "Upstream `{upstream}` generated {rate} tokens per second over the last {secs} seconds, \ + below the minimum of {min}." + ) +} + +#[cfg(test)] +mod tests { + use super::*; + + const S: Duration = Duration::from_secs(1); + + fn at(t: Instant, secs: f64) -> Instant { + t + Duration::from_secs_f64(secs) + } + + /// 每 `every` 秒来一段、每段 `n` 个,从 `from` 到 `to` + fn steady(g: &mut Gauge, t: Instant, from: f64, to: f64, every: f64, n: u64) { + let mut s = from; + while s < to { + g.content(at(t, s), n); + s += every; + } + } + + #[test] + fn nothing_is_said_before_a_whole_window_after_the_first_token() { + let t = Instant::now(); + let mut g = Gauge::new(20, S); + assert_eq!(g.rate(at(t, 30.0)), None, "还没开口"); + steady(&mut g, t, 10.0, 29.5, 0.5, 1); + assert_eq!(g.rate(at(t, 29.5)), None, "开口不到 20 秒"); + let r = g.rate(at(t, 30.0)).unwrap(); + assert!((r - 2.0).abs() < 0.11, "{r}"); + } + + #[test] + fn the_window_only_counts_the_last_seconds() { + let t = Instant::now(); + let mut g = Gauge::new(10, S); + // 前十秒每秒 100 个,之后每秒 5 个 + steady(&mut g, t, 0.0, 10.0, 0.5, 50); + steady(&mut g, t, 10.0, 31.0, 1.0, 5); + let r = g.rate(at(t, 30.0)).unwrap(); + assert!((r - 5.0).abs() < 0.01, "{r}"); + } + + /// 中途停了 10 秒(长静默):那段时间不算,速度按说话的时间算 + #[test] + fn a_long_silence_is_taken_out_of_the_window() { + let t = Instant::now(); + let mut g = Gauge::new(20, S); + steady(&mut g, t, 0.0, 20.0, 0.25, 5); + // 20 秒到 30 秒一个字都没有,之后照常 + steady(&mut g, t, 30.0, 40.0, 0.25, 5); + let r = g.rate(at(t, 40.0)).unwrap(); + assert!((r - 20.0).abs() < 1.1, "{r}"); + // 静着的时候:扣掉之后不到窗口的一半就不说 + let mut g = Gauge::new(20, S); + steady(&mut g, t, 0.0, 20.0, 0.25, 5); + assert!(g.rate(at(t, 25.0)).is_some(), "静了 5 秒,剩 15 秒"); + assert_eq!(g.rate(at(t, 31.0)), None, "静了 11 秒,只剩 9 秒"); + } + + /// 短于门限的停顿照算:每 4 秒来一段,算的是平均下来的慢 + #[test] + fn short_pauses_count_as_slow() { + let t = Instant::now(); + let mut g = Gauge::new(20, S); + steady(&mut g, t, 0.0, 40.0, 4.0, 8); + let r = g.rate(at(t, 40.0)).unwrap(); + assert!((r - 2.0).abs() < 0.5, "{r}"); + } + + #[test] + fn the_watch_reports_the_rate_rounded_down_and_only_when_below() { + let w = Watch { + min: 15, + secs: 20, + tick: S, + }; + assert_eq!(w.judge(None), None); + assert_eq!(w.judge(Some(15.0)), None); + assert_eq!(w.judge(Some(14.9)), Some(14)); + assert_eq!(w.judge(Some(0.0)), Some(0)); + } + + #[test] + fn the_sentence_carries_every_argument() { + let m = too_slow("local", 3, 15, 20); + assert_eq!(m.code, "gw.upstream.too_slow"); + assert_eq!( + ( + m.arg("upstream"), + m.arg("rate"), + m.arg("min"), + m.arg("secs") + ), + ("local", "3", "15", "20") + ); + assert_eq!( + m.text, + "Upstream `local` generated 3 tokens per second over the last 20 seconds, below the \ + minimum of 15." + ); + } +} diff --git a/crates/tw-gateway/src/state.rs b/crates/tw-gateway/src/state.rs index 63b2c61e..ab0fb69f 100644 --- a/crates/tw-gateway/src/state.rs +++ b/crates/tw-gateway/src/state.rs @@ -302,6 +302,9 @@ pub struct AppState { /// `failover.idle_timeout_secs` 的一秒有多长(见 `server::pipeline::idle`)。**测试会把它 /// 调短**,否则一条无响应超时的测试至少要干等三十秒 pub idle_tick: std::time::Duration, + /// 最低生成速度(`providers[].min_speed`,见 [`crate::slow`])的一秒有多长:判定窗口、长静默 + /// 和每秒一次的判断都按它算。**测试会把它调短**,否则一条生成过慢的测试至少要干等二十秒 + pub speed_tick: std::time::Duration, /// 流式回答最多压多久不给响应头([`crate::OPENING_HOLD`])。**测试会把它调短** pub opening_hold: std::time::Duration, /// 响应头先交出去之后隔多久发一行保活([`crate::KEEPALIVE_EVERY`])。**测试会把它调短** @@ -395,6 +398,7 @@ impl AppState { aborts: Default::default(), contents: Default::default(), idle_tick: std::time::Duration::from_secs(1), + speed_tick: std::time::Duration::from_secs(1), opening_hold: crate::OPENING_HOLD, keepalive_every: crate::KEEPALIVE_EVERY, retry_backoff: Default::default(), diff --git a/crates/tw-gateway/src/wire.rs b/crates/tw-gateway/src/wire.rs index 2184a85a..67999ef4 100644 --- a/crates/tw-gateway/src/wire.rs +++ b/crates/tw-gateway/src/wire.rs @@ -15,6 +15,7 @@ impl From for FailureSource { Source::Denied => Self::Denied, Source::NotSupported | Source::TooLarge => Self::Request, Source::Timeout => Self::Upstream, + Source::Slow => Self::Upstream, Source::Aborted => Self::Aborted, } } diff --git a/crates/tw-gateway/tests/min_speed.rs b/crates/tw-gateway/tests/min_speed.rs new file mode 100644 index 00000000..9f45ca1d --- /dev/null +++ b/crates/tw-gateway/tests/min_speed.rs @@ -0,0 +1,515 @@ +//! 生成过慢时中断(`providers[].min_speed`,见 `tw_gateway::slow`),端到端。 +//! +//! 假上游按脚本隔一阵发一段:每段正文 80 个 ASCII 字符,估出来是 20 个 token。测试把一秒调成 +//! 五十毫秒(`AppState::speed_tick`):判定窗口 20「秒」是一秒,长静默 5「秒」是 250 毫秒, +//! 每 50 毫秒看一次。最低速度写 15,也就是每 50 毫秒 15 个 token(每秒 300 个): +//! +//! - 快:每 10 毫秒一段(每秒一千多个,在计时粗的机器上也有好几百) +//! - 慢:每 200 毫秒一段(每秒 100 个) +//! +//! 它记下自己的响应被丢掉了没有:回答被结束时,连接要真的断开,上游才会停下。 + +use std::net::SocketAddr; +use std::sync::Arc; +use std::sync::atomic::{AtomicBool, Ordering}; +use std::time::Duration; + +use axum::Router; +use bytes::Bytes; +use futures::StreamExt; +use serde_json::{Value, json}; +use tw_api::Event; +use tw_config::{Client, Config, Failover, MinSpeed, Protocol, Provider}; + +/// 测试里的「一秒」 +const TICK: Duration = Duration::from_millis(50); +/// 最低速度 +const MIN: u32 = 15; + +/// 一段正文:80 个字符,估出来 20 个 token +const TEXT: &str = + "abcdefghijklmnopqrstuvwxyzabcdefghijklmnopqrstuvwxyzabcdefghijklmnopqrstuvwxyzab"; + +const MESSAGE_START: &str = "event: message_start\ndata: {\"type\":\"message_start\",\"message\":{\"id\":\"msg_1\",\"type\":\"message\",\"role\":\"assistant\",\"model\":\"claude-sonnet-5\",\"content\":[],\"usage\":{\"input_tokens\":1200,\"output_tokens\":1}}}\n\n"; +const BLOCK_START: &str = "event: content_block_start\ndata: {\"type\":\"content_block_start\",\"index\":0,\"content_block\":{\"type\":\"text\",\"text\":\"\"}}\n\n"; +const PING: &str = "event: ping\ndata: {\"type\":\"ping\"}\n\n"; +const END: &str = "event: content_block_stop\ndata: {\"type\":\"content_block_stop\",\"index\":0}\n\n\ + event: message_delta\ndata: {\"type\":\"message_delta\",\"delta\":{\"stop_reason\":\"end_turn\"},\"usage\":{\"output_tokens\":2000}}\n\n\ + event: message_stop\ndata: {\"type\":\"message_stop\"}\n\n"; + +fn word() -> Bytes { + Bytes::from(format!( + "event: content_block_delta\ndata: {{\"type\":\"content_block_delta\",\"index\":0,\"delta\":{{\"type\":\"text_delta\",\"text\":\"{TEXT}\"}}}}\n\n" + )) +} + +/// 一个假上游的回答:隔多少毫秒发哪一段,发完了挂不挂着(挂着时每 100 毫秒一行注释) +#[derive(Clone, Default)] +struct Script { + steps: Vec<(u64, Bytes)>, + hang: bool, + content_type: Option<&'static str>, +} + +impl Script { + /// Anthropic 的流开了头 + fn opened() -> Self { + Self { + steps: vec![ + (0, Bytes::from_static(MESSAGE_START.as_bytes())), + (0, Bytes::from_static(BLOCK_START.as_bytes())), + ], + ..Default::default() + } + } + + /// 每 `every` 毫秒一段,持续 `ms` 毫秒 + fn words(mut self, every: u64, ms: u64) -> Self { + for _ in 0..(ms / every).max(1) { + self.steps.push((every, word())); + } + self + } + + /// `ms` 毫秒没有内容(中间隔 100 毫秒一个心跳) + fn quiet(mut self, ms: u64) -> Self { + for _ in 0..ms / 100 { + self.steps.push((100, Bytes::from_static(PING.as_bytes()))); + } + self + } + + fn fast(self, ms: u64) -> Self { + self.words(10, ms) + } + + fn slow(self, ms: u64) -> Self { + self.words(200, ms) + } + + fn end(mut self) -> Self { + self.steps.push((0, Bytes::from_static(END.as_bytes()))); + self + } + + fn hang(mut self) -> Self { + self.hang = true; + self + } +} + +struct Upstream { + addr: SocketAddr, + dropped: Arc, +} + +/// 被丢掉时举旗 +struct Flag(Arc); + +impl Drop for Flag { + fn drop(&mut self) { + self.0.store(true, Ordering::SeqCst); + } +} + +async fn upstream(script: Script) -> Upstream { + let dropped = Arc::new(AtomicBool::new(false)); + let d = dropped.clone(); + let app = Router::new().fallback(axum::routing::any(move || { + let (script, d) = (script.clone(), d.clone()); + async move { + let flag = Flag(d); + let steps = futures::stream::iter(script.steps).then(|(ms, chunk)| async move { + tokio::time::sleep(Duration::from_millis(ms)).await; + Ok::<_, std::convert::Infallible>(chunk) + }); + let hang = script.hang; + let tail = futures::stream::unfold((), move |_| async move { + if !hang { + return None; + } + tokio::time::sleep(Duration::from_millis(100)).await; + Some((Ok(Bytes::from_static(b": keep-alive\n\n")), ())) + }); + let body = steps.chain(tail).map(move |x| { + let _ = &flag; + x + }); + axum::response::Response::builder() + .status(200) + .header( + "content-type", + script.content_type.unwrap_or("text/event-stream"), + ) + .body(axum::body::Body::from_stream(body)) + .unwrap() + } + })); + let l = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + let addr = l.local_addr().unwrap(); + tokio::spawn(async move { axum::serve(l, app).await.unwrap() }); + Upstream { addr, dropped } +} + +fn provider(up: &Upstream, min_speed: Option) -> Provider { + Provider { + name: "local".into(), + base_url: format!("http://{}", up.addr), + key: Some("sk-upstream".into()), + protocol: Some(Protocol::Anthropic), + min_speed: min_speed.map(|tokens_per_sec| MinSpeed { + tokens_per_sec, + window_secs: 20, + }), + ..Default::default() + } +} + +async fn gateway( + p: Provider, +) -> ( + SocketAddr, + tokio::sync::broadcast::Receiver, + tw_gateway::AppState, +) { + let cfg = Config { + version: 1, + clients: vec![Client { + name: "c".into(), + key: "tw-k".into(), + ..Default::default() + }], + providers: vec![p], + // 一次失败就停用:看得出这一家记了一次失败 + failover: Failover { + failures_to_pause: 1, + ..Default::default() + }, + ..Default::default() + }; + let mut state = tw_gateway::AppState::new(cfg).unwrap(); + state.speed_tick = TICK; + let rx = state.bus.subscribe(); + let addr = tw_gateway::serve(state.clone(), ([127, 0, 0, 1], 0).into()) + .await + .unwrap(); + tokio::time::sleep(Duration::from_millis(50)).await; + (addr, rx, state) +} + +fn messages(stream: bool) -> Value { + json!({"model": "claude-sonnet-5", "max_tokens": 1024, "stream": stream, + "messages": [{"role": "user", "content": "Say hello."}]}) +} + +/// 发出去,交回状态码和整个正文 +async fn post(gw: SocketAddr, path: &str, body: &Value) -> (u16, String) { + let resp = reqwest::Client::builder() + .no_proxy() + .build() + .unwrap() + .post(format!("http://{gw}{path}")) + .header("content-type", "application/json") + .header("x-api-key", "tw-k") + .header("x-goog-api-key", "tw-k") + .header("authorization", "Bearer tw-k") + .body(body.to_string()) + .send() + .await + .unwrap(); + let status = resp.status().as_u16(); + (status, resp.text().await.unwrap()) +} + +/// 请求的结局(结束或失败;取消是 panic) +async fn ending(rx: &mut tokio::sync::broadcast::Receiver) -> Event { + loop { + let e = tokio::time::timeout(Duration::from_secs(15), rx.recv()) + .await + .expect("没等到结局") + .unwrap(); + match e { + Event::RequestFinished { .. } | Event::RequestFailed { .. } => return e, + Event::RequestCancelled { .. } => panic!("不该是取消:{e:?}"), + _ => {} + } + } +} + +/// 结局是生成太慢:交回参数和速度 +fn too_slow(e: &Event) -> (tw_api::Msg, Option) { + match e { + Event::RequestFailed { + source, + message, + tokens_per_sec, + .. + } => { + assert_eq!(*source, tw_api::FailureSource::Upstream, "{e:?}"); + assert_eq!(message.code, "gw.upstream.too_slow", "{e:?}"); + (message.clone(), *tokens_per_sec) + } + other => panic!("该是生成太慢:{other:?}"), + } +} + +fn finished(e: &Event) { + assert!( + matches!(e, Event::RequestFinished { status: 200, .. }), + "该好好结束:{e:?}" + ); +} + +/// 等旗举起来,最多两秒 +async fn eventually(flag: &AtomicBool) -> bool { + for _ in 0..40 { + if flag.load(Ordering::SeqCst) { + return true; + } + tokio::time::sleep(Duration::from_millis(50)).await; + } + false +} + +// ───────────────────────────────────────────── 不中断 + +/// 1. 全程都快:照常答完 +#[tokio::test] +async fn a_stream_that_stays_fast_is_left_alone() { + let up = upstream(Script::opened().fast(2_000).end()).await; + let (gw, mut rx, state) = gateway(provider(&up, Some(MIN))).await; + let (status, body) = post(gw, "/v1/messages", &messages(true)).await; + assert_eq!(status, 200); + assert!(body.contains("message_stop"), "{body}"); + assert!(!body.contains("event: error"), "{body}"); + finished(&ending(&mut rx).await); + assert_eq!( + state.health.state("local"), + tw_gateway::health::State::Closed + ); +} + +/// 3. 中途慢了一阵(不到一个窗口)又快起来:不中断 +#[tokio::test] +async fn a_short_slowdown_is_not_enough() { + let up = upstream(Script::opened().fast(1_200).slow(400).fast(1_200).end()).await; + let (gw, mut rx, _) = gateway(provider(&up, Some(MIN))).await; + let (status, body) = post(gw, "/v1/messages", &messages(true)).await; + assert_eq!(status, 200); + assert!(!body.contains("event: error"), "{body}"); + finished(&ending(&mut rx).await); +} + +/// 4. 中途停了 600 毫秒(12「秒」,长静默)之后照常快:静着的那段不算,不中断 +#[tokio::test] +async fn a_long_pause_partway_through_is_not_counted_as_slow() { + let up = upstream(Script::opened().fast(1_200).quiet(600).fast(1_200).end()).await; + let (gw, mut rx, _) = gateway(provider(&up, Some(MIN))).await; + let (status, body) = post(gw, "/v1/messages", &messages(true)).await; + assert_eq!(status, 200); + assert!(!body.contains("event: error"), "{body}"); + finished(&ending(&mut rx).await); +} + +/// 5. 第一个 token 之前等了很久(30「秒」):那是无响应超时的事,不中断 +#[tokio::test] +async fn waiting_for_the_first_token_is_not_judged() { + let up = upstream(Script::opened().quiet(1_500).fast(1_200).end()).await; + let (gw, mut rx, _) = gateway(provider(&up, Some(MIN))).await; + let (status, body) = post(gw, "/v1/messages", &messages(true)).await; + assert_eq!(status, 200); + assert!(!body.contains("event: error"), "{body}"); + finished(&ending(&mut rx).await); +} + +/// 6. 没设最低速度的上游:慢也照常答完 +#[tokio::test] +async fn an_upstream_without_a_minimum_is_left_alone() { + let up = upstream(Script::opened().slow(1_600).end()).await; + let (gw, mut rx, _) = gateway(provider(&up, None)).await; + let (status, body) = post(gw, "/v1/messages", &messages(true)).await; + assert_eq!(status, 200); + assert!(body.contains("message_stop"), "{body}"); + finished(&ending(&mut rx).await); +} + +/// 6. 整包的回答:设了也不管 +#[tokio::test] +async fn a_whole_answer_is_not_judged() { + let whole = json!({"id": "msg_1", "type": "message", "role": "assistant", + "model": "claude-sonnet-5", "content": [{"type": "text", "text": "hello"}], + "stop_reason": "end_turn", "usage": {"input_tokens": 5, "output_tokens": 1}}); + let up = upstream(Script { + steps: vec![(1_500, Bytes::from(whole.to_string()))], + content_type: Some("application/json"), + ..Default::default() + }) + .await; + let (gw, mut rx, _) = gateway(provider(&up, Some(MIN))).await; + let (status, body) = post(gw, "/v1/messages", &messages(false)).await; + assert_eq!(status, 200, "{body}"); + assert!(body.contains("hello"), "{body}"); + finished(&ending(&mut rx).await); +} + +// ───────────────────────────────────────────── 中断 + +/// 2. 中途掉速并持续了一个窗口:以 503 的错误帧结束,断开上游,这一家记一次失败; +/// 失败带着结束那一刻的速度,输出按估数记 +#[tokio::test] +async fn a_stream_that_slows_down_for_a_window_is_ended() { + let up = upstream(Script::opened().fast(500).slow(5_000).hang()).await; + let (gw, mut rx, state) = gateway(provider(&up, Some(MIN))).await; + let t = std::time::Instant::now(); + let (status, body) = post(gw, "/v1/messages", &messages(true)).await; + // 响应头早交出去了:状态码改不了,错误在流里说 + assert_eq!(status, 200); + assert!( + t.elapsed() < Duration::from_secs(4), + "该在掉速一个窗口之后就结束,用了 {:?}", + t.elapsed() + ); + let last = body.trim_end().rsplit("\n\n").next().unwrap(); + assert!(last.starts_with("event: error"), "{body}"); + assert!(last.contains("\"overloaded_error\""), "{last}"); + assert!( + last.contains("[ThinkWatch] Upstream `local` generated"), + "{last}" + ); + assert!(body.contains(TEXT), "已经生成的照常交出去:{body}"); + let e = ending(&mut rx).await; + let (msg, rate) = too_slow(&e); + assert_eq!(msg.arg("upstream"), "local"); + assert_eq!(msg.arg("min"), MIN.to_string()); + assert_eq!(msg.arg("secs"), "20"); + let rate = rate.expect("失败带着速度"); + assert!(rate < MIN, "{rate}"); + assert_eq!(msg.arg("rate"), rate.to_string()); + assert!( + !msg.text.starts_with("the response stream broke"), + "流没断,是按设定结束的:{}", + msg.text + ); + // 上游报的输入照用;输出它只报了个占位的 1,按估数记 + let Event::RequestFailed { + usage: Some(u), + duration_ms, + .. + } = &e + else { + panic!("{e:?}") + }; + assert_eq!(u.input, 1200); + assert!(u.output >= 20 * 20, "估出来的输出:{u:?}"); + assert!(duration_ms.is_some()); + assert!(eventually(&up.dropped).await, "上游的连接要断开"); + assert_eq!( + state.health.state("local"), + tw_gateway::health::State::Open, + "这一家记了一次失败" + ); +} + +/// 估出来的速度还说不出(扣掉长静默之后不到窗口的一半)时不下结论:静着的上游交给无响应超时 +#[tokio::test] +async fn a_stream_that_goes_quiet_is_left_to_the_idle_timeout() { + let up = upstream(Script::opened().fast(1_200).quiet(1_500).fast(300).end()).await; + let (gw, mut rx, _) = gateway(provider(&up, Some(MIN))).await; + let (status, body) = post(gw, "/v1/messages", &messages(true)).await; + assert_eq!(status, 200); + assert!(!body.contains("event: error"), "{body}"); + finished(&ending(&mut rx).await); +} + +// ───────────────────────────────────────────── 各种客户端格式的错误帧 + +/// 7. 转换过的流按客户端的格式、按 503 收尾 +#[tokio::test] +async fn every_client_format_gets_its_own_retryable_error() { + let cases: [(&str, Value, &[&str]); 3] = [ + ( + "/v1/chat/completions", + json!({"model": "claude-sonnet-5", "stream": true, + "messages": [{"role": "user", "content": "hi"}]}), + &["\"error\"", "Upstream `local` generated"], + ), + ( + "/v1/responses", + json!({"model": "claude-sonnet-5", "stream": true, "input": "hi"}), + &["response.failed", "\"server_error\""], + ), + ( + "/v1beta/models/claude-sonnet-5:streamGenerateContent?alt=sse", + json!({"contents": [{"role": "user", "parts": [{"text": "hi"}]}]}), + &["\"UNAVAILABLE\"", "\"code\":503"], + ), + ]; + for (path, body, expect) in cases { + let up = upstream(Script::opened().fast(300).slow(5_000).hang()).await; + let (gw, mut rx, _) = gateway(provider(&up, Some(MIN))).await; + let (status, out) = post(gw, path, &body).await; + assert_eq!(status, 200, "{path}: {out}"); + let last = out.trim_end().rsplit("\n\n").next().unwrap(); + for e in expect { + assert!(out.contains(e), "{path}: {e} not in {last}"); + } + too_slow(&ending(&mut rx).await); + assert!(eventually(&up.dropped).await, "{path}"); + } +} + +/// 7. Bedrock 的 eventstream:拆成 SSE 之后照样测速,照样以 Anthropic 的 503 收尾 +#[tokio::test] +async fn a_bedrock_eventstream_is_measured_too() { + use aws_smithy_eventstream::frame::write_message_to; + use aws_smithy_types::event_stream::{Header, HeaderValue, Message}; + fn event(kind: &str, payload: Value) -> Bytes { + let m = Message::new(payload.to_string().into_bytes()) + .add_header(Header::new( + ":message-type", + HeaderValue::String("event".to_string().into()), + )) + .add_header(Header::new( + ":event-type", + HeaderValue::String(kind.to_string().into()), + )) + .add_header(Header::new( + ":content-type", + HeaderValue::String("application/json".into()), + )); + let mut out = Vec::new(); + write_message_to(&m, &mut out).unwrap(); + Bytes::from(out) + } + let delta = || { + event( + "contentBlockDelta", + json!({"contentBlockIndex": 0, "delta": {"text": TEXT}}), + ) + }; + let mut steps = vec![(0, event("messageStart", json!({"role": "assistant"})))]; + steps.extend((0..30).map(|_| (10, delta()))); + steps.extend((0..25).map(|_| (200, delta()))); + let up = upstream(Script { + steps, + hang: false, + content_type: Some("application/vnd.amazon.eventstream"), + }) + .await; + let p = Provider { + protocol: Some(Protocol::Bedrock), + key: Some("ABSK-test".into()), + ..provider(&up, Some(MIN)) + }; + let (gw, mut rx, _) = gateway(p).await; + let (status, out) = post( + gw, + "/v1/messages", + &json!({"model": "us.anthropic.claude-sonnet-4-5-20250929-v1:0", "max_tokens": 64, + "stream": true, "messages": [{"role": "user", "content": "hi"}]}), + ) + .await; + assert_eq!(status, 200, "{out}"); + assert!(out.contains("\"overloaded_error\""), "{out}"); + too_slow(&ending(&mut rx).await); +} diff --git a/crates/tw-observe/src/bus.rs b/crates/tw-observe/src/bus.rs index 3d98229e..57601db2 100644 --- a/crates/tw-observe/src/bus.rs +++ b/crates/tw-observe/src/bus.rs @@ -447,6 +447,7 @@ mod tests { answered_model: None, }); b.emit(tw_api::Event::RequestFailed { + tokens_per_sec: None, id: 2, model: "m".into(), source: tw_api::FailureSource::Upstream, diff --git a/crates/tw-pack/src/model/tests.rs b/crates/tw-pack/src/model/tests.rs index 2df5eb61..d326bfa7 100644 --- a/crates/tw-pack/src/model/tests.rs +++ b/crates/tw-pack/src/model/tests.rs @@ -49,6 +49,7 @@ providers: pricing: 中转 models: [m1] max_concurrent: 4 + min_speed: {{ tokens_per_sec: 15 }} limits: - per: hour every: 5 @@ -210,10 +211,12 @@ fn an_upstream_splits_into_four_units_by_what_its_fields_are_about() { ); assert_eq!(relay.units["models"], json!({"models": ["m1"]})); assert_eq!(relay.units["billing"], json!({"pricing": "中转"})); - // 用量上限跟着上游走(分享、导出、同步都带着),和并发上限在同一个单位里 + // 用量上限、最低生成速度跟着上游走(分享、导出、同步都带着),和并发上限在同一个单位里。 + // 判定时长没写的也带着默认值:另一边读到的是同一个设定 assert_eq!( relay.units["other"], - json!({"max_concurrent": 4, "limits": [{"per": "hour", "every": 5, "requests": 100}]}) + json!({"max_concurrent": 4, "min_speed": {"tokens_per_sec": 15, "window_secs": 20}, + "limits": [{"per": "hour", "every": 5, "requests": 100}]}) ); // 写明的默认值(proxy: direct)和没写一样;请求头按写的顺序 let official = &s.items[&id(ItemKind::Provider, "官方")]; diff --git a/crates/tw-store/src/recorder.rs b/crates/tw-store/src/recorder.rs index 118d63c0..79a31968 100644 --- a/crates/tw-store/src/recorder.rs +++ b/crates/tw-store/src/recorder.rs @@ -77,7 +77,11 @@ enum Ending<'a> { tokens_per_sec: Option, }, Cancelled, - Failed(&'a tw_api::Msg), + /// 失败了,带着原因。生成太慢被结束的另带着结束那一刻的速度(`RequestFailed::tokens_per_sec`) + Failed { + message: &'a tw_api::Msg, + tokens_per_sec: Option, + }, } pub struct Recorder { @@ -664,6 +668,7 @@ impl Recorder { duration_ms, usage, answered_model, + tokens_per_sec, .. } => self.settle( *id, @@ -672,7 +677,10 @@ impl Recorder { *duration_ms, *usage, answered_model.clone(), - Ending::Failed(message), + Ending::Failed { + message, + tokens_per_sec: *tokens_per_sec, + }, ), Event::LocallyAnswered { id, @@ -878,7 +886,9 @@ impl Recorder { client: p.client.clone(), path: p.path.clone(), local, - reached: p.routing.reached_upstream(matches!(how, Ending::Failed(_))), + reached: p + .routing + .reached_upstream(matches!(how, Ending::Failed { .. })), input: u.map_or(0, |u| u.input), output: u.map_or(0, |u| u.output), cache_read: u.map_or(0, |u| u.cache_read), @@ -906,8 +916,10 @@ impl Recorder { ttft_ms: p.ttft_ms, duration_ms: duration_ms.map(|d| d as i64), tokens_per_sec: match how { - Ending::Finished { tokens_per_sec } => tokens_per_sec, - _ => None, + Ending::Finished { tokens_per_sec } | Ending::Failed { tokens_per_sec, .. } => { + tokens_per_sec + } + Ending::Cancelled => None, }, sent_bytes: sent_bytes.map(|b| b.min(i64::MAX as u64) as i64), received_bytes: received_bytes.map(|b| b.min(i64::MAX as u64) as i64), @@ -920,7 +932,7 @@ impl Recorder { cost_micros, cost_estimated: estimated, error: match how { - Ending::Failed(message) => Some((*message).clone()), + Ending::Failed { message, .. } => Some((*message).clone()), _ => None, }, local, @@ -1312,6 +1324,7 @@ mod tests { }); r.on_event(&started(3, "gpt-5")); r.on_event(&Event::RequestFailed { + tokens_per_sec: None, id: 3, model: String::new(), source: tw_api::FailureSource::Upstream, @@ -1407,6 +1420,7 @@ mod tests { egress: None, }); r.on_event(&Event::RequestFailed { + tokens_per_sec: None, id: 1, model: String::new(), source: tw_api::FailureSource::Denied, @@ -1657,6 +1671,7 @@ mod tests { let (_d, mut r) = rec(); r.on_event(&started(1, "claude-sonnet-4-5")); r.on_event(&Event::RequestFailed { + tokens_per_sec: None, id: 1, model: String::new(), source: tw_api::FailureSource::Upstream, @@ -2469,6 +2484,7 @@ mod security_detail_tests { fn denied(id: u64) -> Event { Event::RequestFailed { + tokens_per_sec: None, id, model: String::new(), source: tw_api::FailureSource::Denied, @@ -3203,6 +3219,7 @@ mod failure_tests { fn failed(id: u64, usage: Option) -> Event { Event::RequestFailed { + tokens_per_sec: None, id, model: String::new(), source: tw_api::FailureSource::Upstream, @@ -3279,6 +3296,27 @@ mod failure_tests { assert_eq!(s.cost_micros_estimated, 300_015); } + /// 生成太慢被结束的(`gw.upstream.too_slow`)带着结束那一刻的速度:照记,事后看得到为什么 + /// 结束。别的失败没有速度 + #[test] + fn a_failure_for_being_too_slow_keeps_its_speed() { + let (_d, mut r) = rec(); + r.on_event(&started(1, "claude-sonnet-4-5")); + let mut slow = failed(1, partial()); + if let Event::RequestFailed { tokens_per_sec, .. } = &mut slow { + *tokens_per_sec = Some(3); + } + r.on_event(&slow); + let row = r.db().get(1).unwrap().unwrap(); + assert_eq!(row.tokens_per_sec, Some(3)); + assert!(row.error.is_some()); + assert!(row.cost_estimated); + + r.on_event(&started(2, "claude-sonnet-4-5")); + r.on_event(&failed(2, partial())); + assert_eq!(r.db().get(2).unwrap().unwrap().tokens_per_sec, None); + } + /// 响应头之前就失败的(每家都拒绝、策略不让)。**没有用量就没有金额, /// 不是 0**;耗时照记 —— 「试了二十秒才放弃」是排查的线索。 #[test] @@ -3286,6 +3324,7 @@ mod failure_tests { let (_d, mut r) = rec(); r.on_event(&started(1, "claude-sonnet-4-5")); r.on_event(&Event::RequestFailed { + tokens_per_sec: None, id: 1, model: String::new(), source: tw_api::FailureSource::RateLimited, @@ -3356,6 +3395,7 @@ mod failure_tests { r.on_event(&start(2)); r.on_event(&headers(2, 400)); r.on_event(&Event::RequestFailed { + tokens_per_sec: None, id: 2, model: String::new(), source: tw_api::FailureSource::Upstream, @@ -3516,6 +3556,7 @@ mod settle_hook_tests { answered_model: None, }); r.on_event(&Event::RequestFailed { + tokens_per_sec: None, id: 3, model: String::new(), source: tw_api::FailureSource::Upstream, @@ -3555,6 +3596,7 @@ mod settle_hook_tests { let (_d, mut r, seen) = hooked(); r.on_event(&started(1, "claude-sonnet-4-5")); r.on_event(&Event::RequestFailed { + tokens_per_sec: None, id: 1, model: String::new(), source: tw_api::FailureSource::Denied, diff --git a/docs/config.md b/docs/config.md index 20d2c4ca..4eb405f2 100644 --- a/docs/config.md +++ b/docs/config.md @@ -441,6 +441,7 @@ Upstreams: the APIs requests are forwarded to. | `pricing` | string | — | Name of a price sheet under `pricing.sheets`. Unset: the default price table. | | `model_specs` | map of model id → [`providers[].model_specs.*`](#cfg-providers-model_specs) | `{}` | Context window, output limit, reasoning and image input of single models of this upstream, written by hand, by exact model id. They take precedence over the price table: for models it does not know, or gets wrong. | | `max_concurrent` | integer | — | Most requests sent to this upstream at the same time, from 1 to 1000. When it is full, a conversation that stays on it waits for a free slot and other requests go to the next upstream; see `failover.wait_secs`. Unset: no limit. | +| `min_speed` | object, [`providers[].min_speed`](#cfg-providers-min_speed) | — | End a streamed answer that generates too slowly: from the first token on, once a second, when the average speed over the last `window_secs` is below `tokens_per_sec`, the answer ends with an error clients retry (503, `overloaded_error` for Anthropic clients), and the upstream counts one failure toward `failover.failures_to_pause`. Pauses of 5 seconds or more partway through are left out of the average; the idle timeout deals with them. Answers that are not streamed are not judged. Unset: never. | | `limits` | list of [`providers[].limits[]`](#cfg-providers-limits) | — | Usage limits of this upstream, written like its own quota: requests, tokens or cost every few minutes or hours, per day, week or month. Every one has to have room before a request is sent to it. When a rolling window is full, requests go to the next upstream or wait for room; when a day, week or month is used up, requests go to the next upstream until the next one starts. Unset: no limit. | | `disabled` | bool | `false` | Take the upstream out of routing and out of the model list, and keep its configuration. | | `balance` | `auto` \| `off` \| `openrouter` \| `deepseek` \| `moonshot` \| `sub2api` \| `newapi` \| `thinkwatch` | `auto` | Where the upstream's balance is read from, with its own key and proxy: `openrouter`, `deepseek`, `moonshot`, `sub2api`, `newapi` (New API and One API) or `thinkwatch` (a ThinkWatch Enterprise gateway). `auto`: recognized from `base_url` for OpenRouter, DeepSeek and Moonshot; any other address, except Anthropic's, OpenAI's and Gemini's own, is asked once whether it is a Sub2API, ThinkWatch Enterprise or New API relay, and the answer is kept in memory only. `off`: no balance. ChatGPT and Z.ai accounts and Bedrock have no balance whatever is written here. | @@ -530,6 +531,45 @@ Responses WebSocket connection, each `response.create` takes a slot from when it is sent until its answer ends, and so does a key's `max_concurrent`; an idle connection takes none. +#### `providers[].min_speed` + +A self-hosted model sometimes drops to a few tokens per second and stays +there. Content keeps arriving, so the idle timeout never fires, and retrying +usually brings the speed back. With `min_speed`, a streamed answer is judged +once a second from its first token on: when the average speed over the last +`window_secs` seconds is below `tokens_per_sec`, the gateway ends the answer +with an error the client can retry (503: `overloaded_error` for Anthropic +clients, `response.failed` with `server_error` for Responses clients, +`UNAVAILABLE` for Gemini clients) and closes the connection to the upstream. + +The speed is counted the way the live speed in the app is: text, reasoning and +tool arguments, estimated and calibrated against what the upstream reports. +Pauses of 5 seconds or more partway through are left out, since the idle +timeout deals with them; when what is left of the window is under half of it, +nothing is judged that second. The wait before the first token is not judged +either, nor are answers that are not streamed. + +The request is recorded as failed with the speed it had, the part already +generated is billed, and the upstream counts one failure toward +`failover.failures_to_pause`, so a retry goes to another upstream first when +there is one. + + + + +| Field | Type | Default | Description | +|---|---|---|---| +| `tokens_per_sec` | integer | **required** | The lowest acceptable speed, in tokens per second, counted the way the live speed in the app is. From 1 to 1000. | +| `window_secs` | integer | `20` | How many seconds the average is taken over. The first one is taken this long after the first token. From 5 to 120. | + + +```yaml +providers: + - name: local-vllm + base_url: http://127.0.0.1:8000/v1 + min_speed: { tokens_per_sec: 15, window_secs: 20 } +``` + #### `providers[].limits` Usage limits for an upstream, written like the quota the upstream itself diff --git a/docs/config.zh-CN.md b/docs/config.zh-CN.md index 4d137298..1c32d0e6 100644 --- a/docs/config.zh-CN.md +++ b/docs/config.zh-CN.md @@ -313,6 +313,7 @@ clients: | `pricing` | 字符串 | — | `pricing.sheets` 中某张价目表的名字。不写:默认价目表。 | | `model_specs` | 映射: 模型 ID → [`providers[].model_specs.*`](#cfg-providers-model_specs) | `{}` | 手写这家上游某些模型的上下文窗口、输出上限、会不会推理、收不收图,按模型 ID 完全匹配。写了就优先于价目表,用于价目表里没有或写错的模型。 | | `max_concurrent` | 整数 | — | 同时发给这家的请求最多几个,取值 1 到 1000。满了的时候,留在这家的对话等空位,别的请求换下一家;等多久见 `failover.wait_secs`。不写:不限。 | +| `min_speed` | 对象,见 [`providers[].min_speed`](#cfg-providers-min_speed) | — | 流式回答生成得太慢时结束它:从第一个 token 起每秒看一次,最近 `window_secs` 秒的平均速度低于 `tokens_per_sec` 时,回答以一条客户端会重试的错误结束(503,Anthropic 格式的客户端收到 `overloaded_error`),这家记一次失败,计入 `failover.failures_to_pause`。中途 5 秒以上的停顿不算进平均速度,由无响应超时处理。不是流式的回答不管。不写:不管。 | | `limits` | 对象列表,见 [`providers[].limits[]`](#cfg-providers-limits) | — | 这家上游的用量上限,照着它自己的限额写:每几分钟、每几小时、每天、每周或每月的请求数、token 数或费用。每一条都有余量才往这家发。滚动窗口满了,请求换下一家或者等它空出来;天、周、月用满了,到下一期之前请求换下一家。不写:不限。 | | `disabled` | 布尔 | `false` | 不参与路由,模型也不出现在模型列表里;配置原样保留。 | | `balance` | `auto` \| `off` \| `openrouter` \| `deepseek` \| `moonshot` \| `sub2api` \| `newapi` \| `thinkwatch` | `auto` | 从哪里读这家上游的余额,用它自己的密钥、走它自己的代理:`openrouter`、`deepseek`、`moonshot`、`sub2api`、`newapi`(New API 和 One API)或 `thinkwatch`(ThinkWatch 企业网关)。`auto`:OpenRouter、DeepSeek、Moonshot 按 `base_url` 识别;其他地址(Anthropic、OpenAI、Gemini 的官方地址除外)询问一次它是不是 Sub2API、ThinkWatch 企业网关或 New API 中转站,结果只保存在内存中。`off`:不读余额。ChatGPT、Z.ai 账号和 Bedrock 不读余额,与这里写什么无关。 | @@ -355,6 +356,37 @@ ChatGPT 账号上游(`protocol: chatgpt`)只接受桌面应用登录得到 有的中转站和账号同时只接受几个请求,多出来的直接拒绝。`max_concurrent` 让网关守住这个数:请求发出时占用这家的一个位置,回答完整交给客户端、或者客户端断开时归还。这家满了的时候,为复用提示缓存而留在这家的对话等空位,别的请求直接换下一家。最多等多久由 `failover.wait_secs` 决定。等待不算失败,这家不会因此停用。只计算 token 数的请求不占位置。Responses 的 WebSocket 连接上,每个 `response.create` 从发出起占一个位置,直到它的回答结束,密钥的 `max_concurrent` 也一样;空闲的连接不占位置。 +#### `providers[].min_speed` + +自部署的模型偶尔会掉到每秒几个 token 并一直这样。内容一直在来,无响应超时不会触发,而重试 +通常就能恢复正常速度。写了 `min_speed`,流式回答从第一个 token 起每秒判断一次:最近 +`window_secs` 秒的平均速度低于 `tokens_per_sec` 时,网关以一条客户端可以重试的错误结束回答 +(503:Anthropic 格式的客户端收到 `overloaded_error`,Responses 格式收到带 `server_error` 的 +`response.failed`,Gemini 格式收到 `UNAVAILABLE`),并断开和上游的连接。 + +速度和应用里显示的实时速度同一个算法:正文、推理和工具参数,按估算并用上游报的实数校准。 +中途 5 秒以上的停顿不算在内,由无响应超时处理;扣掉之后剩下的不到窗口的一半时,这一秒不判断。 +第一个 token 之前的等待不判断,不是流式的回答也不判断。 + +请求记为失败并记下当时的速度,已经生成的部分照常计费,这家记一次失败,计入 +`failover.failures_to_pause`;有别的候选时,重试的请求先去别家。 + + + + +| 字段 | 类型 | 默认值 | 说明 | +|---|---|---|---| +| `tokens_per_sec` | 整数 | **必填** | 最低生成速度,单位 token/秒,和应用里显示的实时速度同一个算法。取值 1 到 1000。 | +| `window_secs` | 整数 | `20` | 按最近多少秒的平均速度判断。第一次判断在第一个 token 之后这么久。取值 5 到 120。 | + + +```yaml +providers: + - name: local-vllm + base_url: http://127.0.0.1:8000/v1 + min_speed: { tokens_per_sec: 15, window_secs: 20 } +``` + #### `providers[].limits` 一家上游的用量上限,照着这家自己的限额写:每几分钟、每几小时、每天、每周或每月最多多少个