diff --git a/CHANGELOG.md b/CHANGELOG.md index 24fc63f82d..7f12b2eac8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -661,13 +661,14 @@ silently lost every counter. Reaching a recorder is no longer representable from host accepts only the current one. A sink that wants the observability back-channel implements one defaulted SDK method, `drain_observations`. -**A new `plugins.logs` block is accepted and checked; plugin log lines have not moved.** The block +**A new `plugins.logs` block is accepted and checked, and plugin log lines move to per-plugin log files.** The block names the directory for per-plugin log files (`dir`, default `logs/plugins`), the default level (`level`, default `info`), per-instance levels (`levels`), and rotation (`rotate_mb` and `keep`). -A level word that names no level, or a `rotate_mb` of `0`, refuses the boot and names the key. This -build writes no per-plugin log file: the plugins it loads do not run through the path that writes -them, so a plugin's log lines reach the Busbar log exactly as they did in 1.5.5, and setting -`plugins.logs` changes nothing an operator can observe beyond that check. +A level word that names no level, or a `rotate_mb` of `0`, refuses the boot and names the key. + +- 1.6.0 Changed: a plugin's log lines, the LLM plane's included (it is a plugin in 1.6.0), are written to that plugin's own log file under `plugins.logs.dir` (for example `logs/plugins/llm.log`), not to the Busbar log. + In 1.5.5 the plane's warnings (for example `dropping image_url.detail: ...`) were lines of the + Busbar log. **Migration:** point log collection at `plugins.logs.dir` as well as the Busbar log. ### Spec fidelity diff --git a/Cargo.lock b/Cargo.lock index dbaadb6801..fc2bc2c5d3 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1459,7 +1459,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb" dependencies = [ "libc", - "windows-sys 0.52.0", + "windows-sys 0.61.2", ] [[package]] @@ -3277,7 +3277,7 @@ dependencies = [ "errno", "libc", "linux-raw-sys", - "windows-sys 0.52.0", + "windows-sys 0.61.2", ] [[package]] @@ -3715,7 +3715,7 @@ dependencies = [ "getrandom 0.4.3", "once_cell", "rustix", - "windows-sys 0.52.0", + "windows-sys 0.61.2", ] [[package]] @@ -4401,7 +4401,7 @@ version = "0.1.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c2a7b1c03c876122aa43f3020e6c3c3ee5c05081c9a00739faf7503aeba10d22" dependencies = [ - "windows-sys 0.52.0", + "windows-sys 0.61.2", ] [[package]] diff --git a/crates/busbar-contract/include/busbar_plugin.h b/crates/busbar-contract/include/busbar_plugin.h index daea6e0dac..d94316427c 100644 --- a/crates/busbar-contract/include/busbar_plugin.h +++ b/crates/busbar-contract/include/busbar_plugin.h @@ -417,6 +417,7 @@ extern "C" { #define BB_PLANE_AUDIT_NONE UINT32_C(0) /* [`ServeOut::audit`]: no audit row. */ #define BB_PLANE_AUDIT_APPLIED UINT32_C(1) /* [`ServeOut::audit`]: the request applied; the row's outcome is `applied`. */ #define BB_PLANE_AUDIT_REJECTED UINT32_C(2) /* [`ServeOut::audit`]: the request was rejected after it was judged; the row's outcome is */ +#define BB_PLANE_AUDIT_DEGRADED UINT32_C(3) /* A [`RECORD_AUDIT`] write's outcome only (never [`ServeOut::audit`]): the unit was served with */ #define BB_PLANE_CHAIN_LENGTH_PREFIXED UINT32_C(1) /* [`RecordChain::framing`]: each field of the record's digest is length-prefixed. */ #define BB_PLANE_CHAIN_PIPE_SEPARATED UINT32_C(2) /* [`RecordChain::framing`]: the fields of the record's digest are joined by `|`. */ #define BB_PLANE_CHAIN_DIGESTS_SCOPE UINT32_C(1) /* [`RecordChain::flags`]: the record's scope enters its digest. */ diff --git a/crates/busbar-contract/src/abi/plane/check.rs b/crates/busbar-contract/src/abi/plane/check.rs index e525b744a2..a7f079fe1a 100644 --- a/crates/busbar-contract/src/abi/plane/check.rs +++ b/crates/busbar-contract/src/abi/plane/check.rs @@ -151,7 +151,7 @@ fn records(buf: &[RecordWrite], n: u64, arena: u64, b: &Bounds) -> Result<(), Fa code( u64::from(r.kind), u64::from(super::AUDIT_APPLIED), - u64::from(super::AUDIT_REJECTED), + u64::from(super::AUDIT_DEGRADED), "record.audit_outcome", )?; if r.key.len == 0 { diff --git a/crates/busbar-contract/src/abi/plane/mod.rs b/crates/busbar-contract/src/abi/plane/mod.rs index 841224f904..4dd83bfe6e 100644 --- a/crates/busbar-contract/src/abi/plane/mod.rs +++ b/crates/busbar-contract/src/abi/plane/mod.rs @@ -591,6 +591,10 @@ pub const AUDIT_APPLIED: u32 = 1; /// [`ServeOut::audit`]: the request was rejected after it was judged; the row's outcome is /// `rejected`. pub const AUDIT_REJECTED: u32 = 2; +/// A [`RECORD_AUDIT`] write's outcome only (never [`ServeOut::audit`]): the unit was served with +/// something the caller asked for left out (a control the far end's form cannot carry); the row's +/// outcome is `degraded`, the 1.5.5 row's. +pub const AUDIT_DEGRADED: u32 = 3; /// [`RecordChain::framing`]: each field of the record's digest is length-prefixed. pub const CHAIN_LENGTH_PREFIXED: u32 = 1; diff --git a/crates/busbar-contract/src/abi/tests/plane_check_tests.rs b/crates/busbar-contract/src/abi/tests/plane_check_tests.rs index 08fe973c61..9cdf013569 100644 --- a/crates/busbar-contract/src/abi/tests/plane_check_tests.rs +++ b/crates/busbar-contract/src/abi/tests/plane_check_tests.rs @@ -700,7 +700,7 @@ fn an_audit_record_names_its_outcome_and_action() { record_kinds: 0, ..bounds() }; - for outcome in [AUDIT_APPLIED, AUDIT_REJECTED] { + for outcome in [AUDIT_APPLIED, AUDIT_REJECTED, AUDIT_DEGRADED] { r.kind = outcome; assert_eq!( check_on_piece(Ready, &o, (&[], &[r], &[]), &caps(), &none), @@ -708,7 +708,7 @@ fn an_audit_record_names_its_outcome_and_action() { "outcome {outcome}" ); } - for outcome in [AUDIT_NONE, AUDIT_REJECTED + 1, u32::MAX] { + for outcome in [AUDIT_NONE, AUDIT_DEGRADED + 1, u32::MAX] { r.kind = outcome; assert_eq!( piece(&o, &[], &[r], &[]), diff --git a/crates/busbar-contract/src/plane_calls.rs b/crates/busbar-contract/src/plane_calls.rs index f3e80113e3..5c1579c895 100644 --- a/crates/busbar-contract/src/plane_calls.rs +++ b/crates/busbar-contract/src/plane_calls.rs @@ -270,6 +270,9 @@ pub struct PlaneRegistration { pub trust_keys: Vec, /// The tail's sentence refusing a forwarded caller credential; `None` = it states none. pub caller_credential_refusal: Option<&'static str>, + /// Whether the tail states `TAIL_FALLBACK`: the plane is the catch-all every unclaimed path + /// falls through to, and its card is the flat one (at most one registered plane). + pub fallback: bool, /// The door's own `validate` over a whole section (its settings, JSON): `Ok`, or the door's /// words. The kernel runs it where the section is parsed and where an admin write lands. pub validate: SectionJudge, diff --git a/crates/busbar-kernel-egress/src/lib.rs b/crates/busbar-kernel-egress/src/lib.rs index df1457939a..a9aaa09cc7 100644 --- a/crates/busbar-kernel-egress/src/lib.rs +++ b/crates/busbar-kernel-egress/src/lib.rs @@ -74,7 +74,7 @@ pub use pool::{ Failover, Member, OnExhausted, Pool, DEFAULT_FAILOVER_CAP, DEFAULT_FAILOVER_DEADLINE_SECS, }; pub use select::{RequestCtx, WeightedFloor}; -pub use walk::{Step, Taken, Walk, WalkPorts}; +pub use walk::{Shape, Step, Taken, Walk, WalkPorts}; pub use wire::Shed; #[cfg(test)] diff --git a/crates/busbar-kernel-egress/src/ports.rs b/crates/busbar-kernel-egress/src/ports.rs index 8ce566cf94..97b68294cc 100644 --- a/crates/busbar-kernel-egress/src/ports.rs +++ b/crates/busbar-kernel-egress/src/ports.rs @@ -383,6 +383,14 @@ pub trait Clock: Send + Sync { /// seconds, and a budget close to a second boundary would collapse to zero. fn now_millis(&self) -> u128; + /// Microseconds since the same fixed point, for a latency sample: 1.5.5 measured an + /// attempt's latency in fractional milliseconds, so a sub-millisecond answer still records + /// (an integer `0` is the EWMA's "no sample"). A clock without the precision answers its + /// milliseconds scaled. + fn now_micros(&self) -> u128 { + self.now_millis().saturating_mul(1000) + } + /// A future that completes no earlier than `ms` milliseconds from now. fn sleep(&self, ms: u64) -> BoxFut<'_, ()>; } @@ -395,6 +403,10 @@ pub trait Telemetry: Send + Sync { /// One upstream attempt was started against this member. fn upstream_attempt(&self, pool: &str, destination: DestinationId); + /// A served attempt's time to the upstream's response head, in milliseconds (the destination's + /// latency signal, a bounded proxy that never waits out a streamed body). + fn upstream_latency(&self, _destination: DestinationId, _ms: f64) {} + /// One attempt failed, with the disposition label the previous release used. fn upstream_failure(&self, pool: &str, destination: DestinationId, disposition: &'static str); diff --git a/crates/busbar-kernel-egress/src/walk.rs b/crates/busbar-kernel-egress/src/walk.rs index d239d49cd4..01c14993ef 100644 --- a/crates/busbar-kernel-egress/src/walk.rs +++ b/crates/busbar-kernel-egress/src/walk.rs @@ -54,6 +54,21 @@ pub struct WalkPorts<'p> { pub pools: &'p HashMap, } +/// THE SHAPE A UNIT'S HOOKS PUT ON ITS PICKS (the design's route pump, "Hooks": the routing-stage policy +/// keeps and orders the routed pool's members; a required restrict holds a pool the walk spills +/// into): each pick's admissible members, the ones kept, and the order the walk prefers among them +/// (empty = the weighted floor's own). `None` = a required restriction no member satisfies: the +/// walk answers [`Shed::restrict_no_lane`] from there. +pub trait Shape { + /// `members` of a pick, as the unit's hooks leave them: the walk's own pool (`primary`) or a + /// pool it spills into. + fn shape( + &self, + members: Vec, + primary: bool, + ) -> Option<(Vec, Vec)>; +} + /// The walk's whole budget over `pool`, seconds: the pool's request timeout, `0` for a pool that is /// not configured. #[must_use] @@ -184,6 +199,17 @@ impl Walk { &self.ctx } + /// The pool the walk is in and whether it is the walk's own (`true`) rather than a spill; + /// `None` once every path is spent. + #[must_use] + pub fn walking(&self) -> Option<(&str, bool)> { + match &self.phase { + Phase::Primary(name) => Some((name, true)), + Phase::Spill(name) | Phase::Terminal(name) => Some((name, false)), + Phase::Shed(_) => None, + } + } + /// The walk ends here, before its pool's terminal: every step from now answers `shed`. pub fn refuse(&mut self, shed: Shed) { self.phase = Phase::Shed(shed); @@ -203,6 +229,17 @@ impl Walk { ports: &WalkPorts<'_>, affinity: Option, token: &Pass, + ) -> Step { + self.next_shaped(ports, affinity, None, token) + } + + /// [`Self::next`], each pick's members as the unit's hooks `shape` them. + pub fn next_shaped( + &mut self, + ports: &WalkPorts<'_>, + affinity: Option, + shape: Option<&dyn Shape>, + token: &Pass, ) -> Step { loop { if self.ctx.expired(ports.clock.now_secs()) { @@ -218,9 +255,13 @@ impl Walk { return Step::Shed(Shed::empty_pool()); } if self.hops <= pool.failover.max_hops { - if let Some(taken) = self.pick(ports, pool, affinity, token, true) { - self.hops += 1; - return Step::Take(taken); + match self.pick(ports, pool, affinity, shape, token, true) { + Ok(Some(taken)) => { + self.hops += 1; + return Step::Take(taken); + } + Ok(None) => {} + Err(shed) => return Step::Shed(shed), } } self.phase = Phase::Terminal(name); @@ -229,8 +270,10 @@ impl Walk { let Some(pool) = ports.pools.get(&name) else { return Step::Shed(pool_shed(ports, &name, token)); }; - if let Some(taken) = self.pick(ports, pool, None, token, false) { - return Step::Take(taken); + match self.pick(ports, pool, None, shape, token, false) { + Ok(Some(taken)) => return Step::Take(taken), + Ok(None) => {} + Err(shed) => return Step::Shed(shed), } self.phase = Phase::Terminal(name); } @@ -285,17 +328,34 @@ impl Walk { /// THE ONE PICK, over `pool`'s members this unit has not tried: the walk's own pool /// (`primary`, with the unit's affinity) and every pool it spills into (degraded). + /// + /// # Errors + /// + /// The unit's hooks hold the pool to a restriction no member satisfies: that shed, which every + /// step from here answers. fn pick( &mut self, ports: &WalkPorts<'_>, pool: &Pool, affinity: Option, + shape: Option<&dyn Shape>, token: &Pass, primary: bool, - ) -> Option { - let members = pool.admissible_members(); + ) -> Result, Shed> { + let admissible = pool.admissible_members().into_owned(); + let (members, order) = match shape { + Some(shape) => match shape.shape(admissible, primary) { + Some(shaped) => shaped, + None => { + let shed = Shed::restrict_no_lane(); + self.phase = Phase::Shed(shed.clone()); + return Err(shed); + } + }, + None => (admissible, Vec::new()), + }; let now = ports.clock.now_secs(); - let mut picked = pick_among( + let Some(mut picked) = pick_among( &PickInput { breaker: ports.breaker, capacity: ports.capacity, @@ -303,18 +363,29 @@ impl Walk { pool: &pool.name, members: &members, affinity, - preference: None, + preference: (!order.is_empty()).then_some(order.as_slice()), now, token, }, &mut self.ctx, - )?; - let member = members + ) else { + return Ok(None); + }; + let Some(member) = members .iter() - .find(|m| m.destination == picked.destination)? - .clone(); + .find(|m| m.destination == picked.destination) + .cloned() + else { + return Ok(None); + }; let probe = picked.take_probe_epoch(); - Some(self.take(&pool.name, member, picked.permit, probe, !primary)) + Ok(Some(self.take( + &pool.name, + member, + picked.permit, + probe, + !primary, + ))) } /// Take `member` of `pool` for the next attempt: excluded from every pick of the unit after it. diff --git a/crates/busbar-kernel/src/appbuild.rs b/crates/busbar-kernel/src/appbuild.rs index c046cd5d62..072c511a06 100644 --- a/crates/busbar-kernel/src/appbuild.rs +++ b/crates/busbar-kernel/src/appbuild.rs @@ -540,6 +540,19 @@ impl InstalledLimits { } } +/// WHAT THE COMPOSITION ROOT KEEPS OF A RESOLVED CONFIGURATION beside the generation built from it +/// (spec Part 1 line 569: a door plane is handed one validated `{section: value}` object per +/// `open`/`refresh`): the root projects the sections, providers and pool bounds its door planes are +/// sealed over, and reads the projection back when the generation is swapped in. +pub type ConfigProjection = fn(&config::RootCfg) -> std::sync::Arc; + +static CONFIG_PROJECTION: std::sync::OnceLock = std::sync::OnceLock::new(); + +/// Bind the composition root's [`ConfigProjection`], once, before the first build. +pub fn set_config_projection(project: ConfigProjection) { + let _ = CONFIG_PROJECTION.set(project); +} + #[cold] // boot/admin-only — keeps hot text dense (never inlined into a warm path) #[inline(never)] pub fn build_app_from_config( @@ -551,6 +564,8 @@ pub fn build_app_from_config( config_paths: (Option, Option), prior: Option<&state::App>, ) -> Result<(state::App, Option, InstalledLimits), String> { + // The root's projection of this configuration, taken before any part of it is consumed. + let config_projection = CONFIG_PROJECTION.get().map(|project| project(&cfg)); // Install the resolved operational limits process-wide BEFORE any subsystem reads them — // running here (not in main) so a config APPLY/RELOAD refreshes them too. The values threaded // explicitly (client/store/router/TLS) read `cfg.limits` directly; the deep call-stack sites @@ -1644,6 +1659,14 @@ pub fn build_app_from_config( // Populate the NEUTRAL carrier field-by-field from the already-resolved config (pre-resolved secret // plaintexts + rate-card-derived costs + resolved context/tokens are in `lane_inputs`/`pool_inputs`). + // THE KERNEL'S OWN TABLES over the same resolved sections, for the read seam when no plane + // contributes a runtime of its own. + let config_tables = Arc::new(busbar_kernel::plane_host::ConfigTables::of( + &lane_inputs, + &pool_inputs, + &by_model, + cfg.upstream_credentials, + )); let fallback_build_input = PlaneBuildInput { lanes: lane_inputs, pools: pool_inputs, @@ -1757,6 +1780,7 @@ pub fn build_app_from_config( // read on the hook seam with a single relaxed load. let app = App { + config_projection, plane_sections: Some(cfg.plane_sections.clone()), // Telemetry-bank slot table for this generation (built above as a local, BEFORE the // config-derived collections moved into the fallback plane's runtime bundle so its @@ -1768,6 +1792,7 @@ pub fn build_app_from_config( // seam; the snapshot names only the `&'static str` key, and `App::llm_runtime` downcasts the // slot on the money path. Absent slot (featureless build) reads the empty default. fallback_runtime_key, + config_tables, store, // The container planes' breaker cells: PROCESS-LIFETIME, reused across an apply/reload the // way the HTTP client pool and governance state are — a config swap must not un-trip a diff --git a/crates/busbar-kernel/src/core_routes.rs b/crates/busbar-kernel/src/core_routes.rs index 056700c895..45c97bde5d 100644 --- a/crates/busbar-kernel/src/core_routes.rs +++ b/crates/busbar-kernel/src/core_routes.rs @@ -191,6 +191,15 @@ impl CoreRouter { self } + /// The methods a core route is already mounted under at exactly `path`. + pub(crate) fn methods_at(&self, path: &str) -> Vec { + self.table + .by_path + .get(path) + .map(|rows| rows.iter().map(|(m, _)| *m).collect()) + .unwrap_or_default() + } + /// Split into the mounted router and the table describing it. pub(crate) fn into_parts(self) -> (Router>, CoreRouteTable) { (self.router, self.table) diff --git a/crates/busbar-kernel/src/export/traces.rs b/crates/busbar-kernel/src/export/traces.rs index 4ab6e590ee..c50cb71a0a 100644 --- a/crates/busbar-kernel/src/export/traces.rs +++ b/crates/busbar-kernel/src/export/traces.rs @@ -59,9 +59,13 @@ fn record(projection: Projection, facts: &[(F, Value)]) -> Arc { Arc::new(rec.finish()) } -/// A span as it opened: when (monotonic, and epoch microseconds), and the stream's fields it -/// carries. -struct Opened(Instant, u64, Fields); +/// A span as it opened: when (monotonic, and epoch microseconds), the stream's fields it carries, +/// and its own id for the export, unique for the process (the subscriber's span ids are reused +/// once a span closes, so two requests' spans would otherwise share an id and a trace). +struct Opened(Instant, u64, Fields, u64); + +/// The next export id: one per span opened, never reused, never zero. +static NEXT_ID: std::sync::atomic::AtomicU64 = std::sync::atomic::AtomicU64::new(1); /// The `traces` fields a span records, by the field's own name — and only those: a span field that /// shares a name with another stream's field (`key_id`, say) is not a traces fact. @@ -87,24 +91,37 @@ impl LookupSpan<'a>> Layer for Producer { let mut fields = Fields(Vec::new()); attrs.record(&mut fields); let epoch = UNIX_EPOCH.elapsed().unwrap_or_default().as_micros() as u64; + let own = NEXT_ID.fetch_add(1, std::sync::atomic::Ordering::Relaxed); span.extensions_mut() - .insert(Opened(Instant::now(), epoch, fields)); + .insert(Opened(Instant::now(), epoch, fields, own)); } /// The span's record: its facts, its parent's id, its ROOT's id as the trace id (a root is its /// own trace), its own id and name, when it opened and how long it stayed open. fn on_close(&self, id: Id, ctx: Context<'_, S>) { let Some(span) = ctx.span(&id) else { return }; - let Some(Opened(start, epoch, Fields(mut facts))) = span.extensions_mut().remove() else { + let Some(Opened(start, epoch, Fields(mut facts), own)) = span.extensions_mut().remove() + else { return; }; - let hex = |id: &Id| Value::from(format!("{:016x}", id.into_u64())); - let parent = span.parent().map(|p| (F::ParentSpanId, hex(&p.id()))); - let root = span.scope().last().map_or(id.clone(), |r| r.id()); + let hex = |id: u64| Value::from(format!("{id:016x}")); + // A span's export id, as it opened (`Opened`); one opened before this layer saw it keeps + // the subscriber's id. + let export_id = |s: &tracing_subscriber::registry::SpanRef<'_, S>| { + s.extensions() + .get::() + .map_or_else(|| s.id().into_u64(), |o| o.3) + }; + let parent = span.parent().map(|p| (F::ParentSpanId, hex(export_id(&p)))); + let root = span + .scope() + .last() + .filter(|r| r.id() != id) + .map_or(own, |r| export_id(&r)); let lasted = start.elapsed().as_micros() as u64; facts.extend(parent.into_iter().chain([ - (F::TraceId, hex(&root)), - (F::SpanId, hex(&id)), + (F::TraceId, hex(root)), + (F::SpanId, hex(own)), (F::Name, Value::from(span.name())), (F::Start, Value::from(epoch)), (F::DurationUs, Value::from(lasted)), diff --git a/crates/busbar-kernel/src/ingress/arrival.rs b/crates/busbar-kernel/src/ingress/arrival.rs index 1b07a3405d..f54f226a2e 100644 --- a/crates/busbar-kernel/src/ingress/arrival.rs +++ b/crates/busbar-kernel/src/ingress/arrival.rs @@ -275,6 +275,19 @@ fn test_body_ingress() -> &'static [BodyIngressEntry] { TEST_BODY_INGRESS_HOOK.get().map(|f| f()).unwrap_or(&[]) } +/// Whether any body-model arrival is installed (a plane registered its arrivals on the body axis): +/// the convenience surfaces that hand their unit to one (`/{name}/v1/messages`, +/// `/{provider}/{model}/v1/messages`) are mounted only then; a plane served through its door +/// claims those paths itself. +#[must_use] +pub fn any_body_ingress() -> bool { + #[cfg(any(test, feature = "test-support"))] + if !test_body_ingress().is_empty() { + return true; + } + INSTALLED_BODY_INGRESS.get().is_some_and(|a| !a.is_empty()) +} + /// RESOLVE A BODY-MODEL DIALECT'S UNIVERSAL ARRIVAL BY NAME — the lookup `protocol_dispatch` performs /// for a body-model protocol. Consults the installed table first, then the test hook. `None` when no /// LLM plane is linked (core booted plane-agnostic): the caller then answers the honest no-handler diff --git a/crates/busbar-kernel/src/ingress/mod.rs b/crates/busbar-kernel/src/ingress/mod.rs index ac56b4700d..034a1a9799 100644 --- a/crates/busbar-kernel/src/ingress/mod.rs +++ b/crates/busbar-kernel/src/ingress/mod.rs @@ -189,116 +189,7 @@ pub fn admit_check( // bucket label, not an internal credential handle. Server-side tracing records the // full detail either way. tracing::info!(key_id = %key.id, blocked = ?blocked, "governance: limit bucket blocked admission"); - use crate::governance::LimitBlocked; - let (status, kind, message, retry_after) = match &blocked { - LimitBlocked::Limit { - group, - // The per-tier token caps (`tokens_input`/…) surface as a rate limit exactly like - // the aggregate `tokens` metric — a 429 naming the tier — NOT as an over-quota - // block. Without this arm they would fall to the budget/quota arm below and return - // the vendor quota status (Bedrock 400), silently mislabelling the block. - metric: - metric @ ("requests" | "tokens" | "tokens_input" | "tokens_output" - | "tokens_cache_read" | "tokens_cache_write"), - window, - pool: limit_pool, - retry_after, - .. - } => ( - StatusCode::TOO_MANY_REQUESTS, - crate::proxy::KIND_RATE_LIMIT, - format!( - "Rate limit exceeded (group '{group}': {metric} per {}{}). Please retry \ - after the indicated time.", - window.unwrap_or("total"), - pool_scope_suffix(limit_pool), - ), - *retry_after, - ), - LimitBlocked::Limit { - group, - metric: "concurrent", - .. - } => ( - StatusCode::TOO_MANY_REQUESTS, - crate::proxy::KIND_RATE_LIMIT, - format!( - "Too many concurrent requests (group '{group}' is at its in-flight \ - limit). Please retry shortly." - ), - None, - ), - LimitBlocked::Limit { - group, - metric: "budget", - window, - pool: limit_pool, - retry_after, - .. - } => ( - // Native quota status differs by vendor (Bedrock's - // ServiceQuotaExceededException is 400; every other vendor surfaces - // over-quota as 429). The writer owns that mapping. - crate::proto::decl_for(proto) - .map(|d| d.quota_exceeded_status) - .unwrap_or(StatusCode::TOO_MANY_REQUESTS), - crate::proxy::KIND_INSUFFICIENT_QUOTA, - format!( - "You have exceeded your current quota (group '{group}' budget per {}{} \ - exhausted). Please check your plan and billing details.", - window.unwrap_or("total"), - pool_scope_suffix(limit_pool), - ), - *retry_after, - ), - // FAIL-SAFE catch-all for any FUTURE metric not yet given an explicit arm above. It - // maps to a generic 429 rate limit — never the vendor quota status — so a new metric - // can never silently inherit `budget`'s Bedrock-400 semantics. Every metric that exists - // today (`requests`/`tokens`/the four `tokens_*` tiers/`concurrent`/`budget`) is matched - // explicitly above; this arm only exists to keep the string match exhaustive. - LimitBlocked::Limit { - group, - metric, - window, - pool: limit_pool, - retry_after, - .. - } => ( - StatusCode::TOO_MANY_REQUESTS, - crate::proxy::KIND_RATE_LIMIT, - format!( - "Rate limit exceeded (group '{group}': {metric} per {}{}). Please retry \ - after the indicated time.", - window.unwrap_or("total"), - pool_scope_suffix(limit_pool), - ), - *retry_after, - ), - // A FROZEN group (`enabled: false`) is an administrative freeze, not a quota: the - // vendor-plausible shape is a permission denial. - LimitBlocked::Disabled(group) => ( - StatusCode::FORBIDDEN, - crate::proxy::KIND_PERMISSION, - format!( - "Your API key does not currently have access to this resource (group \ - '{group}' is disabled)." - ), - None, - ), - // FAIL-CLOSED: a key bound to a group this node's config does not know is not - // admitted; the message names the missing bucket so the operator can fix it. - LimitBlocked::MissingGroup(group) => ( - crate::proto::decl_for(proto) - .map(|d| d.quota_exceeded_status) - .unwrap_or(StatusCode::TOO_MANY_REQUESTS), - crate::proxy::KIND_INSUFFICIENT_QUOTA, - format!( - "Your quota configuration is incomplete (group '{group}' is not \ - configured). Please contact your administrator." - ), - None, - ), - }; + let (status, kind, message, retry_after) = limit_refusal(proto, &blocked); let mut resp = ingress_error(proto, status, kind, &message); // Standard `Retry-After` for a rolling window so a well-behaved SDK backs off the // right amount ('total' never rolls: no header). @@ -312,6 +203,126 @@ pub fn admit_check( } } +/// THE LIMIT REFUSAL a blocked admission is answered with: its status in `proto`, its kind word, +/// its message (naming the group, metric and window, never the key), and its Retry-After seconds +/// (a rolling window's; `total` never rolls). One spelling, read by every door that admits. +#[must_use] +pub fn limit_refusal( + proto: &str, + blocked: &crate::governance::LimitBlocked, +) -> (StatusCode, &'static str, String, Option) { + use crate::governance::LimitBlocked; + match blocked { + LimitBlocked::Limit { + group, + // The per-tier token caps (`tokens_input`/…) surface as a rate limit exactly like + // the aggregate `tokens` metric — a 429 naming the tier — NOT as an over-quota + // block. Without this arm they would fall to the budget/quota arm below and return + // the vendor quota status (Bedrock 400), silently mislabelling the block. + metric: + metric @ ("requests" | "tokens" | "tokens_input" | "tokens_output" | "tokens_cache_read" + | "tokens_cache_write"), + window, + pool: limit_pool, + retry_after, + .. + } => ( + StatusCode::TOO_MANY_REQUESTS, + crate::proxy::KIND_RATE_LIMIT, + format!( + "Rate limit exceeded (group '{group}': {metric} per {}{}). Please retry \ + after the indicated time.", + window.unwrap_or("total"), + pool_scope_suffix(limit_pool), + ), + *retry_after, + ), + LimitBlocked::Limit { + group, + metric: "concurrent", + .. + } => ( + StatusCode::TOO_MANY_REQUESTS, + crate::proxy::KIND_RATE_LIMIT, + format!( + "Too many concurrent requests (group '{group}' is at its in-flight \ + limit). Please retry shortly." + ), + None, + ), + LimitBlocked::Limit { + group, + metric: "budget", + window, + pool: limit_pool, + retry_after, + .. + } => ( + // Native quota status differs by vendor (Bedrock's + // ServiceQuotaExceededException is 400; every other vendor surfaces + // over-quota as 429). The writer owns that mapping. + crate::proto::decl_for(proto) + .map(|d| d.quota_exceeded_status) + .unwrap_or(StatusCode::TOO_MANY_REQUESTS), + crate::proxy::KIND_INSUFFICIENT_QUOTA, + format!( + "You have exceeded your current quota (group '{group}' budget per {}{} \ + exhausted). Please check your plan and billing details.", + window.unwrap_or("total"), + pool_scope_suffix(limit_pool), + ), + *retry_after, + ), + // FAIL-SAFE catch-all for any FUTURE metric not yet given an explicit arm above. It + // maps to a generic 429 rate limit — never the vendor quota status — so a new metric + // can never silently inherit `budget`'s Bedrock-400 semantics. Every metric that exists + // today (`requests`/`tokens`/the four `tokens_*` tiers/`concurrent`/`budget`) is matched + // explicitly above; this arm only exists to keep the string match exhaustive. + LimitBlocked::Limit { + group, + metric, + window, + pool: limit_pool, + retry_after, + .. + } => ( + StatusCode::TOO_MANY_REQUESTS, + crate::proxy::KIND_RATE_LIMIT, + format!( + "Rate limit exceeded (group '{group}': {metric} per {}{}). Please retry \ + after the indicated time.", + window.unwrap_or("total"), + pool_scope_suffix(limit_pool), + ), + *retry_after, + ), + // A FROZEN group (`enabled: false`) is an administrative freeze, not a quota: the + // vendor-plausible shape is a permission denial. + LimitBlocked::Disabled(group) => ( + StatusCode::FORBIDDEN, + crate::proxy::KIND_PERMISSION, + format!( + "Your API key does not currently have access to this resource (group \ + '{group}' is disabled)." + ), + None, + ), + // FAIL-CLOSED: a key bound to a group this node's config does not know is not + // admitted; the message names the missing bucket so the operator can fix it. + LimitBlocked::MissingGroup(group) => ( + crate::proto::decl_for(proto) + .map(|d| d.quota_exceeded_status) + .unwrap_or(StatusCode::TOO_MANY_REQUESTS), + crate::proxy::KIND_INSUFFICIENT_QUOTA, + format!( + "Your quota configuration is incomplete (group '{group}' is not \ + configured). Please contact your administrator." + ), + None, + ), + } +} + /// Run the governance guards (pool ACL / unpriced-model / the atomic group-limit admission) for a /// request that is about to be forwarded. Returns the protocol-native rejection response already /// passed through `finish_rejected`. The statuses are deliberately vendor-faithful and never 402: diff --git a/crates/busbar-kernel/src/plane/door.rs b/crates/busbar-kernel/src/plane/door.rs index 114bcb7f2b..d795249749 100644 --- a/crates/busbar-kernel/src/plane/door.rs +++ b/crates/busbar-kernel/src/plane/door.rs @@ -116,7 +116,7 @@ pub fn fold(reg: PlaneRegistration) -> Result<&'static PlaneDecl, String> { fn declaration_of(reg: &PlaneRegistration) -> PlaneDeclaration { PlaneDeclaration { key: reg.key, - fallback: false, + fallback: reg.fallback, config_section: reg.section, scope_kinds: reg.scope_kinds.clone().leak(), subject_noun: reg.subject_noun, diff --git a/crates/busbar-kernel/src/plane/tests/door_tests.rs b/crates/busbar-kernel/src/plane/tests/door_tests.rs index 503c2c8f38..0eb460ea3e 100644 --- a/crates/busbar-kernel/src/plane/tests/door_tests.rs +++ b/crates/busbar-kernel/src/plane/tests/door_tests.rs @@ -58,6 +58,7 @@ fn registration(key: &'static str, section: &'static str) -> PlaneRegistration { }, ], caller_credential_refusal: Some("forwarding a caller credential is refused here"), + fallback: false, validate: Arc::new(move |bytes: &[u8]| { if bytes.is_empty() { return Ok(()); diff --git a/crates/busbar-kernel/src/plane_driver/cancel.rs b/crates/busbar-kernel/src/plane_driver/cancel.rs index fd46c72c61..d81660d8fd 100644 --- a/crates/busbar-kernel/src/plane_driver/cancel.rs +++ b/crates/busbar-kernel/src/plane_driver/cancel.rs @@ -106,6 +106,10 @@ pub(crate) struct Facts { /// The reply's head was stated to the caller (once: a head with no body may precede the /// answer's first byte). pub(crate) headed: bool, + /// The live attempt's answer is relayed as the unit's: a terminal's dispatch (a spill, the + /// least-bad bypass, a queued slot), which 1.5.5 never failed over, so a retry verdict on it + /// renders the answer instead. + pub(crate) relayed: bool, /// The last cumulative units the plane reported. pub(crate) units: Vec, } diff --git a/crates/busbar-kernel/src/plane_driver/far_end.rs b/crates/busbar-kernel/src/plane_driver/far_end.rs index c24b3aabd7..d49183e3f9 100644 --- a/crates/busbar-kernel/src/plane_driver/far_end.rs +++ b/crates/busbar-kernel/src/plane_driver/far_end.rs @@ -55,7 +55,7 @@ use crate::proxy::egress_unit::{ Journal, Outcome, Permit, Telemetry, UpstreamStatus, }, walk::budget_secs, - Member, Pool, Shed, Step, Taken, Walk, WalkPorts, WeightedFloor, + Member, Pool, Shape, Shed, Step, Taken, Walk, WalkPorts, WeightedFloor, }; use busbar_contract::abi::auth::{AuthPoint, AuthPoints, STYLE_NEEDS_HEADERS}; use busbar_contract::abi::transport::{ @@ -70,6 +70,7 @@ use busbar_contract::redacted::Redacted; use busbar_contract::transport::registry::status_ns; use busbar_contract::transport::wire::{WireStatus, WireStatusClass}; +use super::hooks::{CandidateFacts, Candidates, Constraint}; use super::route::{FarEnd, FarPiece, OutboundRequest, Pick}; /// The bytes one read of the far end takes. @@ -329,8 +330,12 @@ impl Egress { walk, live: None, probe: None, + failed: None, }), probe_of: None, + described: None, + signals: None, + constraint: Mutex::new(None), } } @@ -360,8 +365,12 @@ impl Egress { walk, live: None, probe: Some(member), + failed: None, }), probe_of: Some(destination), + described: None, + signals: None, + constraint: Mutex::new(None), }) } } @@ -381,6 +390,8 @@ struct Live { passthrough: bool, /// When the send started, ms. anchor_ms: u128, + /// The same anchor, in microseconds: the head's latency sample is taken from it. + anchor_us: u128, /// The far end answered. answered: bool, /// One unit of lifetime budget was spent on the success. @@ -403,6 +414,9 @@ struct State { walk: Walk, live: Option, probe: Option, + /// Why the last attempt failed over, in the walk's failover vocabulary (the `routing` stage + /// tap's `previous_failure`). + failed: Option<&'static str>, } /// ONE UNIT'S FAR END over its [`Egress`]. @@ -412,6 +426,133 @@ pub struct EgressFarEnd<'e> { state: Mutex, /// The member a health probe is pinned to; `None` for a unit's walk. probe_of: Option, + /// What the hooks are shown of each member beyond the walk's own facts, by member name: the + /// operator's tags, tier and cost (the plane's section states them). + described: Option>>, + /// Each member's live standing as the ranking hooks read it ([`MemberSignals`]). + signals: Option>, + /// The hooks' constraint on the walk, once the request stage set one. + constraint: Mutex>, +} + +/// ONE MEMBER'S LIVE STANDING, as the ranking hooks read it (1.5.5 fed every candidate these from +/// the lane store, and the ones a deployment declares as catalog signals): its latency signal, its +/// free concurrency, its remaining lifetime budget, and its breaker state, recent error rate and +/// p95 latency in the routing pool. +#[derive(Debug, Clone, Copy, Default, PartialEq)] +pub struct MemberStanding { + /// The latency signal (milliseconds), once a sample is recorded. + pub latency_ms: Option, + /// The free concurrency. + pub available_concurrency: usize, + /// The remaining lifetime request budget; `None` = unlimited. + pub budget_remaining: Option, + /// The breaker state in the pool (`closed` | `open` | `half_open`). + pub breaker_state: Option<&'static str>, + /// The recent error rate in the pool, once an outcome is in its window. + pub error_rate: Option, + /// The p95 latency (whole milliseconds), once a sample is held. + pub latency_p95_ms: Option, +} + +/// WHERE A MEMBER'S STANDING IS READ: the composition root's lane store, by the routing pool and +/// the member's destination. +pub trait MemberSignals: Send + Sync { + /// `destination`'s standing in `pool`. + fn standing(&self, pool: &str, destination: DestinationId) -> MemberStanding; +} + +/// What the operator states of one member for the hooks: its tags, its tier and its cost. +#[derive(Debug, Clone, Default, PartialEq)] +pub struct MemberFacts { + /// The member's model, as the hooks name it (empty = the member's name). + pub model: String, + /// The operator's tags. + pub tags: Vec, + /// The operator's tier. + pub tier: Option, + /// The operator's cost per million tokens. + pub cost_per_mtok: Option, +} + +impl EgressFarEnd<'_> { + /// What the hooks are shown of each member beyond the walk's own facts, by member name. + #[must_use] + pub fn described(mut self, facts: Arc>) -> Self { + self.described = Some(facts); + self + } + + /// Where each member's live standing is read for the hooks. + #[must_use] + pub fn signals(mut self, signals: Arc) -> Self { + self.signals = Some(signals); + self + } + + fn facts_of(&self, member: &Member) -> Option<&MemberFacts> { + self.described.as_ref().and_then(|d| d.get(&member.name)) + } + + fn tags_of(&self, member: &Member) -> &[String] { + self.facts_of(member).map_or(&[], |f| f.tags.as_slice()) + } + + /// `members` as the hooks' constraint leaves them for one pick: the routed pool's members the + /// hooks kept, in their order; a fallback pool's held to their restricts (`Err` = a required + /// restrict no member satisfies). The order, as destinations, rides beside. + fn constrained( + &self, + members: Vec, + primary: bool, + ) -> Result<(Vec, Vec), ()> { + let constraint = self + .constraint + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) + .clone(); + let Some(c) = constraint else { + return Ok((members, Vec::new())); + }; + if primary { + let kept: Vec = members + .iter() + .enumerate() + .filter(|(i, _)| c.keep.as_ref().is_none_or(|k| k.contains(i))) + .map(|(_, m)| m.clone()) + .collect(); + let order: Vec = c + .order + .iter() + .flatten() + .filter_map(|i| members.get(*i)) + .map(|m| m.destination) + .collect(); + return Ok((kept, order)); + } + let tagged = members + .iter() + .enumerate() + .map(|(i, m)| (i, self.tags_of(m))); + let kept = c.enforce(tagged).map_err(|_| ())?; + Ok(( + kept.into_iter() + .filter_map(|i| members.get(i).cloned()) + .collect(), + Vec::new(), + )) + } +} + +/// The walk's picks as the unit's hooks shape them ([`EgressFarEnd::constrained`]). +impl Shape for EgressFarEnd<'_> { + fn shape( + &self, + members: Vec, + primary: bool, + ) -> Option<(Vec, Vec)> { + self.constrained(members, primary).ok() + } } fn exhausted(shed: &Shed) -> Pick { @@ -420,6 +561,20 @@ fn exhausted(shed: &Shed) -> Pick { retry_after: shed .retry_after_secs .map(|s| u32::try_from(s).unwrap_or(u32::MAX)), + detail: shed.detail, + } +} + +/// The walk's shed as the unit is answered: a required gate restriction no member satisfies is the +/// hooks' veto in the shed's words; any other shed is the walk's exhaustion. +fn shed_pick(shed: &Shed) -> Pick { + if shed.gate_rejected { + Pick::Vetoed { + status: u32::from(shed.status), + text: shed.detail.to_string(), + } + } else { + exhausted(shed) } } @@ -580,6 +735,7 @@ impl EgressFarEnd<'_> { }; let name = member.name.clone(); let route = self.egress.routes.get(&member.destination); + // Through its pool (`upstream_credentials: passthrough`), or by its own binding. let passthrough = member.passthrough || route .and_then(|r| r.auth.as_ref()) @@ -595,6 +751,7 @@ impl EgressFarEnd<'_> { degraded, passthrough, anchor_ms: 0, + anchor_us: 0, answered: false, spent: false, delivered: false, @@ -624,9 +781,12 @@ impl EgressFarEnd<'_> { return self.claim_member(&mut w, String::new(), member, None, None, false, 1); } } - match w.walk.next(&ports, self.route.affinity, token) { + match w + .walk + .next_shaped(&ports, self.route.affinity, Some(self), token) + { Step::Take(taken) => return self.admit_taken(&mut w, taken), - Step::Shed(shed) => return exhausted(&shed), + Step::Shed(shed) => return shed_pick(&shed), Step::Wait(wait) => wait, } }; @@ -651,6 +811,7 @@ impl EgressFarEnd<'_> { fn no_answer(&self, token: &Pass, cause: NoAnswer) -> FarPiece { let e = self.egress; let mut w = self.lock(); + let mut failed = None; if let Some(live) = w.live.as_mut() { let now = e.clock.now_secs(); live.probe = None; @@ -676,6 +837,10 @@ impl EgressFarEnd<'_> { e.telemetry.upstream_failure(&pool, destination, failure); e.telemetry.failover(&pool, failover); } + failed = Some(failover); + } + if failed.is_some() { + w.failed = failed; } self.settle(&mut w); fail_over() @@ -692,12 +857,9 @@ impl EgressFarEnd<'_> { .map_or(remaining.max(1), |ms| attempt_cap_ms(ms, remaining)) } - /// Whether the live attempt's member is reached through its pool with the caller's credential. + /// Whether the live attempt's member relays the caller's own credential. fn live_passthrough(&self) -> bool { - self.lock() - .live - .as_ref() - .is_some_and(|l| l.member.passthrough) + self.lock().live.as_ref().is_some_and(|l| l.passthrough) } /// ONE CALL to the member's auth binding for this attempt's fields (THE DESIGN, section 6). @@ -881,14 +1043,18 @@ impl EgressFarEnd<'_> { .map(|(n, v)| (n.as_str(), v.as_slice())) .collect(); let cap_ms = self.attempt_cap(); - let pool = { + let (pool, degraded) = { let w = self.lock(); - w.live - .as_ref() - .map(|l| Self::metric_pool(&l.pool, &l.member).to_string()) - .unwrap_or_default() + w.live.as_ref().map_or((String::new(), false), |l| { + ( + Self::metric_pool(&l.pool, &l.member).to_string(), + l.degraded, + ) + }) }; - if self.probe_of.is_none() { + // A terminal's dispatch (a spill, the least-bad bypass, a queued slot) is not one of the + // walk's attempts: 1.5.5's degraded forward counted none (`busbar_upstream_attempts_total`). + if self.probe_of.is_none() && !degraded { e.telemetry.upstream_attempt(&pool, destination); } // 3. The connector: the judged, pinned dial and the framer's encode. @@ -911,6 +1077,7 @@ impl EgressFarEnd<'_> { }, ); let now_ms = e.clock.now_millis(); + let now_us = e.clock.now_micros(); match opened { Ok(conn) => { { @@ -919,6 +1086,7 @@ impl EgressFarEnd<'_> { live.keep = keep; live.conn = Some(conn); live.anchor_ms = now_ms; + live.anchor_us = now_us; } } if request.text && !request.body.is_empty() { @@ -1201,6 +1369,12 @@ impl EgressFarEnd<'_> { live.probe = None; live.spent = e.breaker.spend_budget(destination); live.delivered = self.route.wants_stream && !bytes.is_empty(); + // In fractional milliseconds, as 1.5.5 measured it (`elapsed().as_secs_f64() * 1000`): + // a sub-millisecond head is a sample, not the EWMA's "none". + let head_us = e.clock.now_micros().saturating_sub(live.anchor_us); + #[allow(clippy::cast_precision_loss)] + e.telemetry + .upstream_latency(destination, head_us as f64 / 1000.0); return FarPiece { bytes, status: far_status, @@ -1208,6 +1382,7 @@ impl EgressFarEnd<'_> { fail_over: false, fields: false, head: Vec::new(), + relayed: live.degraded, }; } let classified = e.breaker.classify(destination, status); @@ -1240,6 +1415,7 @@ impl EgressFarEnd<'_> { .upstream_failure(&pool, destination, classified.label); } live.error_left = Some(e.error_body_max); + let relayed = live.degraded; let piece = cap( live, FarPiece { @@ -1249,6 +1425,7 @@ impl EgressFarEnd<'_> { fail_over: false, fields: false, head: Vec::new(), + relayed, }, ); if piece.last { @@ -1265,6 +1442,7 @@ impl EgressFarEnd<'_> { let (pool, failed) = (live.pool.clone(), live.member.clone()); w.walk.refused_for_size(&e.ports(), &pool, &failed); } + w.failed = Some(classified.label); self.settle(&mut w); fail_over() } @@ -1307,6 +1485,78 @@ impl FarEnd for EgressFarEnd<'_> { self.next_piece(token) } + fn candidates(&self, _token: &Pass) -> Option { + let pool = self.egress.pools.get(&self.route.pool)?; + let members = pool.admissible_members(); + Some(Candidates { + pool: pool.name.clone(), + members: members + .iter() + .enumerate() + .map(|(idx, m)| { + let facts = self.facts_of(m); + let standing = self + .signals + .as_ref() + .map(|s| s.standing(&pool.name, m.destination)) + .unwrap_or_default(); + CandidateFacts { + idx, + model: facts + .filter(|f| !f.model.is_empty()) + .map_or_else(|| m.name.clone(), |f| f.model.clone()), + provider: self + .egress + .routes + .get(&m.destination) + .map(|r| r.provider.clone()) + .unwrap_or_default(), + weight: m.weight, + context_max: m.context_max.and_then(|c| usize::try_from(c).ok()), + tier: facts.and_then(|f| f.tier.clone()), + cost_per_mtok: facts.and_then(|f| f.cost_per_mtok), + tags: facts.map(|f| f.tags.clone()).unwrap_or_default(), + latency_ms: standing.latency_ms, + available_concurrency: standing.available_concurrency, + budget_remaining: standing.budget_remaining, + breaker_state: standing.breaker_state, + error_rate: standing.error_rate, + latency_p95_ms: standing.latency_p95_ms, + } + }) + .collect(), + }) + } + + fn remaining(&self, _token: &Pass) -> Option { + let w = self.lock(); + let Some((name, primary)) = w.walk.walking() else { + return Some(0); + }; + let pool = self.egress.pools.get(name)?; + let members = self + .constrained(pool.admissible_members().into_owned(), primary) + .map(|(members, _)| members) + .unwrap_or_default(); + Some( + members + .iter() + .filter(|m| !w.walk.ctx().is_excluded(m.destination)) + .count(), + ) + } + + fn failure(&self, _token: &Pass) -> Option<&'static str> { + self.lock().failed + } + + fn constrain(&self, _token: &Pass, constraint: Constraint) { + *self + .constraint + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) = Some(constraint); + } + fn write<'a>( &'a self, _token: &'a Pass, diff --git a/crates/busbar-kernel/src/plane_driver/hooks/mod.rs b/crates/busbar-kernel/src/plane_driver/hooks/mod.rs index 68b8550950..2372a0407d 100644 --- a/crates/busbar-kernel/src/plane_driver/hooks/mod.rs +++ b/crates/busbar-kernel/src/plane_driver/hooks/mod.rs @@ -111,6 +111,8 @@ pub struct Bind<'a> { pub pool: &'a str, /// The principal the kernel verified, when there is one. pub principal: Option<&'a str>, + /// The dialect the unit arrived in: the index into the plane's dialects. + pub dialect: u32, } /// The stage taps a unit binds, by stage. @@ -177,6 +179,8 @@ pub struct UnitHooks { pub groups: GroupScope, /// The access amendment of a hook handed the prompt. pub reads: HookRead, + /// The dialect the unit arrived in, by name (the label of a view the plane could not read). + pub dialect: String, } impl std::fmt::Debug for UnitHooks { @@ -238,8 +242,11 @@ pub trait CallerFacts: Send + Sync { fn rate_headroom(&self, principal: &str, pool: &str) -> Option; /// The caller's budget chain. fn budget(&self, principal: &str) -> Vec; - /// Leave the access amendment of a hook handed the prompt. - fn hook_read(&self, hook: &str, principal: Option<&str>, dialect: &str, identity: bool); + /// Leave the access amendment of a hook handed the prompt: the kernel's one seam (the node + /// journal), field names only. + fn hook_read(&self, hook: &str, principal: Option<&str>, dialect: &str, identity: bool) { + crate::audit::amend::hook_read(hook, principal, dialect, identity); + } /// Fire the `response` taps of a unit refused at authentication. fn denied_taps(&self, taps: &[TapEntry], request_id: u64, dialect: &str, status: u16) { let shape = StageShape::zeroed(request_id, "", dialect, false); @@ -302,7 +309,12 @@ impl Parts { } /// The unit's hooks, with what `caller` knows of the verified principal. - fn bind(self, bind: &Bind<'_>, caller: &Arc, request_id: u64) -> UnitHooks { + fn bind( + self, + bind: &Bind<'_>, + (caller, dialects): (&Arc, &[String]), + request_id: u64, + ) -> UnitHooks { let principal = bind.principal.map(str::to_string); let groups_caller = Arc::clone(caller); let reads_caller = Arc::clone(caller); @@ -326,6 +338,10 @@ impl Parts { reads_caller.hook_read(hook, principal, dialect, id); }, ), + dialect: dialects + .get(bind.dialect as usize) + .cloned() + .unwrap_or_default(), } } } @@ -368,7 +384,11 @@ impl HookBinder for BoundHooks { self.taps.clone(), self.requested, )?; - Some(parts.bind(bind, &self.caller, (self.next_request_id)())) + Some(parts.bind( + bind, + (&self.caller, &self.dialects), + (self.next_request_id)(), + )) } fn denied(&self, dialect: u32, status: u16) { @@ -420,7 +440,7 @@ impl HookBinder for HostHooks { }, *h.requested_signals(), )?; - Some(parts.bind(bind, &self.caller, h.next_request_id())) + Some(parts.bind(bind, (&self.caller, &self.dialects), h.next_request_id())) } fn denied(&self, dialect: u32, status: u16) { @@ -798,7 +818,6 @@ struct Asked<'a> { hooks: &'a UnitHooks, pool: &'a str, candidates: &'a [Candidate<'a>], - principal: Option<&'a str>, /// The walk named its candidates (an empty pool, then, has nothing to rank). named: bool, } @@ -812,8 +831,11 @@ impl Asked<'_> { } } + /// The access amendment of a hook handed the prompt, naming whose content it was: the + /// caller's governance key, as the previous release named it. fn read(&self, hook: &str, identity: bool) { - (self.hooks.reads)(hook, self.principal, &self.view.dialect, identity); + let whose = self.hooks.key.as_ref().map(|k| k.id.as_str()); + (self.hooks.reads)(hook, whose, &self.view.dialect, identity); } /// One decision of `resolved`, under its deadline, its failure walked down its on-error chain. @@ -1234,14 +1256,22 @@ impl PlaneUnits<'_, S, F, C> { return; }; let d = self.driver; - let (unit, principal, dialect) = { + // A unit reaches its route leg only through the decode step, which refuses every unit + // its plane did not decode (`DecodeFailed`), so its dialect is always known here; a unit + // with none would state no stage, and the in-session services refuse a unit they do not + // hold by name, never binding it under some other dialect. + let (unit, principal, arrived_in) = { let st = self.lock(); + let Some(arrived_in) = st.decoded.as_ref().map(|x| x.dialect) else { + return; + }; ( st.unit, st.principal.as_ref().map(|p| p.as_str().to_string()), - st.decoded.as_ref().map_or(0, |x| x.dialect), + arrived_in, ) }; + let dialect = arrived_in; let (scope, dialect) = match &d.hooks { Some(binder) => ( self.far.scope(token).unwrap_or_else(|| { @@ -1264,6 +1294,7 @@ impl PlaneUnits<'_, S, F, C> { runtime, d.hooks.clone(), (scope, principal, dialect), + arrived_in, ); let _held = d.services.units().staged(unit, Arc::new(stage)); } @@ -1294,21 +1325,42 @@ impl PlaneUnits<'_, S, F, C> { .principal .as_ref() .map(|p| p.as_str().to_string()); + let dialect = self.lock().decoded.as_ref().map_or(0, |d| d.dialect); let Some(hooks) = binder.bind(&Bind { pool: &walk_pool, principal: principal.as_deref(), + dialect, }) else { return Ok(()); }; + self.lock().hooked_pool.clone_from(&walk_pool); // The unit's correlation id, a native `u64` on its span (never a formatted string), the // same value every hook payload and tap of the unit carries. let span = tracing::debug_span!("forward", request_id = tracing::field::Empty); span.record("request_id", hooks.request_id); let mut view = match self.project(None) { Ok(view) => view, + // A request the plane cannot read is the request's failure where a hook must judge + // it (a rewrite, a gate, a route policy); a tap only observes, and observes the zeroed + // shape (the previous release's stage capture), the request proceeding. + Err(Stopped::Unreadable) + if hooks.rewrites.is_empty() + && hooks.gates.is_empty() + && hooks.policy.is_none() => + { + Projection { + pool: walk_pool.clone(), + dialect: hooks.dialect.clone(), + ..Projection::default() + } + } Err(stopped) => { // The unit's response taps still see it, in the zeroed shape. - self.lock().hooked = Some((hooks, Projection::default())); + let zeroed = Projection { + dialect: hooks.dialect.clone(), + ..Projection::default() + }; + self.lock().hooked = Some((hooks, zeroed)); return Err(stopped); } }; @@ -1317,7 +1369,7 @@ impl PlaneUnits<'_, S, F, C> { } else { walk_pool }; - let principal = principal.as_deref(); + self.lock().hooked_pool.clone_from(&pool); // 1. THE REWRITE CHAIN. for (timeout, hook) in &hooks.rewrites { @@ -1377,7 +1429,6 @@ impl PlaneUnits<'_, S, F, C> { &all, &mut keep, &mut constraint, - principal, ) .await; if let Err(stopped) = decided { @@ -1405,12 +1456,10 @@ impl PlaneUnits<'_, S, F, C> { self.far.constrain(token, constraint); } self.lock().hooked = Some((hooks, view)); - self.lock().hooked_pool = pool; Ok(()) } /// Steps 3 and 4: the gates reconciled, then the base policy when no gate ordered. - #[allow(clippy::too_many_arguments)] async fn decide_route( &self, hooks: &UnitHooks, @@ -1419,7 +1468,6 @@ impl PlaneUnits<'_, S, F, C> { all: &[CandidateFacts], keep: &mut HashSet, constraint: &mut Constraint, - principal: Option<&str>, ) -> Result<(), Stopped> { let mut gate_order: Option<(Vec, &'static str)> = None; if !hooks.gates.is_empty() { @@ -1429,7 +1477,6 @@ impl PlaneUnits<'_, S, F, C> { hooks, pool, candidates: &shown, - principal, named, }; let outcomes: Vec = @@ -1552,7 +1599,6 @@ impl PlaneUnits<'_, S, F, C> { hooks, pool, candidates: &shown, - principal, named, }; match Box::pin(asked.decide(policy)).await { @@ -1783,6 +1829,9 @@ pub struct SessionStage { container: String, principal: Option, dialect: String, + /// The dialect the unit arrived in, as the index into the plane's dialects (what its hooks + /// are bound under). + arrived_in: u32, bound: std::sync::OnceLock>, } @@ -1797,8 +1846,9 @@ impl std::fmt::Debug for SessionStage { /// A gate-first unit's hooks, as the in-session stage runs them: its entry's gates and rewrite /// chain, its caller's key; no tap and no route policy (the gate-first order has none), and a hook -/// handed the prompt leaves the kernel's own access amendment. -fn gated_unit(g: GatedHooks) -> UnitHooks { +/// handed the prompt leaves the kernel's own access amendment; `dialect` is the name of the dialect +/// the unit arrived in. +fn gated_unit(g: GatedHooks, dialect: String) -> UnitHooks { UnitHooks { request_id: g.request_id, rewrites: g.rewrites, @@ -1818,6 +1868,7 @@ fn gated_unit(g: GatedHooks) -> UnitHooks { crate::audit::amend::hook_read(hook, principal, dialect, identity); }, ), + dialect, } } @@ -1842,13 +1893,17 @@ impl Drop for OwedStage { impl SessionStage { /// The stage of a unit of instance `instance`, routed over `scope` for `principal`, its hooks - /// bound by `binder` (none = no hook binds) in the binder's order, run on `runtime`. + /// bound by `binder` (none = no hook binds) in the binder's order, run on `runtime`; the unit + /// arrived in dialect `arrived_in` (the index into the plane's dialects; its name is + /// `dialect`), and its hooks bind there. There is no default dialect: a stage is stated only for + /// a unit its plane decoded. #[must_use] pub fn new( instance: Arc, runtime: tokio::runtime::Handle, binder: Option>, (scope, principal, dialect): (super::RoutedScope, Option, String), + arrived_in: u32, ) -> Self { let gated = binder .as_ref() @@ -1865,6 +1920,7 @@ impl SessionStage { container: scope.container, principal, dialect, + arrived_in, bound: std::sync::OnceLock::new(), } } @@ -1878,11 +1934,12 @@ impl SessionStage { if b.order() == HookOrder::Gated { return b .bind_gated(&self.container, self.principal.as_deref()) - .map(gated_unit); + .map(|g| gated_unit(g, self.dialect.clone())); } b.bind(&Bind { pool: &self.pool, principal: self.principal.as_deref(), + dialect: self.arrived_in, }) }) .as_ref() diff --git a/crates/busbar-kernel/src/plane_driver/mod.rs b/crates/busbar-kernel/src/plane_driver/mod.rs index 87f5324a1d..f6fa8907d1 100644 --- a/crates/busbar-kernel/src/plane_driver/mod.rs +++ b/crates/busbar-kernel/src/plane_driver/mod.rs @@ -88,8 +88,8 @@ use tokio::sync::{watch, Notify}; pub use cancel::{CancelBill, Checkpoint, MoneySeam}; pub use epoch::FlushEpoch; pub use far_end::{ - AuthBinding, Egress, EgressFarEnd, MemberRoute, ResponseKeep, Ride, UnitRoute, - DEFAULT_ERROR_BODY_MAX, + AuthBinding, Egress, EgressFarEnd, MemberFacts, MemberRoute, MemberSignals, MemberStanding, + ResponseKeep, Ride, UnitRoute, DEFAULT_ERROR_BODY_MAX, }; pub use hooks::{ Bind, BoundHooks, CallerFacts, CallerKey, CandidateFacts, Candidates, Constraint, GroupScope, @@ -266,10 +266,11 @@ pub(crate) fn audit_row_to( value: &[u8], principal: Option<&PrincipalId>, ) -> Result<(), ()> { - use busbar_contract::abi::plane::{AUDIT_APPLIED, AUDIT_REJECTED}; + use busbar_contract::abi::plane::{AUDIT_APPLIED, AUDIT_DEGRADED, AUDIT_REJECTED}; let outcome = match outcome { AUDIT_APPLIED => busbar_contract::vocab::OUTCOME_APPLIED, AUDIT_REJECTED => busbar_contract::vocab::OUTCOME_REJECTED, + AUDIT_DEGRADED => busbar_contract::vocab::OUTCOME_DEGRADED, _ => return Err(()), }; let action = std::str::from_utf8(key).map_err(|_| ())?; @@ -526,11 +527,24 @@ pub struct Decoded { pub pool: Option>, /// What [`Decoded::pool`] names: `ROUTE_POOL` or `ROUTE_DIRECT` (ARCHITECT Q-FL3). pub route: u8, - /// The `ROUTE_*` flag bits its `arrive` stated (`ROUTE_ONCE`, `ROUTE_SESSION`). + /// The `ROUTE_*` flag bits its `arrive` stated (`ROUTE_ONCE`, `ROUTE_SESSION`, + /// `ROUTE_STREAM`). pub route_flags: u8, /// The trust facts its `arrive` stated (a counterparty, and an item at a digest), which /// the kernel's Approve judges; `None` = the unit rests on no counterparty. pub trust: Option, + /// The sticky-routing key its `arrive` stated, opaque; `None` = none. + pub affinity: Option>, +} + +/// THE WALK'S AFFINITY POSITION for a plane's opaque sticky key: FNV-1a over its bytes, the hash +/// 1.5.5 put on a session key (v1.5.5 `stable_hash`, `store::fnv1a_u64`), so one session lands on +/// the member it landed on before. +#[must_use] +pub fn sticky_hash(key: &[u8]) -> u64 { + key.iter().fold(crate::store::FNV1A_OFFSET_BASIS, |h, b| { + (h ^ u64::from(*b)).wrapping_mul(crate::store::FNV1A_PRIME) + }) } /// THE KERNEL STEPS A PLANE'S UNIT IS SERVED UNDER ([`PlaneDriver::unit`]'s `steps`): the loop's @@ -545,10 +559,28 @@ pub trait DriverSteps: Units { /// not retried on another member), told with [`Self::decoded`]. fn route_flags(&self, _ctx: &UnitCtx, _flags: u8) {} + /// The sticky-routing key the unit's `arrive` stated (ARCHITECT Q1 ArriveOut), as the walk's + /// affinity position: [`sticky_hash`] of the plane's opaque key. Told with [`Self::decoded`], + /// and only when the plane stated a key. + fn affinity(&self, _ctx: &UnitCtx, _hash: u64) {} + /// The units the plane's `arrive` expects the unit to do (its admission estimate, THE DESIGN /// §7 `admission: estimate`), told with [`Self::decoded`]. An estimate never bills. fn expected(&self, _ctx: &UnitCtx, _units: &[UnitCount]) {} + /// The words of the refusal one of these steps raised, where it has its own: its Retry-After + /// seconds (a rolling limit's wait, `RefusalIn::retry_after_s`, the previous release's + /// `Retry-After` on a limit's 429) and its message (`RefusalIn::text`, the kernel's own message + /// for the refusal: the previous release's sentence for a pool the key may not use or a model + /// no rate prices). `(None, None)` = the reason's own spelling, no wait. + fn refused_words(&self) -> (Option, Option) { + (None, None) + } + + /// An attempt of the unit starts on a member of `provider`, the unit's request read in its + /// `dialect` (the one `arrive` answered): the previous release's translation counter counts an + /// attempt whose far end speaks another dialect. + fn attempting(&self, _ctx: &UnitCtx, _dialect: u32, _provider: &str) {} /// THE STEPS' OWN WORDS for a refusal they decided with `reason`, handed to the plane's /// `refusal` as [`busbar_contract::abi::plane::RefusalIn::text`] (as a limit names the bucket /// that blocked); `None` = the reason's word. A `ROUTE_SCOPE` unit refused because several @@ -665,6 +697,11 @@ impl PlaneUnits<'_, S, F, C> { self.lock().decoded.clone() } + /// The status the plane's `arrive` refused the arrival under, when it refused it. + pub fn declined_status(&self) -> Option { + self.lock().declined.map(|(_, status)| status) + } + /// The status a refusal for `reason` goes out under, as [`Self::render`] chooses it. fn status_for(&self, reason: ReasonCode) -> u32 { let st = self.lock(); @@ -752,6 +789,10 @@ impl PlaneUnits<'_, S, F, C> { trust: (outcome == AbiOutcome::Ready) .then(|| self.driver.calls.arrived_trust(&o)) .flatten(), + affinity: (outcome == AbiOutcome::Ready) + .then(|| self.driver.calls.arrived_affinity(&o)) + .flatten() + .filter(|k| !k.is_empty()), }) } } @@ -760,7 +801,7 @@ impl PlaneUnits<'_, S, F, C> { /// The plane renders a refusal (`refusal`, ticketless, one re-call when short); the kernel's /// generic failure, with no body, when it cannot. fn render(&self, reason: ReasonCode) -> Rendered { - self.render_as(reason, None, None, None) + self.render_as(reason, None, None, None, None) } /// [`Self::render`], or under the status and Retry-After the walk chose (an exhaustion @@ -771,6 +812,7 @@ impl PlaneUnits<'_, S, F, C> { reason: ReasonCode, walk: Option<(u32, Option)>, said: Option<&str>, + retry_after: Option, hook: Option<&str>, ) -> Rendered { let (unit, dialect, declined, words, refused_wait) = { @@ -793,9 +835,14 @@ impl PlaneUnits<'_, S, F, C> { (None, Some((_, status))) => status, (None, None) => self.driver.config.status(dialect, reason), }; - // The walk's terminal floor; else the wait the kernel's own refusal carries (ARCHITECT Q5: - // an admission refusal's window reset, on every plane); `0` = none, and no Retry-After. - let retry_after_s = walk.and_then(|(_, r)| r).or(refused_wait).unwrap_or(0); + // The walk's terminal floor; else the wait this refusal states; else the wait the kernel's + // own refusal carries (ARCHITECT Q5: an admission refusal's window reset, on every plane); + // `0` = none, and no Retry-After. + let retry_after_s = walk + .and_then(|(_, r)| r) + .or(retry_after) + .or(refused_wait) + .unwrap_or(0); let steps_words = if words.is_none() { self.steps.refusal_words(reason) } else { @@ -945,6 +992,7 @@ impl PlaneUnits<'_, S, F, C> { ReasonCode::HookVeto, Some((veto.status, None)), Some(veto.text.as_str()), + None, veto.hook.as_deref(), ); self.lock().rendered = Some(rendered); @@ -1030,18 +1078,19 @@ impl PlaneUnits<'_, S, F, C> { route::End::Failed(reason) | route::End::Cancel(reason, _) => { self.response_tap(false, self.status_for(*reason)); } - route::End::Exhausted(status, _) => self.response_tap(false, *status), + route::End::Exhausted(status, ..) => self.response_tap(false, *status), route::End::Vetoed(status, _) => self.response_tap(true, *status), } let answer = match end { route::End::Done => StepAnswer::proceed(token, RoutePlan::default()), route::End::Failed(reason) => failed(token, reason), - route::End::Exhausted(status, retry_after) => { + route::End::Exhausted(status, retry_after, detail) => { // THE WALK'S EXHAUSTION TERMINAL: its status and its Retry-After floor, handed to // the plane's `refusal` (RefusalIn.retry_after_s), which renders them in its dialect. let rendered = self.render_as( ReasonCode::BreakerOpen, Some((status, retry_after)), + Some(detail), None, None, ); @@ -1055,6 +1104,7 @@ impl PlaneUnits<'_, S, F, C> { Some((status, None)), Some(&text), None, + None, ); self.lock().rendered = Some(rendered); StepAnswer::refuse(token, Refusal::new(ReasonCode::HookVeto)) @@ -1213,6 +1263,9 @@ impl Units for PlaneUnits<'_, S, if let (Some(op), Some(d)) = (op, decoded.as_ref()) { self.steps.decoded(ctx, op, d.route, d.pool.as_deref()); self.steps.route_flags(ctx, d.route_flags); + if let Some(key) = &d.affinity { + self.steps.affinity(ctx, sticky_hash(key)); + } self.steps.expected(ctx, &d.expected); } self.lock().decoded = decoded; @@ -1252,7 +1305,8 @@ impl Units for PlaneUnits<'_, S, }; let body = match reason { Some(reason) if !pending => { - let rendered = self.render(reason); + let (retry_after, said) = self.steps.refused_words(); + let rendered = self.render_as(reason, None, said.as_deref(), retry_after, None); if let (Some(binder), true) = (&self.driver.hooks, is_authentication(reason)) { // A unit refused at authentication: the response taps see the previous // release's synthetic completion, under the status the caller is answered. diff --git a/crates/busbar-kernel/src/plane_driver/route.rs b/crates/busbar-kernel/src/plane_driver/route.rs index 42a61ab9c1..142b639941 100644 --- a/crates/busbar-kernel/src/plane_driver/route.rs +++ b/crates/busbar-kernel/src/plane_driver/route.rs @@ -61,6 +61,10 @@ pub struct FarPiece { /// attempt fails over: the pump moves to the next member WITHOUT pushing the piece to the /// plane, while no byte has reached the caller. pub fail_over: bool, + /// On the reply's first piece: the far end relays this answer as the unit's (a terminal's + /// dispatch: a spill, the least-bad bypass, a queued slot), so it is never failed over: a + /// retry verdict renders it, as 1.5.5 relayed a degraded dispatch's answer. + pub relayed: bool, } /// What the walk answers for an attempt: the member to try, or the pool's exhaustion terminal. @@ -86,6 +90,9 @@ pub enum Pick { status: u32, /// The Retry-After seconds. retry_after: Option, + /// The terminal's words (`busbar_kernel_egress::wire::Shed::detail`), the previous + /// release's sentence for that shed. + detail: &'static str, }, /// The walk refuses for a hook's restriction: a fallback pool no member of which satisfies a /// required restrict the hooks decided (the previous release failed closed there rather than @@ -461,8 +468,9 @@ pub(crate) enum End { Done, /// The unit failed; nothing was cancelled. Failed(ReasonCode), - /// The walk had no member left: its exhaustion terminal's status and Retry-After seconds. - Exhausted(u32, Option), + /// The walk had no member left: its exhaustion terminal's status, Retry-After seconds and + /// words. + Exhausted(u32, Option, &'static str), /// The walk refused for a hook's restriction: the status and the words. Vetoed(u32, String), /// The driver cancels the unit; the answer of the op that was in flight, if one was. @@ -842,7 +850,22 @@ async fn until(left: Option) { } } -impl super::PlaneUnits<'_, S, F, C> { +/// What the attempt loop tells a unit's kernel steps of each attempt: its [`super::DriverSteps`] +/// hook, and nothing for a unit with no steps of its own (a health probe). +pub trait AttemptSteps { + /// An attempt starts on a member of `provider`, the unit's request read in `dialect`. + fn attempting(&self, _ctx: &UnitCtx, _dialect: u32, _provider: &str) {} +} + +impl AttemptSteps for S { + fn attempting(&self, ctx: &UnitCtx, dialect: u32, provider: &str) { + super::DriverSteps::attempting(self, ctx, dialect, provider); + } +} + +impl AttemptSteps for () {} + +impl super::PlaneUnits<'_, S, F, C> { /// S3: every attempt of the unit, until its reply is complete, it fails, or the driver cancels. /// A session's turn leg ([`session::Turn`]) is the same walk: each attempt picks a member /// inside the destination set sealed at the open (a member outside it is passed over, never @@ -876,19 +899,23 @@ impl super::PlaneUnits<'_, S, F, C> { provider, }) => { run.bufs.passthrough = passthrough; + self.steps.attempting(run.ctx, run.bufs.dialect, &provider); run.bufs.provider = provider; ((name, pool), None) } Ok(Pick::Exhausted { status, retry_after, - }) if attempt_no == 1 => { - ((String::new(), String::new()), Some((status, retry_after))) - } + detail, + }) if attempt_no == 1 => ( + (String::new(), String::new()), + Some((status, retry_after, detail)), + ), Ok(Pick::Exhausted { status, retry_after, - }) => return End::Exhausted(status, retry_after), + detail, + }) => return End::Exhausted(status, retry_after, detail), Ok(Pick::Vetoed { status, text }) => return End::Vetoed(status, text), Err(end) => return end, }; @@ -956,7 +983,9 @@ impl super::PlaneUnits<'_, S, F, C> { } if !far_bound { return match terminal { - Some((status, retry_after)) => End::Exhausted(status, retry_after), + Some((status, retry_after, detail)) => { + End::Exhausted(status, retry_after, detail) + } None => End::Failed(ReasonCode::DestinationUnreachable), }; } @@ -985,7 +1014,13 @@ impl super::PlaneUnits<'_, S, F, C> { failed = self.far.failure(run.token); continue 'attempt; } - run.lock().facts.far_end_answered = true; + { + let mut facts = run.lock(); + facts.facts.far_end_answered = true; + if first { + facts.facts.relayed = piece.relayed; + } + } if let Some(t) = turn { // The far end answered this attempt: it is the session's held far end. t.held.send_replace(true); @@ -1124,9 +1159,13 @@ impl super::PlaneUnits<'_, S, F, C> { } _ => { let n = emitted.len(); - let streamed = run.lock().facts.streamed; + let (streamed, relayed) = { + let held = run.lock(); + (held.facts.streamed, held.facts.relayed) + }; // A retry verdict fails over only before the first byte; after it, it is hard. - if out.verdict == VERDICT_RETRY && !streamed { + // A relayed answer (a terminal's dispatch) is never failed over. + if out.verdict == VERDICT_RETRY && !streamed && !relayed { return Step::Retry; } let headed = run.lock().facts.headed; diff --git a/crates/busbar-kernel/src/plane_driver/tests/far_end_tests.rs b/crates/busbar-kernel/src/plane_driver/tests/far_end_tests.rs index f79cf0ce13..f786af27bc 100644 --- a/crates/busbar-kernel/src/plane_driver/tests/far_end_tests.rs +++ b/crates/busbar-kernel/src/plane_driver/tests/far_end_tests.rs @@ -763,6 +763,42 @@ async fn a_401_takes_the_member_down_and_is_relayed_to_the_plane() { ); } +/// THE LEAST-BAD TERMINAL'S ANSWER IS RELAYED (1.5.5 relayed a degraded dispatch's answer and never +/// failed it over): every member failed over, the terminal takes the least-bad one past its +/// breaker, and its answered failure reaches the plane marked relayed, so a retry verdict renders +/// it rather than ending the unit on the exhaustion shed. An ordinary attempt's answer is not. +#[tokio::test] +async fn the_least_bad_terminals_answer_is_relayed_and_an_ordinary_attempts_is_not() { + let r = rig( + &[("a.test", Script::Answer(503, None, vec![b"down"]))], + OnExhausted::LeastBad, + None, + ); + let t = token(); + let far = r.egress.unit(route()); + assert!(matches!(far.member(&t, 1).await, Pick::Member { .. })); + assert!(far.send(&t, request()).await); + let first = far.next(&t).await.expect("a piece"); + assert!( + first.fail_over, + "the walk's own attempt fails over: {first:?}" + ); + assert!(!first.relayed); + assert!( + matches!(far.member(&t, 2).await, Pick::Member { .. }), + "the least-bad terminal takes the member again" + ); + assert!(far.send(&t, request()).await); + let relayed = far.next(&t).await.expect("a piece"); + assert!(!relayed.fail_over, "{relayed:?}"); + assert!( + relayed.relayed, + "the terminal's answer is the unit's: {relayed:?}" + ); + assert_eq!(relayed.status.map(|(s, _)| s), Some(503)); + assert_eq!(relayed.bytes, b"down"); +} + /// A PASSTHROUGH member's 401 is the caller's own key failing (1.5.5's attempt classifier): nothing /// is recorded against the member, and the answer is relayed to the plane as it came. #[tokio::test] @@ -848,7 +884,8 @@ async fn exhaustion_sheds_with_the_retry_after_floor() { far.member(&t, 2).await, Pick::Exhausted { status: 503, - retry_after: Some(2) + retry_after: Some(2), + detail: busbar_kernel_egress::wire::DETAIL_OVERLOADED, } ); // A genuine cooldown is quoted instead of the floor. @@ -862,7 +899,8 @@ async fn exhaustion_sheds_with_the_retry_after_floor() { far.member(&t, 1).await, Pick::Exhausted { status: 503, - retry_after: Some(9) + retry_after: Some(9), + detail: busbar_kernel_egress::wire::DETAIL_OVERLOADED, } ); } @@ -1411,9 +1449,116 @@ async fn a_context_length_refusal_excludes_only_admissible_smaller_windows() { ); } -#[allow(unsafe_code)] -#[path = "probe_unit_tests.rs"] -mod probe_unit; +/// THE HOOKS' CONSTRAINT ON THE WALK (ARCHITECT K5: `FarEnd` candidates/constrain): the walk +/// reports its pool's members with what the operator states of them, and once constrained picks +/// only the members the hooks kept, the hooks' order first. +#[tokio::test] +async fn the_hooks_constraint_orders_and_restricts_the_pick() { + use crate::plane_driver::{Constraint, MemberFacts}; + let r = rig( + &[ + ("a.test", Script::Answer(200, None, vec![b"ok"])), + ("b.test", Script::Answer(200, None, vec![b"ok"])), + ("c.test", Script::Answer(200, None, vec![b"ok"])), + ], + OnExhausted::Status503, + None, + ); + let t = token(); + let described = Arc::new(HashMap::from([( + "m2".to_string(), + MemberFacts { + tags: vec!["baa".to_string()], + tier: Some("gold".to_string()), + ..MemberFacts::default() + }, + )])); + let far = r.egress.unit(route()).described(described); + let candidates = far + .candidates(&t) + .expect("the walk names its pool's members"); + assert_eq!(candidates.pool, POOL); + let names: Vec<(usize, &str, &str)> = candidates + .members + .iter() + .map(|c| (c.idx, c.model.as_str(), c.provider.as_str())) + .collect(); + assert_eq!(names, [(0, "m0", "p0"), (1, "m1", "p1"), (2, "m2", "p2")]); + assert_eq!(candidates.members[2].tags, ["baa"]); + assert_eq!(candidates.members[2].tier.as_deref(), Some("gold")); + // The hooks keep m1 and m2 and prefer m2: the weighted walk (m0 heaviest) is overridden. + far.constrain( + &t, + Constraint { + keep: Some(vec![1, 2]), + order: Some(vec![2]), + ..Constraint::default() + }, + ); + let picked = |p: Pick| match p { + Pick::Member { name, .. } => name, + other => panic!("{other:?}"), + }; + assert_eq!( + picked(far.member(&t, 1).await), + "m2", + "the hooks' order first" + ); + assert!(far.send(&t, request()).await); + let _ = drain(&far, &t).await; + assert_eq!( + picked(far.member(&t, 2).await), + "m1", + "then what the hooks kept" + ); +} + +/// A required restrict the hooks decided holds at the fallback pool's boundary: no member there +/// carries its tags, so the walk refuses as the hook would (the previous release failed closed +/// there rather than spill to an ineligible far end). +#[tokio::test] +async fn a_required_restrict_refuses_at_the_fallback_pool() { + use crate::plane_driver::{Constraint, Restrict}; + let mut r = rig( + &[("a.test", Script::Refused)], + OnExhausted::FallbackPool("fb".into()), + None, + ); + let fallback = Pool { + name: "fb".into(), + members: vec![Member::new(DestinationId::new(1), "m0", 1)], + failover: Failover { + timeout_secs: 30, + max_hops: 3, + exclusions: Vec::new(), + }, + on_exhausted: OnExhausted::Status503, + }; + r.egress.pools.insert("fb".to_string(), fallback); + let t = token(); + let far = r.egress.unit(route()); + far.constrain( + &t, + Constraint { + restricts: vec![Restrict { + tags_any: vec!["baa".to_string()], + on_empty: crate::config::PolicyOnError::Reject, + name: "compliance", + }], + ..Constraint::default() + }, + ); + assert!(matches!(far.member(&t, 1).await, Pick::Member { .. })); + assert!(!far.send(&t, request()).await, "the dial is refused"); + assert_eq!( + far.member(&t, 2).await, + Pick::Vetoed { + status: 503, + text: "No upstream satisfies a required gate's restriction. Please retry shortly." + .to_string(), + } + ); +} /// THE DISPATCH RECORD NAMES ITS UNIT (ARCHITECT P3 (c), 2026-10-02): the write-ahead record the walk /// makes before each dial carries the unit the walk serves, so the root writes it under that unit's @@ -1437,6 +1582,9 @@ async fn every_dispatch_is_recorded_under_its_unit() { assert_eq!(kept[0].unit, busbar_contract::UnitKey::new(77)); } +#[allow(unsafe_code)] +#[path = "probe_unit_tests.rs"] +mod probe_unit; /// MULTI-NEED (ARCHITECT Q-L5B-NEEDS 2026-10-03): a member binds every need its auth names; a far /// request that names one (its declared index plus one) opens on it, one that names none on the /// member's own, and one naming a need the member has no binding for is refused before any record diff --git a/crates/busbar-kernel/src/plane_driver/tests/money_tests.rs b/crates/busbar-kernel/src/plane_driver/tests/money_tests.rs index deff7001af..559000922e 100644 --- a/crates/busbar-kernel/src/plane_driver/tests/money_tests.rs +++ b/crates/busbar-kernel/src/plane_driver/tests/money_tests.rs @@ -720,6 +720,7 @@ fn a_cancel_bill_carries_the_floor_counts_and_never_an_estimate() { headed: false, far_end_answered: true, streamed: true, + relayed: false, units: vec![ floor(INPUT, 40), UnitCount { diff --git a/crates/busbar-kernel/src/plane_host/engine_view.rs b/crates/busbar-kernel/src/plane_host/engine_view.rs index a889aabaf6..d400ee8a6b 100644 --- a/crates/busbar-kernel/src/plane_host/engine_view.rs +++ b/crates/busbar-kernel/src/plane_host/engine_view.rs @@ -149,3 +149,124 @@ impl EngineTablesView for EmptyEngineTablesView { busbar_contract::config::UpstreamCreds::default() } } + +/// THE KERNEL'S OWN TABLES VIEW over the `pools:`/`models:` sections it resolved (the kernel-owned +/// sections, spec Part 1 line 569: the host resolves and validates them): the lanes in the lane +/// table's order (the order the lane store and the telemetry bank are indexed by), the direct-model +/// index, each pool's members and its fallback pool, and the all-pools upstream-credential default. +/// The read seam answers from it when no plane contributed a runtime of its own, so `/stats`, the +/// `/metrics` lane families, `/v1/models` and the admin pool listing read the same tables a +/// door-served plane walks. The live `on_exhausted: queue` depth is the one the walk reports +/// ([`set_pool_queued_depth`]). +#[derive(Debug, Default)] +pub struct ConfigTables { + lanes: Vec<(String, String, String)>, + by_model: std::collections::HashMap, + pools: std::collections::HashMap>, + fallbacks: std::collections::HashMap, + upstream_credentials: busbar_contract::config::UpstreamCreds, +} + +impl ConfigTables { + /// The tables of one resolved configuration generation. + #[must_use] + pub fn of( + lanes: &[crate::plane_host::LaneInput], + pools: &[crate::plane_host::PoolInput], + by_model: &std::collections::HashMap, + upstream_credentials: busbar_contract::config::UpstreamCreds, + ) -> Self { + ConfigTables { + lanes: lanes + .iter() + .map(|l| (l.model.clone(), l.provider.clone(), l.base_url.clone())) + .collect(), + by_model: by_model.clone(), + pools: pools + .iter() + .map(|p| { + ( + p.name.clone(), + p.members.iter().map(|m| (m.lane_idx, m.weight)).collect(), + ) + }) + .collect(), + fallbacks: pools + .iter() + .filter_map(|p| match &p.on_exhausted { + crate::plane_host::OnExhaustedInput::FallbackPool(to) => { + Some((p.name.clone(), to.clone())) + } + _ => None, + }) + .collect(), + upstream_credentials, + } + } +} + +/// The live `on_exhausted: queue` depth of each pool, as the walk reports it. +static QUEUED: std::sync::LazyLock>> = + std::sync::LazyLock::new(Default::default); + +/// Record `pool`'s live wait-terminal depth (the walk's balanced park/leave count). +pub fn set_pool_queued_depth(pool: &str, depth: u64) { + QUEUED + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) + .insert(pool.to_string(), depth); +} + +impl EngineTablesView for ConfigTables { + fn pools(&self) -> Vec<(&str, Vec)> { + self.pools + .iter() + .map(|(name, members)| (name.as_str(), members.iter().map(|(i, _)| *i).collect())) + .collect() + } + fn pool_exists(&self, pool: &str) -> bool { + self.pools.contains_key(pool) + } + fn model_indices(&self) -> Vec<(&str, usize)> { + self.by_model + .iter() + .map(|(m, &idx)| (m.as_str(), idx)) + .collect() + } + fn model_index(&self, model: &str) -> Option { + self.by_model.get(model).copied() + } + fn lane_view(&self, idx: usize) -> Option> { + self.lanes + .get(idx) + .map(|(model, provider, base_url)| LaneView { + model, + provider, + base_url, + }) + } + fn lane_count(&self) -> usize { + self.lanes.len() + } + fn pool_members(&self, pool: &str) -> Vec<(usize, u32)> { + self.pools.get(pool).cloned().unwrap_or_default() + } + fn queued_depth(&self, pool: &str) -> u64 { + QUEUED + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) + .get(pool) + .copied() + .unwrap_or(0) + } + fn on_exhausted_fallback(&self, pool: &str) -> Option { + self.fallbacks.get(pool).cloned() + } + fn upstream_creds(&self) -> busbar_contract::config::UpstreamCreds { + self.upstream_credentials + } +} + +#[cfg(test)] +#[path = "tests/config_tables_tests.rs"] +mod config_tables_tests; diff --git a/crates/busbar-kernel/src/plane_host/mod.rs b/crates/busbar-kernel/src/plane_host/mod.rs index 01f6797d74..a0085adaf4 100644 --- a/crates/busbar-kernel/src/plane_host/mod.rs +++ b/crates/busbar-kernel/src/plane_host/mod.rs @@ -1671,7 +1671,8 @@ pub use crate::plane_host::build_input::{ PoolMemberInput, TripInput, TripModeInput, }; pub use crate::plane_host::engine_view::{ - EmptyEngineTablesView, EngineTablesView, LaneView, EMPTY_VIEW, + set_pool_queued_depth, ConfigTables, EmptyEngineTablesView, EngineTablesView, LaneView, + EMPTY_VIEW, }; use busbar_contract::abi::hot::{AdmissionId, Signal, StatusClass}; use busbar_contract::records::PlaneRequestCtx; diff --git a/crates/busbar-kernel/src/plane_host/tests/config_tables_tests.rs b/crates/busbar-kernel/src/plane_host/tests/config_tables_tests.rs new file mode 100644 index 0000000000..59d4abe27a --- /dev/null +++ b/crates/busbar-kernel/src/plane_host/tests/config_tables_tests.rs @@ -0,0 +1,122 @@ +// SPDX-License-Identifier: Apache-2.0 +// Copyright (C) 2026 Busbar Inc and contributors + +//! The kernel's own tables view over the `pools:`/`models:` sections it resolved: the read seam's +//! answer when no plane contributes a runtime, so it must project exactly what the sections say. + +use std::collections::HashMap; + +use super::{ConfigTables, EngineTablesView}; +use crate::plane_host::{AuthStyleInput, LaneInput, OnExhaustedInput, PoolInput, PoolMemberInput}; + +fn lane(model: &str, provider: &str) -> LaneInput { + LaneInput { + model: model.to_string(), + provider: provider.to_string(), + protocol: "widget".to_string(), + base_url: format!("https://{provider}.invalid"), + path: None, + path_base: None, + organization: None, + project: None, + upstream_model: None, + api_key: busbar_contract::redacted::Redacted::new(String::new()), + auth_style: AuthStyleInput::Default, + scope: None, + token_url: None, + subject: None, + error_map: HashMap::new(), + health: None, + allow_metadata_hosts: Vec::new(), + context_max: None, + lane_default_max_tokens: None, + attempt_timeout_ms: None, + reasoning: false, + prompt_caching: false, + lane_caps: Default::default(), + max_concurrent: 1, + limited: false, + budget: -1, + } +} + +fn member(model: &str, lane_idx: usize, weight: u32) -> PoolMemberInput { + PoolMemberInput { + model: model.to_string(), + lane_idx, + weight, + reasoning: None, + attempt_timeout_ms: None, + tier: None, + cost_per_mtok: None, + tags: Vec::new(), + } +} + +fn pool(name: &str, members: Vec, on_exhausted: OnExhaustedInput) -> PoolInput { + PoolInput { + name: name.to_string(), + members, + failover: None, + affinity: None, + on_exhausted, + upstream_credentials: None, + breaker: None, + } +} + +#[test] +fn the_tables_project_the_resolved_sections_in_lane_order() { + let lanes = vec![lane("m-a", "pa"), lane("m-b", "pb")]; + let pools = vec![ + pool( + "primary", + vec![member("m-a", 0, 3), member("m-b", 1, 1)], + OnExhaustedInput::FallbackPool("spare".to_string()), + ), + pool( + "spare", + vec![member("m-b", 1, 1)], + OnExhaustedInput::default(), + ), + ]; + let by_model: HashMap = [("m-a".to_string(), 0), ("m-b".to_string(), 1)].into(); + let t = ConfigTables::of( + &lanes, + &pools, + &by_model, + busbar_contract::config::UpstreamCreds::Passthrough, + ); + assert_eq!(t.lane_count(), 2); + let b = t.lane_view(1).expect("lane 1"); + assert_eq!( + (b.model, b.provider, b.base_url), + ("m-b", "pb", "https://pb.invalid") + ); + assert!(t.lane_view(2).is_none()); + assert_eq!(t.model_index("m-a"), Some(0)); + assert!(t.pool_exists("primary") && !t.pool_exists("m-a")); + assert_eq!(t.pool_members("primary"), vec![(0, 3), (1, 1)]); + assert_eq!(t.on_exhausted_fallback("primary").as_deref(), Some("spare")); + assert_eq!(t.on_exhausted_fallback("spare"), None); + assert_eq!( + t.upstream_creds(), + busbar_contract::config::UpstreamCreds::Passthrough + ); + let mut pools_seen: Vec<(&str, Vec)> = t.pools(); + pools_seen.sort(); + assert_eq!( + pools_seen, + vec![("primary", vec![0, 1]), ("spare", vec![1])] + ); +} + +#[test] +fn the_queue_depth_is_the_one_the_walk_reports() { + let t = ConfigTables::default(); + assert_eq!(t.queued_depth("config-tables-q"), 0); + super::set_pool_queued_depth("config-tables-q", 2); + assert_eq!(t.queued_depth("config-tables-q"), 2); + super::set_pool_queued_depth("config-tables-q", 0); + assert_eq!(t.queued_depth("config-tables-q"), 0); +} diff --git a/crates/busbar-kernel/src/router.rs b/crates/busbar-kernel/src/router.rs index 053fbc1f1e..d8c1e1462e 100644 --- a/crates/busbar-kernel/src/router.rs +++ b/crates/busbar-kernel/src/router.rs @@ -353,6 +353,9 @@ pub(crate) fn base_data_router( crate::core_routes::CoreRouteTable, ) { use busbar_contract::abi::mechanism::route::{RouteAuth, RouteMethod}; + // A door plane whose claim is the whole tree (`/` and every path under it) claims the + // convenience surfaces' paths too, so the kernel mounts none of its own there. + let door_claims_root = doors.iter().any(|d| d.path.starts_with("/{*")); // EVERY core route is mounted through `CoreRouter::route`, which takes the handler and the // admission bar in ONE act (`core_routes`): a route the auth middleware knows nothing about is // not a thing this function can produce. @@ -405,19 +408,27 @@ pub(crate) fn base_data_router( RouteMethod::Get, RouteAuth::Key, endpoints::list_models_v1beta, - ) - .route( - "/{name}/v1/messages", - RouteMethod::Post, - RouteAuth::Key, - ingress::named, - ) - .route( - "/{provider}/{model}/v1/messages", - RouteMethod::Post, - RouteAuth::Key, - ingress::adhoc, ); + // THE CONVENIENCE SURFACES hand their unit to a body-model arrival, so they are mounted only + // where a plane registered one and no door plane claims the whole tree (a plane served through + // its door parses these paths itself). + let router = if crate::ingress::arrival::any_body_ingress() && !door_claims_root { + router + .route( + "/{name}/v1/messages", + RouteMethod::Post, + RouteAuth::Key, + ingress::named, + ) + .route( + "/{provider}/{model}/v1/messages", + RouteMethod::Post, + RouteAuth::Key, + ingress::adhoc, + ) + } else { + router + }; // THE PLANES' DATA ROUTES, contributed through the registry rather than named here. For every // registered plane with a `mount` fn AND a runtime object this generation (its slot), the plane's // own `mount` mounts its routes from that slot, each door opening only when that plane's own @@ -447,6 +458,8 @@ pub(crate) fn base_data_router( // the protocol fallback, by the same adapter and at the same bar a plane route takes. A claim // carries no slot: its handler holds what it serves. let unslotted: std::sync::Arc = std::sync::Arc::new(()); + let door_paths: std::collections::BTreeSet = + doors.iter().map(|spec| spec.path.clone()).collect(); for spec in doors { router = mount_plane_route(router, std::sync::Arc::clone(&unslotted), spec); } @@ -465,6 +478,21 @@ pub(crate) fn base_data_router( "door session routes are not mounted: this build has no duplex upgrade" ); } + // A verb no route on a door path takes is answered as the previous release answered it on that + // path, from its catch-all: the native-envelope 405, after the data-plane bar, and never with + // an `Allow` field (a door path is no fixed route of the previous release's, so it named none). + for path in door_paths { + let claimed = router + .methods_at(&path) + .into_iter() + .map(crate::plugin_routes::method_filter_of) + .reduce(axum::routing::MethodFilter::or); + if let Some(rest) = claimed.and_then(unclaimed_verbs) { + router = router.map_router(|r| { + r.route(&path, axum::routing::on(rest, method_not_allowed_handler)) + }); + } + } // THE PLANES' INBOUND WS-ACCEPT ARRIVALS, drained from the neutral substrate registry the // composition root installed (`install_ws_arrivals`). Behind the neutral `duplex-ws` feature: the // default/shipped money-path build compiles neither this line nor `mount_ws_arrivals`, so the @@ -511,6 +539,47 @@ pub(crate) fn base_data_router( .into_parts() } +/// Every verb a door path's claims leave unclaimed, as one filter; `None` when they claim them all. +/// `HEAD` rides `GET` where `GET` is claimed (axum serves it from the `GET` handler). +fn unclaimed_verbs(claimed: axum::routing::MethodFilter) -> Option { + use axum::routing::MethodFilter; + let all = [ + (MethodFilter::GET, axum::http::Method::GET), + (MethodFilter::POST, axum::http::Method::POST), + (MethodFilter::PUT, axum::http::Method::PUT), + (MethodFilter::PATCH, axum::http::Method::PATCH), + (MethodFilter::DELETE, axum::http::Method::DELETE), + (MethodFilter::OPTIONS, axum::http::Method::OPTIONS), + (MethodFilter::TRACE, axum::http::Method::TRACE), + (MethodFilter::CONNECT, axum::http::Method::CONNECT), + (MethodFilter::HEAD, axum::http::Method::HEAD), + ]; + let get_claimed = claimed_has(claimed, &axum::http::Method::GET); + all.into_iter() + .filter(|(_, m)| !claimed_has(claimed, m)) + .filter(|(_, m)| !(get_claimed && *m == axum::http::Method::HEAD)) + .map(|(f, _)| f) + .reduce(MethodFilter::or) +} + +/// Whether `filter` takes `method`. +fn claimed_has(filter: axum::routing::MethodFilter, method: &axum::http::Method) -> bool { + use axum::routing::MethodFilter; + let one = match *method { + axum::http::Method::GET => MethodFilter::GET, + axum::http::Method::POST => MethodFilter::POST, + axum::http::Method::PUT => MethodFilter::PUT, + axum::http::Method::PATCH => MethodFilter::PATCH, + axum::http::Method::DELETE => MethodFilter::DELETE, + axum::http::Method::OPTIONS => MethodFilter::OPTIONS, + axum::http::Method::TRACE => MethodFilter::TRACE, + axum::http::Method::CONNECT => MethodFilter::CONNECT, + axum::http::Method::HEAD => MethodFilter::HEAD, + _ => return false, + }; + filter.or(one) == filter +} + /// Mount ONE neutral [`busbar_kernel::plane_routes::PlaneRouteSpec`] onto the core router (S4a /// Option A). This is the SINGLE place a plane's neutral route touches core router vocabulary: it /// calls the SAME [`crate::core_routes::CoreRouter::route`] the legacy `mount` fns called, with the diff --git a/crates/busbar-kernel/src/state.rs b/crates/busbar-kernel/src/state.rs index 9935f2dc4c..e9f17565a6 100644 --- a/crates/busbar-kernel/src/state.rs +++ b/crates/busbar-kernel/src/state.rs @@ -82,6 +82,13 @@ pub struct App { /// An ABSENT slot — the featureless binary boots with no fallback plane configured, so none was /// inserted — reads as the substrate-resident empty view, never a panic. Neutral: names no dialect. pub fallback_runtime_key: &'static str, + /// The composition root's projection of the configuration this generation was built from + /// ([`crate::appbuild::ConfigProjection`]); `None` when the root bound none. + pub config_projection: Option>, + /// The kernel's own tables over the `pools:`/`models:` sections this generation resolved: the + /// read seam's answer when no plane contributed a runtime of its own (a door-served plane walks + /// these sections through its door). + pub config_tables: Arc, pub store: Arc, /// THE CONTAINER PLANES' BREAKER CELLS — the degenerate single-member cell per registered /// container-plane member (the breaker-all-planes audit's closing design). Live state, shared by @@ -542,7 +549,7 @@ impl App { self.plane_slot(key), ) { (Some(viewer), Some(slot)) => viewer(slot.as_ref()), - _ => &busbar_kernel::plane_host::EMPTY_VIEW, + _ => &*self.config_tables, } } diff --git a/crates/busbar-kernel/src/store/in_memory/mod.rs b/crates/busbar-kernel/src/store/in_memory/mod.rs index dde0ccb2cd..89e3df3023 100644 --- a/crates/busbar-kernel/src/store/in_memory/mod.rs +++ b/crates/busbar-kernel/src/store/in_memory/mod.rs @@ -526,7 +526,7 @@ use busbar_kernel::store::{BreakerCfg, TripConfig, TripMode}; /// back to the ADR-0002 defaults. Was an inherent `config::BreakerCfg::to_runtime` method; now a /// free function because BOTH the config type and the runtime type are foreign to this crate /// (substrate-owned), which the orphan rule forbids an inherent impl over. -pub(crate) fn breaker_cfg_to_runtime(cfg: &crate::config::BreakerCfg) -> BreakerCfg { +pub fn breaker_cfg_to_runtime(cfg: &crate::config::BreakerCfg) -> BreakerCfg { let trip = cfg .trip .as_ref() diff --git a/crates/busbar-kernel/src/telemetry.rs b/crates/busbar-kernel/src/telemetry.rs index 8b649e0db2..44e7f09df1 100644 --- a/crates/busbar-kernel/src/telemetry.rs +++ b/crates/busbar-kernel/src/telemetry.rs @@ -303,6 +303,44 @@ pub(crate) fn request_finished( } } +/// `busbar_requests_total` + `busbar_request_duration_seconds` for one finished request of the +/// plane that serves the `pools` map (the fallback plane's 1.5.5 families, `{ingress_protocol, +/// pool, outcome}` and no `plane` label), its outcome classified from the caller's `status` +/// ([`outcome_of`]): the request families a unit served through that plane's door emits at its end, +/// as the previous release's `ingress::finish_inner` emitted them. +pub fn model_request_finished( + app: &App, + ingress_protocol: &str, + pool: &str, + status: u16, + seconds: f64, +) { + let outcome = outcome_of(status); + request_finished( + app, + crate::plane::fallback_key(), + ingress_protocol, + pool, + outcome, + seconds, + ); + // The best-effort request log, under the same compute gate the previous release's request path + // read (`ingress::finish_inner`): produced only when a configured export subscribes to it. + if app + .export_projections + .wants_stream(busbar_contract::abi::export::ExportStream::Logs) + { + #[allow(clippy::cast_possible_truncation, clippy::cast_sign_loss)] + crate::export::deliver_request_log(&crate::export::RequestLogFacts { + ts: busbar_kernel::store::now(), + ingress_protocol, + pool, + outcome, + latency_ms: (seconds * 1000.0) as u64, + }); + } +} + // `upstream_attempt_on` / `upstream_failure_on` — THE EMIT for this family, on EVERY plane — MOVED // DOWN to the neutral substrate (`busbar_kernel::telemetry`) so a plane's own synchronous client // leg can name them without reaching into core. They take NO `&App` and diff --git a/crates/busbar-kernel/src/test_support/mod.rs b/crates/busbar-kernel/src/test_support/mod.rs index 404f0c6698..490769edce 100644 --- a/crates/busbar-kernel/src/test_support/mod.rs +++ b/crates/busbar-kernel/src/test_support/mod.rs @@ -1738,7 +1738,12 @@ impl TestApp { // Built and inserted ONLY when a real fallback (LLM) plane owns the key — otherwise `lanes`/ // `by_model`/the `self.*` tables simply drop unused, and `App::llm_runtime` reads the empty // default (a surface with no LLM plane never routes through `engine_tables` anyway). - if let Some(build_runtime) = fallback.and_then(|d| d.build_runtime) { + // THE KERNEL'S OWN TABLES over the same lanes and pools, as production `appbuild` builds them: + // the read seam answers from them when the fallback plane contributes no runtime of its own + // (a door plane), and reads the empty default when no fallback plane is registered. + let mut config_tables = + std::sync::Arc::::default(); + if let Some(decl) = fallback { // Assemble the NEUTRAL `PlaneBuildInput` (money-path Phase 3-4 C) exactly as production // `appbuild` does, then hand it to the fallback (LLM) plane's REGISTERED `build_runtime` // fn-pointer — so the fixture names no `Lane`/`NativeRuntime` and exercises the SAME in-plane @@ -1802,6 +1807,12 @@ impl TestApp { name, }) .collect(); + config_tables = std::sync::Arc::new(busbar_kernel::plane_host::ConfigTables::of( + &lane_inputs, + &pool_inputs, + &by_model, + self.upstream_credentials, + )); let default_failover = self .failover_cfg .as_ref() @@ -1841,8 +1852,12 @@ impl TestApp { crate::test_support::outbound_auth::axis(), )), }; - let slot = build_runtime(&build_input as &dyn std::any::Any, None); - plane_slots.insert(fallback_runtime_key, slot); + // A fallback plane that contributes no runtime of its own (a door plane) is read through + // the kernel's own tables above. + if let Some(build_runtime) = decl.build_runtime { + let slot = build_runtime(&build_input as &dyn std::any::Any, None); + plane_slots.insert(fallback_runtime_key, slot); + } } let requested_signals = crate::hooks::requested_signals(&self.hook_registry); let any_content_hook = crate::hooks::any_content_hook(&self.hook_registry); @@ -1948,6 +1963,10 @@ impl TestApp { // above into `plane_slots` under this interned key; the snapshot names only the key, and // `App::llm_runtime` downcasts the slot on the money path. fallback_runtime_key, + // The kernel's own tables over the fixture's lanes and pools (built above when a fallback + // plane is registered), read when that plane contributes no runtime of its own. + config_tables, + config_projection: None, store: store.clone(), plane_breakers: std::sync::Arc::new(crate::store::PlaneBreakers::new()), session_store: std::sync::Arc::new(crate::session::SessionStore::new(1024, None)), diff --git a/crates/busbar-kernel/src/tests/host_stage_tests.rs b/crates/busbar-kernel/src/tests/host_stage_tests.rs index 62834510c2..5c0498192c 100644 --- a/crates/busbar-kernel/src/tests/host_stage_tests.rs +++ b/crates/busbar-kernel/src/tests/host_stage_tests.rs @@ -289,6 +289,7 @@ fn staged_over( tokio::runtime::Handle::current(), Some(binder), (scope, None, "d0".to_string()), + 0, ); assert!(s.units().staged(unit, Arc::new(stage))); } @@ -558,6 +559,7 @@ async fn the_stage_refuses_every_unit_it_does_not_hold() { tokio::runtime::Handle::current(), None, (RoutedScope::default(), None, String::new()), + 0, ); assert!(!s.units().staged(3, Arc::new(late))); } @@ -580,6 +582,7 @@ async fn a_unit_no_hook_binds_passes_unchanged() { tokio::runtime::Handle::current(), None, (RoutedScope::default(), None, String::new()), + 0, )) )); assert_eq!( @@ -597,6 +600,64 @@ async fn a_unit_no_hook_binds_passes_unchanged() { assert_eq!(scan(&s, Some(1), b"secret").await.value, svc::CONTENT_PASS); } +/// A binder that records the dialect each unit it binds arrived in. +struct DialectSeen { + hooks: BoundHooks, + seen: std::sync::Mutex>, +} + +impl HookBinder for DialectSeen { + fn bind(&self, bind: &Bind<'_>) -> Option { + self.seen.lock().unwrap().push(bind.dialect); + self.hooks.bind(bind) + } +} + +/// A session that arrived in a dialect other than the plane's first binds its hooks THERE: the +/// stage carries the dialect its unit arrived in, never a default. +#[tokio::test] +async fn a_session_on_a_later_dialect_binds_its_hooks_in_that_dialect() { + let s = services(&Arc::new(AtomicU64::new(0))); + let mut bound = hooks(vec![], vec![], None); + bound.dialects = vec!["d0".into(), "d1".into(), "d2".into()]; + let binder = Arc::new(DialectSeen { + hooks: bound, + seen: std::sync::Mutex::new(Vec::new()), + }); + s.units().admitted( + 4, + UnitRecord { + principal: None, + depth: 0, + }, + ); + let stage = SessionStage::new( + Arc::from("inst"), + tokio::runtime::Handle::current(), + Some(Arc::clone(&binder) as Arc), + ( + RoutedScope { + pool: "pool-z".to_string(), + container: "pool-z".to_string(), + }, + None, + "d2".to_string(), + ), + 2, + ); + assert!(s.units().staged(4, Arc::new(stage))); + let pass = scan(&s, Some(4), b"a tool result").await; + assert_eq!( + (pass.outcome, pass.value), + (Outcome::Ready, svc::CONTENT_PASS) + ); + assert_eq!( + *binder.seen.lock().unwrap(), + vec![2], + "bound in the dialect it arrived in" + ); +} + /// A gate-first binder: `screen` is attached to the entry `srv-a` only; the routed order binds a /// gate that blocks everything, so a stage that bound the wrong order would block both units. struct EntryGates { diff --git a/crates/busbar-kernel/src/tests/members/egress/route/mod.rs b/crates/busbar-kernel/src/tests/members/egress/route/mod.rs index 04ccfeb3b2..93219b32ef 100644 --- a/crates/busbar-kernel/src/tests/members/egress/route/mod.rs +++ b/crates/busbar-kernel/src/tests/members/egress/route/mod.rs @@ -250,6 +250,7 @@ impl Node { Pick::Exhausted { status, retry_after, + .. } => { return Routed::Refused(Refusal { status, diff --git a/crates/busbar-kernel/tests/endpoints_cross_plane.rs b/crates/busbar-kernel/tests/endpoints_cross_plane.rs index 9de1e27d21..d5a8774947 100644 --- a/crates/busbar-kernel/tests/endpoints_cross_plane.rs +++ b/crates/busbar-kernel/tests/endpoints_cross_plane.rs @@ -461,6 +461,7 @@ fn with_listing_door( record_kinds: Vec::new(), trust_keys: Vec::new(), caller_credential_refusal: None, + fallback: false, validate: Arc::new(|_: &[u8]| Ok(())), facing: Arc::new( move |_: &[u8], @@ -590,6 +591,7 @@ fn opening_door() -> &'static busbar_kernel::plane::registry::PlaneDecl { record_kinds: Vec::new(), trust_keys: Vec::new(), caller_credential_refusal: None, + fallback: false, validate: Arc::new(|_: &[u8]| Ok(())), facing: Arc::new( |_: &[u8], diff --git a/crates/busbar-kernel/tests/plane_driver.rs b/crates/busbar-kernel/tests/plane_driver.rs index 3a82c12889..915c80da22 100644 --- a/crates/busbar-kernel/tests/plane_driver.rs +++ b/crates/busbar-kernel/tests/plane_driver.rs @@ -15,6 +15,9 @@ mod common; #[path = "support/plane_driver_cases.rs"] mod cases; +#[path = "support/plane_driver_hook_cases.rs"] +mod hook_cases; + use std::collections::HashMap; use std::future::Future; use std::pin::Pin; @@ -23,6 +26,7 @@ use std::sync::{Arc, Mutex, OnceLock}; use std::task::{Context, Poll, Waker}; use std::time::{Duration, Instant}; +use busbar_contract::abi::hook::{MessageView, RequestView}; use busbar_contract::abi::mechanism::call::{AbiStr, Blob, Outcome, Span}; use busbar_contract::abi::mechanism::ticket::Ticket; use busbar_contract::abi::plane::{ @@ -30,10 +34,10 @@ use busbar_contract::abi::plane::{ RefusalIn, RefusalOut, UnitCount, AUDIT_APPLIED, CANCEL_ABORTED, CANCEL_FAILED, CANCEL_OK_PARTIAL, EMIT_DONE, EMIT_FINAL_STATUS, EMIT_MESSAGE_END, EMIT_TO_FAR_END, FROM_CALLER, FROM_FAR_END, FROM_KERNEL, PIECE_FIELDS, PIECE_HAS_STATUS, PIECE_LAST, - PIECE_OUT_TEXT, PRINCIPAL_OPTIONAL, RECORD_AUDIT, RECORD_PUT, REFUSAL_ARRIVE, REFUSAL_KERNEL, - ROUTE_LOCAL, ROUTE_SESSION, UNITS_ESTIMATED, UNITS_REPORTED, VERDICT_RETRY, + PIECE_OUT_TEXT, PRINCIPAL_OPTIONAL, RECORD_AUDIT, RECORD_PUT, REFUSAL_ARRIVE, ROUTE_LOCAL, + ROUTE_SESSION, UNITS_ESTIMATED, UNITS_REPORTED, VERDICT_RETRY, }; -use busbar_contract::abi::plane::{ServeIn, ServeOut}; +use busbar_contract::abi::plane::{ServeIn, ServeOut, REFUSAL_GATE}; use busbar_contract::caps::OpClassId; use busbar_contract::plane::{TrustKeyDecl, TrustRole}; use busbar_contract::plane_calls::{ @@ -66,6 +70,11 @@ pub(crate) struct Rig { } impl Rig { + /// Every `project` crossing: the body it was handed and the rewrite, when one came. + pub(crate) fn projects(&self) -> Vec<(Vec, Vec)> { + self.plane.projects.lock().unwrap().clone() + } + pub(crate) fn stats(&self) -> [u64; stat::COUNT] { let mut out = [0; stat::COUNT]; for (k, v) in out.iter_mut().enumerate() { @@ -75,7 +84,17 @@ impl Rig { } } -pub(crate) fn rig(_way: Way, caps: BufferCaps, book: cases::Book) -> Rig { +pub(crate) fn rig(way: Way, caps: BufferCaps, book: cases::Book) -> Rig { + rig_with_hooks(way, caps, book, None) +} + +/// [`rig`], the driver binding `hooks` to each unit. +pub(crate) fn rig_with_hooks( + _way: Way, + caps: BufferCaps, + book: cases::Book, + hooks: Option, +) -> Rig { let plane = Arc::new(Double::default()); let book = Arc::new(book); let driver = PlaneDriver::new( @@ -92,6 +111,10 @@ pub(crate) fn rig(_way: Way, caps: BufferCaps, book: cases::Book) -> Rig { ("test_plane", &serde_yaml::Value::Null), ) .expect("the instance is admitted"); + let driver = match hooks { + Some(h) => driver.with_hooks(Arc::new(h)), + None => driver, + }; Rig { plane, driver, @@ -188,6 +211,8 @@ struct Double { held: Mutex, OnPieceOut)>>, next: AtomicU32, stats: [AtomicU64; stat::COUNT], + /// `project` crossings, and the bodies each was handed (with the rewrite, when one came). + projects: Mutex, Vec)>>, /// The driver tickets minted, and every ticket recycled. drivers: Mutex>, recycled: Mutex>, @@ -567,34 +592,6 @@ impl PlaneCalls for Double { now_ns() } - // The hook stage's projection: the unit's body, unchanged, over the empty entry (the double's - // `arrive` names none). A projection with no body would leave a gate-first unit nothing to - // screen, and the kernel would bind no gate for it. - fn project( - &self, - input: &mut ProjectIn, - out: &mut ProjectOut, - _grow: Grow<'_, ProjectIn, ProjectOut>, - ) -> Outcome { - let body: &[u8] = if input.body.ptr.is_null() || input.body.len == 0 { - &[] - } else { - // SAFETY: the driver's body, lent for the call. - unsafe { std::slice::from_raw_parts(input.body.ptr, input.body.len) } - }; - if body.is_empty() || body.len() > input.arena_cap { - return Outcome::Ready; - } - // SAFETY: the driver's arena of `arena_cap` bytes. - unsafe { std::ptr::copy_nonoverlapping(body.as_ptr(), input.arena_buf, body.len()) }; - out.body = Span { - offset: 0, - len: body.len() as u32, - }; - out.arena_written = body.len() as u64; - Outcome::Ready - } - fn arrive( &self, input: &mut ArriveIn, @@ -710,12 +707,13 @@ impl PlaneCalls for Double { out: &mut RefusalOut, grow: Grow<'_, RefusalIn, RefusalOut>, ) -> Outcome { - // The reason crosses beside its text: a refusal whose code names another reason is FAULT - // (the plane's own words for an arrival it refused are its text, not the kernel's). - let named = - busbar_contract::abi::plane::reason_of(input.reason).map(|r| r.as_str().as_bytes()); - // A gate refusal's text is the vetoing hook's own words. - if input.cause == REFUSAL_KERNEL && named != Some(unsafe { text(input.text) }) { + // The reason crosses beside its text (`RefusalIn::text`, the kernel's own message: the + // reason's spelling, or the walk terminal's own words): a refusal whose code names no reason, + // or that carries no text, is FAULT. A gate's refusal carries the hook's own words instead. + let named = busbar_contract::abi::plane::reason_of(input.reason); + if input.cause != REFUSAL_GATE + && (named.is_none() || unsafe { text(input.text) }.is_empty()) + { return Outcome::Fault; } let mut body = [ @@ -832,6 +830,148 @@ impl PlaneCalls for Double { Outcome::Fault } + /// The view this plane projects: each body line `role:text` is a turn, a `sys=` line the + /// system prompt and a `user=` line the end user. A rewrite (1.5.5's `rewrite` object) is + /// applied by replacing the turns with its messages' `role:content`, unless the target is + /// `/no-rewrite` (a rewrite the plane cannot read); `/unprojectable` refuses to project. The + /// view names the entry its `arrive` recorded (none) and states the unit's body: a projection + /// with no body would leave a gate-first unit nothing to screen, and the kernel would bind no + /// gate for it. + fn project( + &self, + input: &mut ProjectIn, + out: &mut ProjectOut, + grow: Grow<'_, ProjectIn, ProjectOut>, + ) -> Outcome { + let target = unsafe { text(input.target) }.to_vec(); + let body = unsafe { bytes(input.body) }.to_vec(); + let rewrite = unsafe { bytes(input.rewrite) }.to_vec(); + self.projects + .lock() + .unwrap() + .push((body.clone(), rewrite.clone())); + if target == b"/unprojectable" { + return Outcome::Refused; + } + let rewritten = (!rewrite.is_empty() && target != b"/no-rewrite") + .then(|| serde_json::from_slice::(&rewrite).ok()) + .flatten() + .and_then(|v| v.get("messages").and_then(|m| m.as_array()).cloned()) + .map(|messages| { + let lines: Vec = messages + .iter() + .map(|m| { + let field = |k: &str| m.get(k).and_then(|v| v.as_str()).unwrap_or(""); + format!("{}:{}", field("role"), field("content")) + }) + .collect(); + lines.join("\n").into_bytes() + }); + let unit_body = body.clone(); + let effective = rewritten.clone().unwrap_or(body); + let effective = String::from_utf8_lossy(&effective).into_owned(); + let (mut system, mut user, mut turns) = (None, None, Vec::new()); + for line in effective.split('\n').filter(|l| !l.is_empty()) { + if let Some(v) = line.strip_prefix("sys=") { + system = Some(v.to_string()); + } else if let Some(v) = line.strip_prefix("user=") { + user = Some(v.to_string()); + } else { + let (role, text) = line.split_once(':').unwrap_or(("user", line)); + turns.push((role.to_string(), text.to_string())); + } + } + let mut strings: Vec<&[u8]> = vec![b"", b"test-dialect"]; + strings.push(system.as_deref().unwrap_or("").as_bytes()); + strings.push(user.as_deref().unwrap_or("").as_bytes()); + for (role, text) in &turns { + strings.push(role.as_bytes()); + strings.push(text.as_bytes()); + } + let rewritten_bytes = rewritten.unwrap_or_default(); + strings.push(&unit_body); + strings.push(&rewritten_bytes); + let arena: usize = strings.iter().map(|b| b.len()).sum(); + for call in 0..2 { + if turns.len() > input.messages_cap || arena > input.arena_cap { + if call == 1 { + return Outcome::Fault; + } + out.messages_needed = turns.len() as u32; + out.arena_needed = arena as u64; + grow(out, input); + *out = busbar_contract::abi::sdk::door::blank_out(); + continue; + } + // SAFETY (below): the driver's buffers, of the capacities checked above. + let mut at = 0usize; + let mut place = |b: &[u8]| -> AbiStr { + unsafe { + std::ptr::copy_nonoverlapping(b.as_ptr(), input.arena_buf.add(at), b.len()) + }; + let s = AbiStr { + ptr: unsafe { input.arena_buf.add(at) }.cast_const(), + len: b.len(), + }; + at += b.len(); + s + }; + let pool = place(strings[0]); + let dialect = place(strings[1]); + let sys = place(strings[2]); + let end_user = place(strings[3]); + for (k, pair) in strings[4..4 + 2 * turns.len()].chunks(2).enumerate() { + let role = place(pair[0]); + let text = place(pair[1]); + unsafe { *input.messages_buf.add(k) = MessageView { role, text } }; + } + let _ = place(&unit_body); + let _ = place(&rewritten_bytes); + let start = arena - rewritten_bytes.len(); + let body_start = start - unit_body.len(); + out.view = RequestView { + request_id: 0, + pool, + ingress_dialect: dialect, + message_count: turns.len() as u64, + total_chars: turns.iter().map(|(_, t)| t.len() as u64).sum(), + max_tokens: 0, + flags: 0, + signals: input.signals_buf.cast_const(), + signals_len: 0, + session: Blob { + ptr: std::ptr::null(), + len: 0, + fmt: busbar_contract::abi::mechanism::call::BLOB_ABSENT, + flags: 0, + }, + }; + out.prompt.system = sys; + out.prompt.message_count = turns.len() as u64; + out.prompt.messages = input.messages_buf.cast_const(); + out.prompt.messages_len = turns.len(); + out.end_user = end_user; + out.body = if unit_body.is_empty() { + Span { + offset: busbar_contract::abi::plane::SPAN_ABSENT, + len: 0, + } + } else { + Span { + offset: body_start as u32, + len: unit_body.len() as u32, + } + }; + out.rewritten = Span { + offset: start as u32, + len: rewritten_bytes.len() as u32, + }; + out.arena_written = arena as u64; + return Outcome::Ready; + } + Outcome::Fault + } + fn cancel(&self, ticket: Ticket) -> Option { self.count(stat::CANCELS); let u = self.unit_of(ticket); @@ -1978,6 +2118,8 @@ async fn a_gate_first_plane_screens_its_entry_before_the_far_end_and_stops_at_th .contains(&busbar_contract::caps::StepName::Admit), "a veto admits nothing: the door was never asked" ); + // The entry the kernel recorded at decode (this double's `arrive` names none, and its + // projection names the same). assert_eq!(*binder.asked.lock().unwrap(), vec![String::new()]); assert!(far.sent().is_empty(), "nothing reached the far end"); let rendered = units.take_rendered().expect("the veto is rendered"); @@ -2224,6 +2366,7 @@ impl busbar_kernel::plane_driver::FarEnd for Unpresentable { None => Pick::Exhausted { status: 503, retry_after: Some(2), + detail: "The service is temporarily overloaded. Please retry shortly.", }, }; async move { pick } @@ -2297,11 +2440,11 @@ async fn drive_over( /// answers) the kernel's own resolver refuses with the operator's text, which names the source (the /// control: the markers are there to be found). A unit whose every member's credential cannot be /// presented (the auth call refused; the program never spawned) ends at the walk's terminal: the -/// plane's `refusal` is handed the one reason (`breaker_open`, class `Unreachable`) and that -/// reason's own word, and the caller's render is byte-identical whichever source failed and carries -/// none of its markers. RED: a kernel that handed the plane any operator text would fail the -/// double's own rule (a kernel refusal's text is its reason's word, else FAULT and an empty body), -/// the byte-identity across sources, and the marker scan. +/// plane's `refusal` is handed the one reason (`breaker_open`, class `Unreachable`) and the walk +/// terminal's own words (the exhausted pool's 1.5.5 sentence, a `&'static str` the far end's seam +/// carries: no operator text fits it), and the caller's render is byte-identical whichever source +/// failed and carries none of its markers. RED: a kernel that handed the plane any operator text +/// would fail the exact render, the byte-identity across sources, and the marker scan. #[tokio::test] async fn a_credential_that_cannot_be_presented_reaches_the_caller_as_a_fixed_class_refusal() { use busbar_contract::abi::plane::{class_of, RefusalClass}; @@ -2343,8 +2486,9 @@ async fn a_credential_that_cannot_be_presented_reaches_the_caller_as_a_fixed_cla String::from_utf8_lossy(&rendered.body) ); assert_eq!( - rendered.body, b"refused:503:breaker_open:retry=2", - "the plane is handed the fixed reason and its own word: {render}" + rendered.body, + b"refused:503:The service is temporarily overloaded. Please retry shortly.:retry=2", + "the plane is handed the fixed reason and the walk terminal's own words: {render}" ); let found = markers_in(&render, &failing); assert!( diff --git a/crates/busbar-kernel/tests/support/plane_driver_cases.rs b/crates/busbar-kernel/tests/support/plane_driver_cases.rs index 907a50242a..120fd6f726 100644 --- a/crates/busbar-kernel/tests/support/plane_driver_cases.rs +++ b/crates/busbar-kernel/tests/support/plane_driver_cases.rs @@ -102,6 +102,7 @@ impl Far { fail_over: false, fields: false, head: Vec::new(), + relayed: false, }) .collect(); Far { @@ -194,6 +195,7 @@ impl FarEnd for Far { None => Pick::Exhausted { status: 503, retry_after: Some(2), + detail: "The service is temporarily overloaded. Please retry shortly.", }, }; async move { pick } @@ -213,6 +215,7 @@ impl FarEnd for Far { fail_over: true, fields: false, head: Vec::new(), + relayed: false, }] } else if request.member.starts_with("trailers") { // The body, then the far end's trailers after it. @@ -1126,8 +1129,9 @@ async fn exhaustion_answers_the_walks_status_and_retry_after() { let rendered = units.take_rendered().expect("the terminal is rendered"); assert_eq!(rendered.status, 503, "{way:?}"); assert_eq!( - rendered.body, b"refused:503:breaker_open:retry=2", - "{way:?}: the plane renders the walk's status and Retry-After" + rendered.body, + b"refused:503:The service is temporarily overloaded. Please retry shortly.:retry=2", + "{way:?}: the plane renders the walk's status, Retry-After and the shed's own words" ); assert_eq!(caller.text(), "", "{way:?}: nothing had streamed"); } diff --git a/crates/busbar-kernel/tests/support/plane_driver_hook_cases.rs b/crates/busbar-kernel/tests/support/plane_driver_hook_cases.rs new file mode 100644 index 0000000000..9a93a332de --- /dev/null +++ b/crates/busbar-kernel/tests/support/plane_driver_hook_cases.rs @@ -0,0 +1,286 @@ +// SPDX-License-Identifier: Apache-2.0 +// Copyright (C) 2026 Busbar Inc and contributors + +//! THE HOOK STAGE ON THE DRIVER, against the contract-level plane double (`BUSBAR-1.6.0.md` Part 3, +//! section 12, "Hooks"; ARCHITECT K5): no hook bound projects nothing; a request-stage hook's +//! rewrite goes back to the plane (`ProjectIn::rewrite`), which applies it and answers the body the +//! kernel keeps and re-pushes on every attempt; a veto wears the hook's own status and words, +//! rendered by the plane (`REFUSAL_GATE`), before any attempt; a request the plane cannot project +//! is refused before any attempt. The 1.5.5 hook tests themselves run over the real plane serving +//! the `pools` map in the composition root's `hook_parity_driver` suite. + +use std::sync::{Arc, Mutex}; +use std::time::Duration; + +use busbar_contract::caps::{Outcome, ReasonCode}; +use busbar_contract::hooks::{ + BudgetBucketState, Candidate, PolicyResult, RewriteReply, RoutingContext, RoutingDecision, + RoutingPolicy, RoutingRequest, TransformOutcome, +}; +use busbar_kernel::config::PolicyOnError; +use busbar_kernel::hooks::{RequestedSignals, ResolvedPolicy}; +use busbar_kernel::plane_driver::{BoundHooks, BufferCaps, CallerFacts, CallerKey, StageTaps}; + +use super::cases::{arrival, drive, Book, Caller, Far}; +use super::common::TestUnits; +use super::{rig_with_hooks, Way}; + +/// A caller the hooks know nothing of. +struct Nobody; + +impl CallerFacts for Nobody { + fn key(&self, _: &str) -> Option { + None + } + fn in_groups(&self, _: Option<&str>, groups: &[String]) -> bool { + groups.is_empty() + } + fn rate_headroom(&self, _: &str, _: &str) -> Option { + None + } + fn budget(&self, _: &str) -> Vec { + Vec::new() + } +} + +fn bound( + rewrites: Vec<(Duration, Arc)>, + gates: Vec<(u16, ResolvedPolicy)>, +) -> BoundHooks { + BoundHooks { + rewrites, + gates, + pool_rewrites: Default::default(), + pool_gates: Default::default(), + pool_policies: Default::default(), + taps: StageTaps::default(), + requested: RequestedSignals::default(), + next_request_id: Arc::new(|| 1), + caller: Arc::new(Nobody), + dialects: vec!["test-dialect".to_string()], + } +} + +/// The turns a gate was shown, per call. +type Seen = Arc>>>; + +/// A gate that sees the prompt, records it, and decides `reject`. +struct Gate { + seen: Seen, + reject: Option<(u16, &'static str)>, +} + +#[async_trait::async_trait] +impl RoutingPolicy for Gate { + async fn decide( + &self, + req: &RoutingRequest<'_>, + _: &[Candidate<'_>], + _: &RoutingContext<'_>, + _: Duration, + ) -> PolicyResult { + let turns = req + .prompt + .as_ref() + .map(|p| { + p.messages + .iter() + .map(|(r, t)| (r.to_string(), t.to_string())) + .collect() + }) + .unwrap_or_default(); + self.seen.lock().unwrap().push(turns); + Ok(match self.reject { + Some((status, message)) => RoutingDecision::Reject { + status, + message: message.to_string(), + }, + None => RoutingDecision::Abstain, + }) + } + fn name(&self) -> &'static str { + "gate" + } +} + +fn gate(reject: Option<(u16, &'static str)>) -> (Seen, ResolvedPolicy) { + let seen: Seen = Arc::new(Mutex::new(Vec::new())); + ( + seen.clone(), + ResolvedPolicy::Policy { + policy: Arc::new(Gate { seen, reject }), + on_error: PolicyOnError::default(), + on_error_chain: Vec::new(), + timeout: Duration::from_millis(500), + send_prompt: true, + send_user: false, + on_empty: PolicyOnError::Reject, + }, + ) +} + +/// A rewrite hook replacing the turns with one. +struct Rewriter; + +#[async_trait::async_trait] +impl RoutingPolicy for Rewriter { + async fn decide( + &self, + _: &RoutingRequest<'_>, + _: &[Candidate<'_>], + _: &RoutingContext<'_>, + _: Duration, + ) -> PolicyResult { + Ok(RoutingDecision::Abstain) + } + fn name(&self) -> &'static str { + "rewriter" + } + async fn transform(&self, _: &RoutingRequest<'_>, _: Duration) -> TransformOutcome { + TransformOutcome::Rewrite(RewriteReply { + messages: vec![serde_json::json!({"role": "user", "content": "COMPRESSED"})], + tools: vec![], + }) + } +} + +#[tokio::test] +async fn no_hook_bound_projects_nothing() { + let r = rig_with_hooks(Way::Double, BufferCaps::default(), Book::default(), None); + let (steps, far, caller) = ( + TestUnits::passing(), + Far::new(&["ok"], &[b"done"]), + Caller::default(), + ); + let units = r + .driver + .unit(&steps, &far, &caller, arrival("/call", b"user:hello"), 0); + assert!(matches!(drive(&units).await, Outcome::Completed)); + assert!( + r.projects().is_empty(), + "a unit no hook binds crosses no `project`" + ); +} + +#[tokio::test] +async fn a_rewrite_is_applied_by_the_plane_and_re_pushed_on_every_attempt() { + let (seen, g) = gate(None); + let hooks = bound( + vec![(Duration::from_millis(500), Arc::new(Rewriter))], + vec![(0, g)], + ); + let r = rig_with_hooks( + Way::Double, + BufferCaps::default(), + Book::default(), + Some(hooks), + ); + let (steps, far, caller) = ( + TestUnits::passing(), + Far::new(&["overloaded", "ok"], &[b"done"]), + Caller::default(), + ); + let units = r + .driver + .unit(&steps, &far, &caller, arrival("/call", b"user:hello"), 0); + assert!(matches!(drive(&units).await, Outcome::Completed)); + let projects = r.projects(); + assert_eq!( + projects.len(), + 2, + "the first view, then the rewrite applied and re-projected" + ); + assert!( + projects[0].1.is_empty(), + "the first `project` carries no rewrite" + ); + assert!( + !projects[1].1.is_empty(), + "the second carries the hook's rewrite" + ); + assert_eq!( + *seen.lock().unwrap(), + vec![vec![("user".to_string(), "COMPRESSED".to_string())]], + "the gate after the rewrite sees the rewritten request" + ); + let sent = far.sent(); + assert_eq!( + sent.len(), + 2, + "one failed-over attempt, then the served one" + ); + for request in &sent { + assert_eq!( + request.body, b"user:COMPRESSED", + "every attempt carries the body the rewrite left" + ); + } +} + +#[tokio::test] +async fn a_veto_is_rendered_by_the_plane_with_the_hooks_status_and_words_before_any_attempt() { + let (_, g) = gate(Some((451, "blocked\r\nby policy"))); + let r = rig_with_hooks( + Way::Double, + BufferCaps::default(), + Book::default(), + Some(bound(Vec::new(), vec![(0, g)])), + ); + let (steps, far, caller) = ( + TestUnits::passing(), + Far::new(&["ok"], &[b"done"]), + Caller::default(), + ); + let units = r + .driver + .unit(&steps, &far, &caller, arrival("/call", b"user:hello"), 0); + let outcome = drive(&units).await; + assert!( + matches!(outcome, Outcome::Failed(_, ReasonCode::HookVeto)), + "{outcome:?}" + ); + let rendered = units.take_rendered().expect("the plane rendered the veto"); + assert_eq!(rendered.status, 451); + assert_eq!( + rendered.body, b"refused:451:blockedby policy", + "the hook's status and its sanitised words, in the plane's rendering" + ); + assert!(far.sent().is_empty(), "no attempt was made"); +} + +#[tokio::test] +async fn a_request_the_plane_cannot_project_is_refused_before_any_attempt() { + let (seen, g) = gate(None); + let r = rig_with_hooks( + Way::Double, + BufferCaps::default(), + Book::default(), + Some(bound(Vec::new(), vec![(0, g)])), + ); + let (steps, far, caller) = ( + TestUnits::passing(), + Far::new(&["ok"], &[b"done"]), + Caller::default(), + ); + let units = r.driver.unit( + &steps, + &far, + &caller, + arrival("/unprojectable", b"user:hello"), + 0, + ); + let outcome = drive(&units).await; + assert!( + matches!(outcome, Outcome::Failed(_, ReasonCode::DecodeFailed)), + "{outcome:?}" + ); + assert!( + seen.lock().unwrap().is_empty(), + "no hook saw an unreadable request" + ); + assert!(far.sent().is_empty(), "no attempt was made"); + let rendered = units + .take_rendered() + .expect("the plane rendered the refusal"); + assert_eq!(rendered.status, 400); +} diff --git a/crates/busbar-llm/src/engine/tests/plane_attempt_parity_tests.rs b/crates/busbar-llm/src/engine/tests/plane_attempt_parity_tests.rs index 3767f39a7b..6264f9dddc 100644 --- a/crates/busbar-llm/src/engine/tests/plane_attempt_parity_tests.rs +++ b/crates/busbar-llm/src/engine/tests/plane_attempt_parity_tests.rs @@ -30,6 +30,7 @@ fn plane_lane(dialect: &'static str, model: &str, path: Option<&str>, base: Opti prompt_caching: false, caps: Default::default(), error_map: Default::default(), + statics: &[], } } diff --git a/crates/busbar-llm/src/engine/tests/plane_reply_parity_tests.rs b/crates/busbar-llm/src/engine/tests/plane_reply_parity_tests.rs index 4faf0bf30a..a5f611e864 100644 --- a/crates/busbar-llm/src/engine/tests/plane_reply_parity_tests.rs +++ b/crates/busbar-llm/src/engine/tests/plane_reply_parity_tests.rs @@ -301,6 +301,7 @@ fn lane(egress: &'static str) -> Lane { prompt_caching: false, caps: Default::default(), error_map: Default::default(), + statics: &[], } } diff --git a/crates/busbar-plane-llm/src/codec/anthropic/mod.rs b/crates/busbar-plane-llm/src/codec/anthropic/mod.rs index 2fe4cc635c..5c73671c64 100644 --- a/crates/busbar-plane-llm/src/codec/anthropic/mod.rs +++ b/crates/busbar-plane-llm/src/codec/anthropic/mod.rs @@ -197,6 +197,9 @@ pub const DECL: ProtocolDecl = ProtocolDecl { // Anthropic Python SDK is Stainless-generated and emits `/Python <ver>`. RELEASE // OBLIGATION: re-verify each version against the latest published SDK before a release and bump // (the `test_egress_ua_versions_are_pinned_and_present` guard forces the change to be conscious). + // OWNER RULING Q10: the plane writes this on a translated route into this dialect, on a + // same-dialect route whose caller sent no user-agent (1.5.5's bytes), and on the + // kernel-originated health probe; a same-dialect caller's own user-agent replaces it. egress_user_agent: "Anthropic/Python 0.39.0", has_model_in_url: false, auth_failure_status_and_kind: ( diff --git a/crates/busbar-plane-llm/src/codec/anthropic/reader.rs b/crates/busbar-plane-llm/src/codec/anthropic/reader.rs index fd98f66675..85bf6662e2 100644 --- a/crates/busbar-plane-llm/src/codec/anthropic/reader.rs +++ b/crates/busbar-plane-llm/src/codec/anthropic/reader.rs @@ -210,6 +210,7 @@ impl ProtocolReader for AnthropicReader { map: super::map::RESPONSE_PATHS, code: super::RESPONSE_CODE, drops: super::RESPONSE_DROPS, + defaults: &[], }) } @@ -218,6 +219,7 @@ impl ProtocolReader for AnthropicReader { map: super::map::STREAM_PATHS, code: super::STREAM_CODE, drops: super::STREAM_DROPS, + defaults: &[], }) } diff --git a/crates/busbar-plane-llm/src/codec/anthropic/tests/anthropic_hardening_tests.rs b/crates/busbar-plane-llm/src/codec/anthropic/tests/anthropic_hardening_tests.rs index f075be2774..df823a2ca5 100644 --- a/crates/busbar-plane-llm/src/codec/anthropic/tests/anthropic_hardening_tests.rs +++ b/crates/busbar-plane-llm/src/codec/anthropic/tests/anthropic_hardening_tests.rs @@ -4135,3 +4135,42 @@ fn unmodelled_block_is_left_out_of_the_ir_and_spliced_back_same_dialect() { "{out}" ); } + +/// THE TAIL RESTATES THIS SCHEME as the `api-key` style's parameters (ARCHITECT RULING 2026-10-03, +/// Q-L6-AUTHPARAMS): the same families, own header and passthrough, in the auth plugin's words. +#[test] +fn the_tails_api_key_parameters_restate_the_credential_family_table() { + use busbar_contract::protocol::CredentialHeader as H; + let Some(EgressScheme::Static { + families, + own, + passthrough, + }) = DECL.egress_scheme + else { + panic!("anthropic declares a static credential scheme"); + }; + let words = |h: H| match h { + H::Raw { header, trim_start } => { + let mut o = serde_json::json!({ "header": header }); + if trim_start { + o["trim_start"] = serde_json::json!(true); + } + o + } + H::Bearer => serde_json::json!({}), + }; + let params: serde_json::Value = + serde_json::from_str(crate::dialect::KEY_FAMILY_PARAMS).expect("the parameters are JSON"); + let stated: Vec<serde_json::Value> = families + .iter() + .map(|f| { + let mut row = words(f.presented_as); + row["prefix"] = serde_json::json!(f.prefix); + row + }) + .collect(); + assert_eq!(params["families"], serde_json::json!(stated)); + assert_eq!(params["own"], words(own)); + assert_eq!(params["header"], words(own)["header"]); + assert_eq!(params["passthrough"], words(passthrough)); +} diff --git a/crates/busbar-plane-llm/src/codec/bedrock/mod.rs b/crates/busbar-plane-llm/src/codec/bedrock/mod.rs index 7b227c43a0..9c8f24a063 100644 --- a/crates/busbar-plane-llm/src/codec/bedrock/mod.rs +++ b/crates/busbar-plane-llm/src/codec/bedrock/mod.rs @@ -237,6 +237,9 @@ pub const DECL: ProtocolDecl = ProtocolDecl { max_citations_per_delta: Some(1), // AWS Bedrock is reached via boto3/botocore. RELEASE OBLIGATION: re-verify/bump per release; // `test_egress_ua_versions_are_pinned_and_present` guards drift. + // OWNER RULING Q10: the plane writes this on a translated route into this dialect, on a + // same-dialect route whose caller sent no user-agent (1.5.5's bytes), and on the + // kernel-originated health probe; a same-dialect caller's own user-agent replaces it. egress_user_agent: "Boto3/1.35.0 md/Botocore#1.35.0", has_model_in_url: true, auth_failure_status_and_kind: ( diff --git a/crates/busbar-plane-llm/src/codec/bedrock/reader.rs b/crates/busbar-plane-llm/src/codec/bedrock/reader.rs index b364c4df27..a7e1c4f983 100644 --- a/crates/busbar-plane-llm/src/codec/bedrock/reader.rs +++ b/crates/busbar-plane-llm/src/codec/bedrock/reader.rs @@ -1489,6 +1489,7 @@ impl ProtocolReader for BedrockReader { map: super::map::RESPONSE_PATHS, code: super::RESPONSE_CODE, drops: super::RESPONSE_DROPS, + defaults: &[], }) } @@ -1497,6 +1498,7 @@ impl ProtocolReader for BedrockReader { map: super::map::STREAM_PATHS, code: super::STREAM_CODE, drops: super::STREAM_DROPS, + defaults: &[], }) } diff --git a/crates/busbar-plane-llm/src/codec/bedrock/tests/tests.rs b/crates/busbar-plane-llm/src/codec/bedrock/tests/tests.rs index 84d6406bd0..d108b63a45 100644 --- a/crates/busbar-plane-llm/src/codec/bedrock/tests/tests.rs +++ b/crates/busbar-plane-llm/src/codec/bedrock/tests/tests.rs @@ -6729,3 +6729,45 @@ fn the_served_tier_is_read_and_written() { }); assert_eq!(tier.as_deref(), Some("flex")); } + +/// THE TAIL RESTATES THIS SCHEME as the `sigv4` style's parameters (ARCHITECT RULING 2026-10-03, +/// Q-L6-AUTHPARAMS): the service, the content type, and the region read from the provider's host +/// after the service labels the codec's rule reads, else its default with its warning. +#[test] +fn the_tails_sigv4_parameters_restate_the_signing_scheme() { + let Some(busbar_contract::protocol::EgressScheme::SigV4 { + service, + region_of_host, + default_region, + content_type, + }) = super::DECL.egress_scheme + else { + panic!("the signing dialect declares a SigV4 scheme"); + }; + let params: serde_json::Value = + serde_json::from_str(crate::dialect::SIGNED_PARAMS).expect("the parameters are JSON"); + assert_eq!(params["service"], serde_json::json!(service)); + assert_eq!(params["content_type"], serde_json::json!(content_type)); + assert_eq!( + params["region"]["default"], + serde_json::json!(default_region) + ); + let labels: Vec<&str> = params["region"]["host_label_after"] + .as_array() + .expect("the region is read after the service labels") + .iter() + .filter_map(serde_json::Value::as_str) + .collect(); + // Every label the codec's rule reads a region after, and no other. + for label in &labels { + assert_eq!( + region_of_host(&format!("{label}.eu-west-3.amazonaws.com")), + Some("eu-west-3"), + "{label}" + ); + } + assert_eq!(region_of_host("runtime.eu-west-3.amazonaws.com"), None); + assert!(params["region"]["unread"] + .as_str() + .is_some_and(|w| w.contains("defaulting SigV4 scope to us-east-1"))); +} diff --git a/crates/busbar-plane-llm/src/codec/cohere/mod.rs b/crates/busbar-plane-llm/src/codec/cohere/mod.rs index b76a382e1e..87476b3e99 100644 --- a/crates/busbar-plane-llm/src/codec/cohere/mod.rs +++ b/crates/busbar-plane-llm/src/codec/cohere/mod.rs @@ -107,6 +107,9 @@ pub const DECL: ProtocolDecl = ProtocolDecl { max_citations_per_delta: Some(1), // Cohere Python SDK UA. RELEASE OBLIGATION: re-verify/bump per release; // `test_egress_ua_versions_are_pinned_and_present` guards drift. + // OWNER RULING Q10: the plane writes this on a translated route into this dialect, on a + // same-dialect route whose caller sent no user-agent (1.5.5's bytes), and on the + // kernel-originated health probe; a same-dialect caller's own user-agent replaces it. egress_user_agent: "cohere-python/5.11.0", has_model_in_url: false, auth_failure_status_and_kind: ( diff --git a/crates/busbar-plane-llm/src/codec/cohere/reader.rs b/crates/busbar-plane-llm/src/codec/cohere/reader.rs index 500f565d37..1dd8d1ecb5 100644 --- a/crates/busbar-plane-llm/src/codec/cohere/reader.rs +++ b/crates/busbar-plane-llm/src/codec/cohere/reader.rs @@ -1251,6 +1251,7 @@ impl ProtocolReader for CohereReader { map: super::map::RESPONSE_PATHS, code: super::RESPONSE_CODE, drops: super::RESPONSE_DROPS, + defaults: &[], }) } @@ -1259,6 +1260,7 @@ impl ProtocolReader for CohereReader { map: super::map::STREAM_PATHS, code: super::STREAM_CODE, drops: super::STREAM_DROPS, + defaults: &[], }) } diff --git a/crates/busbar-plane-llm/src/codec/drops.rs b/crates/busbar-plane-llm/src/codec/drops.rs index 0835bbc298..0c9b58c4f2 100644 --- a/crates/busbar-plane-llm/src/codec/drops.rs +++ b/crates/busbar-plane-llm/src/codec/drops.rs @@ -642,6 +642,12 @@ pub struct Carried { /// never scanned (Responses lists 763 stream paths; test /// `every_drop_list_is_sorted_for_the_walks_search`). pub drops: &'static [&'static str], + /// The request defaults the far end ECHOES in its answer, each a member's key and its default + /// as JSON text (`("tool_choice", "\"auto\"")`). A member equal to its default carries nothing + /// the caller set, so its drop is no unrepresentable control and the walk does not name it; + /// any other value of it is named like every drop (ARCHITECT RULING 2026-10-04: a dropped value + /// equal to the protocol default, or an echoed default, writes no audit row). + pub defaults: &'static [(&'static str, &'static str)], } /// Where a wire path stands against what a dialect carries. @@ -710,6 +716,13 @@ impl Carried { } } + /// `value`, under `key`, is the default the dialect declares the far end echoes for it. + fn echoes_default(&self, key: &str, value: &Value) -> bool { + self.defaults.iter().any(|(k, default)| { + *k == key && serde_json::from_str::<Value>(default).is_ok_and(|d| d == *value) + }) + } + /// The objects at `path` are a union keyed by their `type` member (notation A `type=<arm>`). fn uses_arms(&self, path: &str) -> bool { let arm = join(path, "type="); @@ -782,7 +795,10 @@ impl Carried { None => path.to_string(), }; for (key, child) in obj { - if child.is_null() || (arm.is_some() && key == crate::codec::keys::TYPE) { + if child.is_null() + || (arm.is_some() && key == crate::codec::keys::TYPE) + || self.echoes_default(key, child) + { continue; } let p = join(&base, key); diff --git a/crates/busbar-plane-llm/src/codec/gemini/mod.rs b/crates/busbar-plane-llm/src/codec/gemini/mod.rs index 80b8e1f205..e11c04400e 100644 --- a/crates/busbar-plane-llm/src/codec/gemini/mod.rs +++ b/crates/busbar-plane-llm/src/codec/gemini/mod.rs @@ -182,6 +182,9 @@ pub const DECL: ProtocolDecl = ProtocolDecl { max_citations_per_delta: None, // Google GenAI SDK UA. RELEASE OBLIGATION: re-verify/bump per release; // `test_egress_ua_versions_are_pinned_and_present` guards drift. + // OWNER RULING Q10: the plane writes this on a translated route into this dialect, on a + // same-dialect route whose caller sent no user-agent (1.5.5's bytes), and on the + // kernel-originated health probe; a same-dialect caller's own user-agent replaces it. egress_user_agent: "google-genai-sdk/0.8.0 gl-python/3.11", has_model_in_url: true, auth_failure_status_and_kind: ( diff --git a/crates/busbar-plane-llm/src/codec/gemini/reader.rs b/crates/busbar-plane-llm/src/codec/gemini/reader.rs index fcdeb130dd..4777a80a7b 100644 --- a/crates/busbar-plane-llm/src/codec/gemini/reader.rs +++ b/crates/busbar-plane-llm/src/codec/gemini/reader.rs @@ -1618,6 +1618,7 @@ impl ProtocolReader for GeminiReader { map: super::map::RESPONSE_PATHS, code: super::RESPONSE_CODE, drops: super::RESPONSE_DROPS, + defaults: &[], }) } @@ -1626,6 +1627,7 @@ impl ProtocolReader for GeminiReader { map: super::map::RESPONSE_PATHS, code: super::RESPONSE_CODE, drops: super::RESPONSE_DROPS, + defaults: &[], }) } diff --git a/crates/busbar-plane-llm/src/codec/openai_chat/mod.rs b/crates/busbar-plane-llm/src/codec/openai_chat/mod.rs index f180d786ac..378f86ccb8 100644 --- a/crates/busbar-plane-llm/src/codec/openai_chat/mod.rs +++ b/crates/busbar-plane-llm/src/codec/openai_chat/mod.rs @@ -137,6 +137,9 @@ pub const DECL: ProtocolDecl = ProtocolDecl { max_citations_per_delta: None, // OpenAI Python SDK UA (the Responses surface shares it). RELEASE OBLIGATION: re-verify/bump per // release; the `test_egress_ua_versions_are_pinned_and_present` guard forces a conscious change. + // OWNER RULING Q10: the plane writes this on a translated route into this dialect, on a + // same-dialect route whose caller sent no user-agent (1.5.5's bytes), and on the + // kernel-originated health probe; a same-dialect caller's own user-agent replaces it. egress_user_agent: "OpenAI/Python 1.54.0", has_model_in_url: false, auth_failure_status_and_kind: ( diff --git a/crates/busbar-plane-llm/src/codec/openai_chat/reader.rs b/crates/busbar-plane-llm/src/codec/openai_chat/reader.rs index 556264650d..85064b56e9 100644 --- a/crates/busbar-plane-llm/src/codec/openai_chat/reader.rs +++ b/crates/busbar-plane-llm/src/codec/openai_chat/reader.rs @@ -1248,6 +1248,7 @@ impl ProtocolReader for OpenAiReader { map: super::map::RESPONSE_PATHS, code: super::RESPONSE_CODE, drops: super::RESPONSE_DROPS, + defaults: &[], }) } @@ -1256,6 +1257,7 @@ impl ProtocolReader for OpenAiReader { map: super::map::STREAM_PATHS, code: super::STREAM_CODE, drops: super::STREAM_DROPS, + defaults: &[], }) } diff --git a/crates/busbar-plane-llm/src/codec/openai_responses/mod.rs b/crates/busbar-plane-llm/src/codec/openai_responses/mod.rs index dda5c4d002..ba4f3fe1ea 100644 --- a/crates/busbar-plane-llm/src/codec/openai_responses/mod.rs +++ b/crates/busbar-plane-llm/src/codec/openai_responses/mod.rs @@ -102,6 +102,9 @@ pub const DECL: ProtocolDecl = ProtocolDecl { max_citations_per_delta: None, // Same OpenAI Python SDK UA as the Chat surface (one vendor, two surfaces). RELEASE OBLIGATION: // re-verify/bump per release; `test_egress_ua_versions_are_pinned_and_present` guards drift. + // OWNER RULING Q10: the plane writes this on a translated route into this dialect, on a + // same-dialect route whose caller sent no user-agent (1.5.5's bytes), and on the + // kernel-originated health probe; a same-dialect caller's own user-agent replaces it. egress_user_agent: "OpenAI/Python 1.54.0", has_model_in_url: false, auth_failure_status_and_kind: ( @@ -1137,6 +1140,16 @@ const STREAM_CODE: &[&str] = &[ /// The answer paths INSIDE a subtree this dialect carries that its code does not carry, named by the /// drop walk (DF-MAP-IR-GAPS section E: its A, B and C paths that a coarse map row covers). +/// THE REQUEST DEFAULTS A RESPONSES ANSWER ECHOES (the response object restates the request's +/// `tools`, `tool_choice` and `parallel_tool_calls`, at their defaults when the request set none): +/// an echoed default is no control the caller set, so a TRANSLATE answer that cannot carry it +/// names no drop for it (ARCHITECT RULING 2026-10-04). Keyed by the member, its default as JSON. +pub(super) const ANSWER_DEFAULTS: &[(&str, &str)] = &[ + (keys::TOOLS, "[]"), + (keys::TOOL_CHOICE, "\"auto\""), + (keys::PARALLEL_TOOL_CALLS, "true"), +]; + const RESPONSE_DROPS: &[&str] = &[ "error.misalignment", "instructions[].type=additional_tools", diff --git a/crates/busbar-plane-llm/src/codec/openai_responses/reader.rs b/crates/busbar-plane-llm/src/codec/openai_responses/reader.rs index 1bbfdb7326..7e7f97411c 100644 --- a/crates/busbar-plane-llm/src/codec/openai_responses/reader.rs +++ b/crates/busbar-plane-llm/src/codec/openai_responses/reader.rs @@ -1881,6 +1881,7 @@ impl ProtocolReader for ResponsesReader { map: super::map::RESPONSE_PATHS, code: super::RESPONSE_CODE, drops: super::RESPONSE_DROPS, + defaults: super::ANSWER_DEFAULTS, }) } @@ -1889,6 +1890,7 @@ impl ProtocolReader for ResponsesReader { map: super::map::STREAM_PATHS, code: super::STREAM_CODE, drops: super::STREAM_DROPS, + defaults: super::ANSWER_DEFAULTS, }) } diff --git a/crates/busbar-plane-llm/src/codec/tests/drops_tests.rs b/crates/busbar-plane-llm/src/codec/tests/drops_tests.rs index 52f39c5a27..fb10c434c6 100644 --- a/crates/busbar-plane-llm/src/codec/tests/drops_tests.rs +++ b/crates/busbar-plane-llm/src/codec/tests/drops_tests.rs @@ -586,6 +586,7 @@ fn the_answer_walk_names_the_shallowest_uncarried_member() { ], code: &["type=ping"], drops: &[], + defaults: &[], }; let answer = json!({"id": "m", "content": [{"type": "x", "deep": 1}], "container": {"id": "c"}, "usage": {"input_tokens": 1, "output_tokens_details": {"t": 1}}, "stop_details": null}); @@ -612,6 +613,7 @@ fn the_answer_walk_names_the_shallowest_uncarried_member() { map: &["contentBlockDelta.delta.text"], code: &[], drops: &[], + defaults: &[], }; assert_eq!( keyed.unmapped( @@ -622,6 +624,33 @@ fn the_answer_walk_names_the_shallowest_uncarried_member() { ); } +/// AN ECHOED DEFAULT IS NO UNREPRESENTABLE CONTROL (ARCHITECT RULING 2026-10-04): a Responses +/// answer restates the request's `tools`, `tool_choice` and `parallel_tool_calls`, at their +/// defaults when the caller set none; a TRANSLATE answer drops them, and the walk names none, so +/// no `egress.control_unrepresentable` row is written for them (the 1.5.5 rows for +/// `llm|*|responses|request|ok`). RED arm: a value the caller actually set (a real tool, a +/// directed choice, parallel calls off) is still named, so its drop is still audited. +#[test] +fn an_echoed_request_default_names_no_drop_and_a_caller_set_value_still_does() { + let carried = crate::codec::proto_codec::with_reader("responses", |r| r.response_carried()) + .flatten() + .expect("the Responses answer is walked"); + let echoed = json!({"id": "resp_1", "object": "response", "status": "completed", + "output": [], "tools": [], "tool_choice": "auto", "parallel_tool_calls": true}); + assert!( + carried.unmapped("", &echoed).is_empty(), + "an echoed default is no control the caller set: {:?}", + carried.unmapped("", &echoed) + ); + let set = json!({"id": "resp_1", "object": "response", "status": "completed", + "output": [], + "tools": [{"type": "function", "name": "lookup", "parameters": {"type": "object"}}], + "tool_choice": "required", "parallel_tool_calls": false}); + let mut named = carried.unmapped("", &set); + named.sort(); + assert_eq!(named, vec!["parallel_tool_calls", "tool_choice", "tools"]); +} + /// DF-MAP-IR-GAPS SECTION E (ruled: every unmapped response/stream path is named, no extra /// bucket): every wire path of its list is named by the far end's drop walk when an answer carries /// it. RED before: the walk did not exist (a buffered answer named blocks and vendor metadata only; @@ -708,7 +737,17 @@ fn the_answer_walk_names_only_true_drops_over_the_golden_corpus() { .copied() .filter(|p| { let pointer = format!("/{}", p.replace('.', "/")); - body.pointer(&pointer).is_some_and(|v| !v.is_null()) + // An echoed request default is no drop (ARCHITECT RULING 2026-10-04). + let echoed = |v: &Value| { + far == "responses" + && crate::codec::openai_responses::ANSWER_DEFAULTS + .iter() + .any(|(k, d)| { + *k == *p && serde_json::from_str::<Value>(d).is_ok_and(|d| d == *v) + }) + }; + body.pointer(&pointer) + .is_some_and(|v| !v.is_null() && !echoed(v)) }) .collect(); assert_eq!(named, expected, "{name}"); diff --git a/crates/busbar-plane-llm/src/dialect.rs b/crates/busbar-plane-llm/src/dialect.rs index 88e7e392aa..0482006649 100644 --- a/crates/busbar-plane-llm/src/dialect.rs +++ b/crates/busbar-plane-llm/src/dialect.rs @@ -49,6 +49,14 @@ pub struct Dialect { pub scheme_alt: &'static str, /// The egress-auth scheme that decorates a request to an upstream of this dialect. pub egress_scheme: &'static str, + /// THE DIALECT'S DEFAULT OUTBOUND AUTH STYLE (the design's auth points: a member's style is its + /// provider's `auth:`, else its plane's default for its dialect; ARCHITECT Q-L1-AUTH (A)): the + /// style an auth plugin serves, stated in the plane tail's `dialect_auth`. + pub egress_style: &'static str, + /// THE DEFAULT STYLE'S PARAMETERS for this dialect, a JSON object (`""`: none), stated in the + /// plane tail's `dialect_auth.params` and handed the auth plugin's `open_outbound` as its + /// settings at seal, under the provider's own (ARCHITECT RULING 2026-10-03, Q-L6-AUTHPARAMS). + pub egress_params: &'static str, /// The request headers busbar GOVERNS for this dialect (lower-case): its credential headers and /// its tenant selectors. A same-dialect route forwards every other client header unchanged; these /// never pass, because busbar's upstream credential and configuration replace them (OWNER HARD @@ -95,6 +103,28 @@ const OPENAI_GOVERNED: &[&str] = &[ "openai-project", ]; +/// The `api-key` style's parameters for the dialect whose credentials come in families (its +/// codec's `EgressScheme::Static`): an api key presented in `x-api-key` with leading whitespace +/// trimmed, an oauth token as a bearer, the operator's own key in `x-api-key`, a passthrough caller's +/// as a bearer. +pub(crate) const KEY_FAMILY_PARAMS: &str = concat!( + r#"{"header":"x-api-key","#, + r#""families":[{"prefix":"sk-ant-api","header":"x-api-key","trim_start":true},"#, + r#"{"prefix":"sk-ant-oat"}],"#, + r#""own":{"header":"x-api-key"},"passthrough":{}}"#, +); + +/// The `sigv4` style's parameters for the signed dialect (its codec's `EgressScheme::SigV4`): the +/// service, the content type the signature covers, and the region read from the provider's host by +/// the codec's rule (`derive_sigv4_region`), else its default with the codec's warning. +pub(crate) const SIGNED_PARAMS: &str = concat!( + r#"{"service":"bedrock","content_type":"application/json","#, + r#""region":{"host_label_after":["bedrock-runtime","bedrock-runtime-fips","bedrock","bedrock-fips"],"#, + r#""default":"us-east-1","#, + r#""unread":"could not derive AWS region from Bedrock endpoint host; defaulting SigV4 scope to "#, + r#"us-east-1 (set a bedrock-runtime[-fips].<region>.amazonaws.com host)"}}"#, +); + /// The table, one row per dialect, in the order the codec crate declares them. pub const DIALECTS: &[Dialect] = &[ Dialect { @@ -108,6 +138,8 @@ pub const DIALECTS: &[Dialect] = &[ cache_write_pointer: Some("/usage/cache_creation_input_tokens"), scheme_alt: "api-key", egress_scheme: "bearer", + egress_style: "api-key", + egress_params: KEY_FAMILY_PARAMS, governed_headers: &["authorization", "x-api-key"], governed_query: &[], tenant_headers: &[], @@ -128,6 +160,8 @@ pub const DIALECTS: &[Dialect] = &[ cache_write_pointer: None, scheme_alt: "bearer", egress_scheme: "bearer", + egress_style: "bearer", + egress_params: "", governed_headers: OPENAI_GOVERNED, governed_query: &[], tenant_headers: OPENAI_TENANT, @@ -145,6 +179,8 @@ pub const DIALECTS: &[Dialect] = &[ cache_write_pointer: None, scheme_alt: "api-key", egress_scheme: "bearer", + egress_style: "x-goog-api-key", + egress_params: "", governed_headers: &["authorization", "x-goog-api-key", "x-goog-user-project"], governed_query: &["key"], tenant_headers: &[], @@ -162,6 +198,8 @@ pub const DIALECTS: &[Dialect] = &[ cache_write_pointer: Some("/usage/cacheWriteInputTokens"), scheme_alt: "request-signature", egress_scheme: "request-signature", + egress_style: "sigv4", + egress_params: SIGNED_PARAMS, governed_headers: &[ "authorization", "x-amz-date", @@ -183,6 +221,8 @@ pub const DIALECTS: &[Dialect] = &[ cache_write_pointer: Some("/usage/input_tokens_details/cache_write_tokens"), scheme_alt: "bearer", egress_scheme: "bearer", + egress_style: "bearer", + egress_params: "", governed_headers: OPENAI_GOVERNED, governed_query: &[], tenant_headers: OPENAI_TENANT, @@ -200,6 +240,8 @@ pub const DIALECTS: &[Dialect] = &[ cache_write_pointer: None, scheme_alt: "bearer", egress_scheme: "bearer", + egress_style: "bearer", + egress_params: "", governed_headers: &["authorization"], governed_query: &[], tenant_headers: &[], diff --git a/crates/busbar-plane-llm/src/exchange/attempt.rs b/crates/busbar-plane-llm/src/exchange/attempt.rs index e2deb4d10d..77a4e80a57 100644 --- a/crates/busbar-plane-llm/src/exchange/attempt.rs +++ b/crates/busbar-plane-llm/src/exchange/attempt.rs @@ -608,12 +608,23 @@ fn legal_field_value(v: &[u8]) -> bool { v.iter().all(|b| *b == b'\t' || (0x20..0x7f).contains(b)) } +/// The `accept` field: busbar's own, the answer framing it reads the far end's reply in (the +/// operation's single answer or its dialect's stream framing), written on every far request exactly +/// as 1.5.5 wrote it (its reqwest client sent its own `accept` and never fell back to a transport +/// default), and never replaced by a same-dialect caller's value (1.5.5's bytes: the oracle's +/// same-dialect cells carry busbar's `application/json`, not the caller's `*/*`). +const ACCEPT: &str = "accept"; + /// The head fields a native client of `lane`'s dialect sends, /// then, when the caller speaks that dialect, every field the caller sent but the ones busbar /// governs ([`governed`]): /// busbar is invisible to upstreams (OWNER HARD RULE 2026-10-02). The caller's value of a name /// replaces busbar's native default; the per-connection mechanics are the kernel's to drop as it /// writes the head. A translated route forwards no caller field: none maps between dialects. +/// +/// `user-agent` (OWNER RULING Q10, LLM-UA): a translated route writes the far dialect's declared +/// `egress_user_agent`. A same-dialect route carries the caller's own unchanged, and a caller that +/// sent none gets the far dialect's declared fingerprint, as 1.5.5 sent. fn head_fields( lane: &Lane, handler: &dyn OperationHandler, @@ -644,7 +655,7 @@ fn head_fields( d.egress_stream_accept }); let accept = ( - "accept".to_string(), + ACCEPT.to_string(), handler .egress_accept(stream_accept, wants_stream) .as_bytes() @@ -662,7 +673,14 @@ fn head_fields( // A native client's user-agent, never a UA-less request (1.5.5's bytes); a same-dialect // caller's own replaces it below. let user_agent = ("user-agent".to_string(), user_agent.as_bytes().to_vec()); - let mut fields = vec![content_type, user_agent, accept]; + // The dialect's declared static fields first, as the previous release's credential map led + // with them (SD-3 (1)); a same-dialect caller's own value replaces one below. + let mut fields: Vec<(String, Vec<u8>)> = lane + .statics + .iter() + .map(|(n, v)| ((*n).to_string(), v.as_bytes().to_vec())) + .collect(); + fields.extend([content_type, user_agent, accept]); if arrived.dialect != egress { fields.extend(tenant); return Ok(fields); @@ -672,15 +690,50 @@ fn head_fields( .iter() .filter_map(|(name, value)| { let name = std::str::from_utf8(name).ok()?.to_ascii_lowercase(); - (!governed(&name)).then(|| (name, value.to_vec())) + (!governed(&name) && name != ACCEPT).then(|| (name, value.to_vec())) }) .collect(); - fields.retain(|(own, _)| !forwarded.iter().any(|(name, _)| name == own)); - fields.extend(forwarded); + fields = forward_over(fields, forwarded); fields.extend(tenant); Ok(fields) } +/// The caller's forwarded fields laid over busbar's own as the previous release's header map laid +/// them (an `http::HeaderMap`, which keeps one position per name, the position the name was first +/// inserted at): a name busbar already writes keeps busbar's position and takes the caller's values; +/// a name it does not is appended, in the caller's order; every value of a repeated name is kept, +/// together at its name's position. +fn forward_over( + own: Vec<(String, Vec<u8>)>, + forwarded: Vec<(String, Vec<u8>)>, +) -> Vec<(String, Vec<u8>)> { + let mut named: Vec<(String, Vec<Vec<u8>>)> = Vec::new(); + let put = |named: &mut Vec<(String, Vec<Vec<u8>>)>, name: String, value: Vec<u8>| match named + .iter_mut() + .find(|(n, _)| *n == name) + { + Some((_, values)) => values.push(value), + None => named.push((name, vec![value])), + }; + for (name, value) in own { + put(&mut named, name, value); + } + let mut replaced: Vec<String> = Vec::new(); + for (name, value) in forwarded { + if !replaced.contains(&name) { + if let Some((_, values)) = named.iter_mut().find(|(n, _)| *n == name) { + values.clear(); + } + replaced.push(name.clone()); + } + put(&mut named, name, value); + } + named + .into_iter() + .flat_map(|(name, values)| values.into_iter().map(move |v| (name.clone(), v))) + .collect() +} + /// BUILD ONE ATTEMPT'S FAR-END REQUEST for `member` of `pool`. /// /// # Errors diff --git a/crates/busbar-plane-llm/src/exchange/mod.rs b/crates/busbar-plane-llm/src/exchange/mod.rs index 5ad9e01de3..ab0a6a4030 100644 --- a/crates/busbar-plane-llm/src/exchange/mod.rs +++ b/crates/busbar-plane-llm/src/exchange/mod.rs @@ -6,6 +6,7 @@ pub mod arrive; pub mod attempt; pub mod multipart; pub mod probe; +pub mod project; pub mod refuse; pub mod reply; pub mod shaping; diff --git a/crates/busbar-plane-llm/src/exchange/project.rs b/crates/busbar-plane-llm/src/exchange/project.rs new file mode 100644 index 0000000000..a2b636c473 --- /dev/null +++ b/crates/busbar-plane-llm/src/exchange/project.rs @@ -0,0 +1,197 @@ +//! THE HOOK PROJECTION, sans I/O: what a hook is shown of an arrival (`BUSBAR-1.6.0.md` Part 3, +//! section 12, "Hooks"; THE DESIGN section 11.7), and a request-stage hook's rewrite applied to +//! the arrival in its own dialect. +//! +//! One read, through the operation's own reader, as the previous release's hook seam read it: the +//! size signals (turns, characters, tools), the end user the dialect spells, and the prompt view +//! (the system field and one `(role, text)` entry per wire turn, an in-band system turn at its wire +//! position, a turn with nothing readable still an entry). A body the reader refuses is the +//! request's failure ([`Unreadable`]); a body there is no reader for projects the zeroed shape. +//! +//! The output-cap signal is the previous release's own read (`max_tokens_for` at `v1.5.5`): the +//! dialect's cap key off the caller's body, an out-of-range cap saturated to `u32::MAX`, never +//! dropped. The IR's cap is the far end's, and a reader drops a cap it cannot carry; the hook +//! signal is the caller's ask. + +use busbar_contract::hooks::RewriteReply; +use busbar_contract::ir::facts::{ContentItem, IrFacts, Shape, Slot}; +use serde_json::Value; + +use super::arrive::Arrived; +use super::attempt::stream_intent; +use super::{decl, handler_of}; +use crate::codec::translate::TranslateCodec; + +/// The dialect whose output cap the previous release read under `max_output_tokens`. +const RESPONSES: &str = "responses"; + +/// What a hook is shown of one arrival. +#[derive(Clone, Debug, Default, PartialEq, Eq)] +pub struct HookView { + /// The wire turn count. + pub turn_count: usize, + /// The screenable text's characters, system included. + pub text_chars: usize, + /// The request declares tools. + pub has_tools: bool, + /// The caller's output cap, saturated. + pub max_tokens: Option<u32>, + /// The caller asked for a stream. + pub stream: bool, + /// The dialect's own system field, flattened; `None` when empty or absent. + pub system: Option<String>, + /// One `(role, text)` entry per wire turn, in request order. + pub turns: Vec<(String, String)>, + /// The end user, as the dialect spells it. + pub end_user: Option<String>, +} + +/// The operation's reader refused the body: the request fails (the previous release's 400). +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct Unreadable; + +/// The client-facing message for a body the operation's reader refuses: content-free, it names the +/// failure, never the field or the value. +pub const UNREADABLE_BODY_MESSAGE: &str = + "request body could not be read as a valid request for this endpoint"; + +/// The previous release's output-cap signal: the dialect's cap key (`max_output_tokens` for the +/// responses dialect, `max_tokens` for every other) off the caller's body, saturated. +#[must_use] +pub fn max_tokens_for(v: &Value, dialect: &str) -> Option<u32> { + let key = if dialect == RESPONSES { + "max_output_tokens" + } else { + "max_tokens" + }; + v.get(key) + .and_then(Value::as_u64) + .map(|n| u32::try_from(n).unwrap_or(u32::MAX)) +} + +/// The facts the operation's reader reads off `arrived`; `Ok(None)` when there is no reader or no +/// body to read. +fn facts(arrived: &Arrived) -> Result<Option<Box<dyn IrFacts + Send + Sync>>, Unreadable> { + let Some(handler) = handler_of(arrived) else { + return Ok(None); + }; + let read = match &arrived.parsed { + Some(v) if v.is_object() => handler.read_facts_value(v), + _ if arrived.body.is_empty() => return Ok(None), + // A body that is not a JSON object (a multipart upload) is read by the byte reader. + _ => handler.read_facts(&arrived.body, &arrived.content_type), + }; + read.map(Some).map_err(|_| Unreadable) +} + +/// One bucket's pieces joined with a newline. +fn join(pieces: Vec<String>) -> Option<String> { + match pieces.len() { + 0 => None, + 1 => pieces.into_iter().next(), + _ => Some(pieces.join("\n")), + } +} + +/// The prompt view of `ir`: the system slot's text, and one `(role, text)` entry per turn. +fn prompt(ir: &dyn IrFacts) -> (Option<String>, Vec<(String, String)>) { + let mut system: Vec<String> = Vec::new(); + let mut turns: Vec<(&'static str, Vec<String>)> = Vec::new(); + for item in ir.content() { + let piece = match &item { + ContentItem::Text { text, .. } => text.to_string(), + other => other.screenable_text().into_owned(), + }; + match item.slot() { + Slot::System => system.push(piece), + slot => { + let i = slot.turn_index().unwrap_or(0); + while turns.len() <= i { + turns.push((item.author(), Vec::new())); + } + turns[i].1.push(piece); + } + } + } + ( + join(system).filter(|s| !s.is_empty()), + turns + .into_iter() + .map(|(role, pieces)| (role.to_string(), join(pieces).unwrap_or_default())) + .collect(), + ) +} + +/// PROJECT one arrival for the hooks. +/// +/// # Errors +/// +/// [`Unreadable`] when the operation's reader refuses the body. +pub fn project(arrived: &Arrived) -> Result<HookView, Unreadable> { + let stream = handler_of(arrived) + .map(|h| stream_intent(h, arrived.parsed.as_ref()).wants_stream) + .unwrap_or(false); + let max_tokens = arrived + .parsed + .as_ref() + .and_then(|v| max_tokens_for(v, arrived.dialect)); + let Some(ir) = facts(arrived)? else { + let shape = Shape::EMPTY; + return Ok(HookView { + turn_count: shape.turn_count, + text_chars: shape.text_chars, + has_tools: shape.has_tools, + max_tokens, + stream, + ..HookView::default() + }); + }; + let shape = ir.shape(); + let (system, turns) = prompt(&*ir); + Ok(HookView { + turn_count: shape.turn_count, + text_chars: shape.text_chars, + has_tools: shape.has_tools, + max_tokens, + stream, + system, + turns, + end_user: ir.end_user().map(str::to_string), + }) +} + +/// A rewrite as it crosses: the hook's `{messages, tools}`. +fn read_rewrite(rewrite: &[u8]) -> Option<RewriteReply> { + let v: Value = serde_json::from_slice(rewrite).ok()?; + let list = |k: &str| { + v.get(k) + .and_then(Value::as_array) + .cloned() + .unwrap_or_default() + }; + Some(RewriteReply { + messages: list("messages"), + tools: list("tools"), + }) +} + +/// APPLY a request-stage hook's rewrite to `arrived` in its own dialect: the dialect frames the +/// rewrite's messages (and tools) into its conversation container. Answers the rewritten body when +/// it applied; `None` leaves the arrival untouched (the previous release: a rewrite that cannot be +/// applied leaves the request unmodified). +pub fn apply_rewrite(arrived: &mut Arrived, rewrite: &[u8]) -> Option<Vec<u8>> { + let reply = read_rewrite(rewrite)?; + if reply.messages.is_empty() { + return None; + } + let dialect = decl(arrived.dialect).and_then(|d| d.dialect())?; + let mut v = arrived.parsed.clone()?; + let obj = v.as_object_mut()?; + if !dialect.apply_rewrite_to_ingress_body(obj, &reply.messages, &reply.tools) { + return None; + } + let bytes = crate::codec::json::to_vec(&v).ok()?; + arrived.body.clone_from(&bytes); + arrived.parsed = Some(v); + Some(bytes) +} diff --git a/crates/busbar-plane-llm/src/exchange/refuse.rs b/crates/busbar-plane-llm/src/exchange/refuse.rs index 8b767ccc01..7f22b223a4 100644 --- a/crates/busbar-plane-llm/src/exchange/refuse.rs +++ b/crates/busbar-plane-llm/src/exchange/refuse.rs @@ -13,8 +13,9 @@ use busbar_contract::abi::plane::{class_of_word, reason_of, RefusalClass}; use busbar_contract::protocol::{ - ProtocolDecl, APPLICATION_JSON, KIND_API_ERROR, KIND_INSUFFICIENT_QUOTA, KIND_INVALID_REQUEST, - KIND_NOT_FOUND, KIND_OVERLOADED, KIND_PERMISSION, KIND_RATE_LIMIT, KIND_REQUEST_TOO_LARGE, + ProtocolDecl, APPLICATION_JSON, KIND_API_ERROR, KIND_AUTHENTICATION, KIND_INSUFFICIENT_QUOTA, + KIND_INVALID_REQUEST, KIND_NOT_FOUND, KIND_OVERLOADED, KIND_PERMISSION, KIND_RATE_LIMIT, + KIND_REQUEST_TOO_LARGE, KIND_TIMEOUT, }; use serde_json::Value; @@ -113,6 +114,30 @@ fn kind_of_status(status: u16) -> &'static str { } } +/// The kind a hook's refusal wears, by the status the hook chose (the previous release's +/// `reject_kind_for_status`): a hook 429 reads as a rate limit, a 408 as a timeout, the 503 of a +/// load-bearing hook that could not answer as overloaded (retryable), and a status with no natural +/// kind as an invalid request. +#[must_use] +pub fn gate_kind(status: u16) -> &'static str { + match status { + 401 => KIND_AUTHENTICATION, + 403 => KIND_PERMISSION, + 404 => KIND_NOT_FOUND, + 408 => KIND_TIMEOUT, + 429 => KIND_RATE_LIMIT, + 503 => KIND_OVERLOADED, + _ => KIND_INVALID_REQUEST, + } +} + +/// RENDER A HOOK'S REFUSAL in `envelope`'s dialect: the hook's own status and words, as the +/// kernel's hook engine clamped and sanitised them. +#[must_use] +pub fn gate_refusal(envelope: &str, status: u16, text: &str) -> Rendered { + render(envelope, status, gate_kind(status), text, 0) +} + /// The sentence the previous release answered a model with when it named no pool and no configured /// model: the dialect's own copy where the path gave one (`shaped`), else the neutral sentence. #[must_use] diff --git a/crates/busbar-plane-llm/src/exchange/reply/mod.rs b/crates/busbar-plane-llm/src/exchange/reply/mod.rs index 53636b5eac..931c23177e 100644 --- a/crates/busbar-plane-llm/src/exchange/reply/mod.rs +++ b/crates/busbar-plane-llm/src/exchange/reply/mod.rs @@ -113,6 +113,10 @@ pub struct Units { pub cache_write: u64, /// Every open class the answer counted beside its tokens, verbatim. pub open: BTreeMap<String, u64>, + /// The token counts are a FLOOR over the bytes relayed so far, not the far end's report. + pub floor: bool, + /// The token counts are stated even at zero: they replace a floor the answer stated before. + pub stated: bool, } impl Units { @@ -130,6 +134,8 @@ impl Units { cache_read: u.cache_read.unwrap_or(0), cache_write: u.cache_creation.unwrap_or(0), open, + floor: false, + stated: false, } } } @@ -149,6 +155,11 @@ pub struct Piece<'a> { pub fault: Option<Fault>, /// The caller's answer is complete. pub done: bool, + /// The wire paths of a delivered TRANSLATE answer that did not cross to the caller's dialect + /// (design F3 "Drops"), already warned: the door reports one `egress.control_unrepresentable` + /// audit row per path (`<path> from <dialect>`), 1.5.5's row. Empty on a relay of the caller's + /// own dialect and on every end that delivers nothing. + pub dropped: Vec<String>, } enum State { @@ -203,17 +214,31 @@ fn json_array(ctx: &ReplyCtx<'_>) -> bool { .is_some_and(|p| p.json_array) } +/// A WHOLE answer, rendered here in full (a translated answer, a judged failure, a refusal): its head +/// states its length, so the caller is sent it under that length, as the previous release sent a +/// buffered answer; a relayed answer states none and is sent as it is relayed. fn answered<'a>(r: Rendered, units: Units, verdict: Verdict, fault: Option<Fault>) -> Piece<'a> { + let mut fields = r.fields; + if !fields + .iter() + .any(|(n, _)| n.eq_ignore_ascii_case("content-length")) + { + fields.push(( + "content-length".to_string(), + r.body.len().to_string().into_bytes(), + )); + } Piece { head: Some(Head { status: r.status, - fields: r.fields, + fields, }), bytes: Cow::Owned(r.body), units, verdict, fault, done: true, + dropped: Vec::new(), } } @@ -346,6 +371,11 @@ impl Reply { if !last { if let Some(u) = relay.streamed_usage() { *units = Units::of(Some(&u), BTreeMap::new()); + } else if let Some(floor) = relay.relayed_floor() { + *units = Units { + floor: true, + ..Units::of(Some(&floor), BTreeMap::new()) + }; } return Piece { verdict: if head.is_some() { @@ -358,6 +388,7 @@ impl Reply { units: units.clone(), fault: None, done: false, + dropped: Vec::new(), }; } let end = relay.end(); @@ -367,7 +398,11 @@ impl Reply { let fault = end.stream_fault.map(Fault::Transient).or(end .generation_failed .then_some(Fault::Transient("upstream-generation-failed"))); - let units = Units::of(end.usage.as_ref(), end.open_units); + // The end's report replaces a floor stated while relaying, even when it is zero. + let units = Units { + stated: units.floor, + ..Units::of(end.usage.as_ref(), end.open_units) + }; self.state = State::Done; Piece { head, @@ -380,11 +415,19 @@ impl Reply { }, fault, done: true, + dropped: end.dropped, } } } } + /// The caller had a usable part of a success answer when it left (see [`relay::Relay::partial`]): + /// its cancel bills the units the answer stated so far. + #[must_use] + pub fn partial(&self) -> bool { + matches!(&self.state, State::Relay { relay, .. } if relay.partial()) + } + /// CUT the answer: the far end's transfer failed (`transport`) or the kernel's ceiling for it /// expired, before its last piece. pub fn cut(&mut self, ctx: &ReplyCtx<'_>, transport: bool) -> Piece<'static> { @@ -414,6 +457,7 @@ impl Reply { verdict: Verdict::Hard, fault: Some(Fault::Transient(cut.reason)), done: true, + dropped: Vec::new(), } } } @@ -492,6 +536,10 @@ fn whole_piece<'a>( wire::token_usage_of(&w.usage).as_ref(), wire::open_units_of(&w.usage), ); + let dropped = match w.end { + whole::WholeEnd::Delivered => std::mem::take(&mut w.dropped), + _ => Vec::new(), + }; let (verdict, fault) = match w.end { whole::WholeEnd::Delivered => (Verdict::Ok, None), whole::WholeEnd::FailedGeneration => ( @@ -504,5 +552,8 @@ fn whole_piece<'a>( whole::WholeEnd::Cut => (Verdict::Hard, Some(Fault::Transient("transport"))), whole::WholeEnd::IngressUnsupported | whole::WholeEnd::OverCap => (Verdict::Hard, None), }; - answered(w.answer, units, verdict, fault) + Piece { + dropped, + ..answered(w.answer, units, verdict, fault) + } } diff --git a/crates/busbar-plane-llm/src/exchange/reply/relay.rs b/crates/busbar-plane-llm/src/exchange/reply/relay.rs index 9cb68a1051..ee4ba1ad5e 100644 --- a/crates/busbar-plane-llm/src/exchange/reply/relay.rs +++ b/crates/busbar-plane-llm/src/exchange/reply/relay.rs @@ -384,6 +384,11 @@ impl Relay { open_billing = read.billing(); None } + // A JSON body that stops short of its end is one the far end CUT (its + // transfer failed before its last byte): it bills only the usage the far + // end reported in the bytes in hand, never the floor (OWNER RULING Q31, + // oracle cell `route.failover|fo|primary-cut-body`). + Err(_) if cut_short(&buf) => wire::reported_usage(self.ingress, &buf), Err(_) => { fault_unreadable(self.ingress, buf.len()); Some(wire::unrecovered_usage(self.ingress, &buf)) @@ -451,6 +456,18 @@ impl Relay { } } + /// THE CALLER HAD A USABLE PART: a byte of the answer was relayed and nothing failed it (no + /// reader's terminal error, no translate abort), so a caller that leaves now leaves a partial + /// answer whose incurred units bill (v1.5.5 `FirstByteBody`'s drop arm). + #[must_use] + pub fn partial(&self) -> bool { + self.first_byte + && !self + .translate + .as_ref() + .is_some_and(|t| t.aborted() || t.terminal_error().is_some()) + } + /// The usage a stream's readers have read so far (cheap: nothing is scanned); `None` for a /// relay that reads its usage only at the end. #[must_use] @@ -458,6 +475,20 @@ impl Relay { self.translate.as_ref().and_then(|t| t.usage()) } + /// THE FLOOR A NON-STREAM SAME-DIALECT ANSWER HAS INCURRED SO FAR, as it relays: it was + /// generated whole before its first byte, so a caller that leaves mid-relay bills the floor + /// over the bytes relayed, never 0 (item 367, Q33 (a), told; #62 applied to buffered relays). + /// `None` for a stream, a translated answer, or one that meters nothing. + #[must_use] + pub fn relayed_floor(&self) -> Option<TokenUsage> { + (self.translate.is_none() + && !self.far_is_stream + && self.meter + && self.handler.taps_usage() + && self.upstream_bytes > 0) + .then(|| wire::estimate_usage_from_truncated_tail(self.upstream_bytes)) + } + /// The usage an answer that ended early had incurred: what a stream's readers accumulated; /// for a same-dialect whole body, the usage the bytes in hand report, else the floor over them /// (a failed far-end transfer bills only what the far end reported, never the floor). @@ -472,6 +503,11 @@ impl Relay { } } +/// A JSON body that ends before its value does: the far end's transfer stopped short. +fn cut_short(buf: &[u8]) -> bool { + serde_json::from_slice::<serde::de::IgnoredAny>(buf).is_err_and(|e| e.is_eof()) +} + fn bytes_or_nothing<'a>(out: Vec<u8>) -> Fed<'a> { if out.is_empty() { Fed::Nothing diff --git a/crates/busbar-plane-llm/src/exchange/shaping.rs b/crates/busbar-plane-llm/src/exchange/shaping.rs index c31d018b58..9ab04f6d47 100644 --- a/crates/busbar-plane-llm/src/exchange/shaping.rs +++ b/crates/busbar-plane-llm/src/exchange/shaping.rs @@ -27,6 +27,14 @@ use serde_json::Value; use crate::codec::DECLS; +/// The provider `auth:` overrides that present under a scheme of their own, not the dialect's, so +/// carry none of its static fields (the previous release's `egress_auth::resolve`). +const AUTH_API_KEY: &str = "api-key"; +/// See [`AUTH_API_KEY`]. +const AUTH_JWT_BEARER: &str = "jwt-bearer"; +/// See [`AUTH_API_KEY`]. +const AUTH_OAUTH_CLIENT_CREDENTIALS: &str = "oauth-client-credentials"; + /// The dialect a provider speaks when it names none. pub const DEFAULT_PROTOCOL: &str = "anthropic"; /// The global output-token default when `limits` names none. @@ -65,6 +73,12 @@ pub struct Lane { pub caps: LaneCaps, /// The provider's error-code map. pub error_map: HashMap<String, String>, + /// The far end's dialect's declared static fields (`ProtocolDecl::static_headers`, ARCHITECT + /// SD-3 (1) "anthropic S2-a"), written on every far request of a lane presented under the + /// dialect's own credential scheme; none under an operator's `auth:` override that is not the + /// dialect's scheme (`api-key`, `jwt-bearer`, `oauth-client-credentials`), whose declaration + /// carries none, as the previous release wrote them. + pub statics: &'static [(&'static str, &'static str)], } impl Lane { @@ -122,6 +136,9 @@ pub struct Shaping { pub lanes: BTreeMap<String, Lane>, /// Every pool's members, in the pool's order, by pool name. pub pools: BTreeMap<String, Vec<Member>>, + /// The session-affinity header a pool's `affinity.header_name` names, by pool name; a pool that + /// names none reads [`DEFAULT_AFFINITY_HEADER`]. + pub affinity_headers: BTreeMap<String, String>, /// The global output-token default. pub default_max_tokens: u32, /// The effort → thinking-budget table. @@ -175,6 +192,8 @@ struct ProviderCfg { #[serde(default)] error_map: HashMap<String, String>, #[serde(default)] + auth: Option<String>, + #[serde(default)] max_output_key: Option<MaxOutputKeyCfg>, #[serde(default)] anthropic_adaptive_thinking: Option<bool>, @@ -322,7 +341,20 @@ pub mod sections { pub const LIMITS: &str = "limits"; } +/// The header a pool's session affinity reads when its `affinity` names none (1.5.5's). +pub const DEFAULT_AFFINITY_HEADER: &str = "x-session-id"; + impl Shaping { + /// The header the session affinity of the unit routed over `pool` reads: the pool's own + /// `affinity.header_name`, else [`DEFAULT_AFFINITY_HEADER`] (v1.5.5 + /// `crates/busbar/src/ingress/mod.rs` `affinity_header_for`). + #[must_use] + pub fn affinity_header(&self, pool: &str) -> &str { + self.affinity_headers + .get(pool) + .map_or(DEFAULT_AFFINITY_HEADER, String::as_str) + } + /// READ THE TABLES from the settings object. /// /// # Errors @@ -337,6 +369,7 @@ impl Shaping { section(settings, sections::MODELS)?.unwrap_or_default(); let limits: Option<LimitsCfg> = section(settings, sections::LIMITS)?; let mut pools: BTreeMap<String, Vec<Member>> = BTreeMap::new(); + let mut affinity_headers: BTreeMap<String, String> = BTreeMap::new(); let mut context: HashMap<String, Option<usize>> = HashMap::new(); if let Some(Value::Object(sec)) = settings.get(sections::POOLS) { for (name, pool) in sec { @@ -366,6 +399,12 @@ impl Shaping { members.push(m); } pools.insert(name.clone(), members); + if let Some(h) = pool + .pointer("/affinity/header_name") + .and_then(Value::as_str) + { + affinity_headers.insert(name.clone(), h.to_string()); + } } } let mut lanes = BTreeMap::new(); @@ -377,13 +416,17 @@ impl Shaping { ) })?; let asked = p.protocol.as_deref().unwrap_or(DEFAULT_PROTOCOL); - let dialect = DECLS + let decl = DECLS .iter() .find(|d| d.name == asked && d.codec.is_some()) - .map(|d| d.name) .ok_or_else(|| { format!("provider '{}' uses unknown protocol '{asked}'", m.provider) })?; + let dialect = decl.name; + let statics = match p.auth.as_deref() { + Some(AUTH_API_KEY | AUTH_JWT_BEARER | AUTH_OAUTH_CLIENT_CREDENTIALS) => &[][..], + _ => decl.static_headers, + }; let wire = m.upstream_model.as_deref().unwrap_or(&name); let lane = Lane { model: name.clone(), @@ -400,6 +443,7 @@ impl Shaping { prompt_caching: m.prompt_caching.unwrap_or(false), caps: lane_caps(p, wire), error_map: p.error_map.clone(), + statics, }; lanes.insert(name, lane); } @@ -422,6 +466,7 @@ impl Shaping { Ok(Shaping { lanes, pools, + affinity_headers, default_max_tokens: limits .as_ref() .and_then(|l| l.default_max_tokens) diff --git a/crates/busbar-plane-llm/src/plane_door.rs b/crates/busbar-plane-llm/src/plane_door.rs index 40dff5821d..3692271252 100644 --- a/crates/busbar-plane-llm/src/plane_door.rs +++ b/crates/busbar-plane-llm/src/plane_door.rs @@ -11,8 +11,9 @@ //! The composition root links [`door`] on its `plane-door` axis under the llm fold's //! development-only switch `llm-on-driver` (`BUSBAR-1.6.0.md` Part 3, section 12, "The switch") and //! binds it through the loader's one load beside the dropped-in plane doors. The default build links -//! no llm door, and with the switch on the root's existing llm row still answers every arrival until -//! the serve path hands this door the arrivals it takes, so one arrival never has two servers. +//! no llm door. With the switch on, the serve path composes this door with the kernel-owned sections +//! it reads and mounts its claims as the data routes, ahead of the root's existing llm row, so every +//! arrival this door claims is served through the plane driver and the row answers none of them. //! //! One unit, as the plane driver serves it: //! @@ -42,11 +43,14 @@ use std::mem::size_of; use std::ptr; use std::sync::{Arc, Mutex, MutexGuard, PoisonError}; +use busbar_contract::abi::hook::{REQUEST_HAS_MAX_TOKENS, REQUEST_HAS_TOOLS, REQUEST_STREAM}; use busbar_contract::abi::host::conn::connector::{ Need, DIRECTION_OUTBOUND, EGRESS_PROVIDER, KEEP_ALL_EXCEPT_DENIED, }; use busbar_contract::abi::host::service::ClockReading; -use busbar_contract::abi::mechanism::call::{AbiStr, Blob, InHead, OutHead, Outcome, BLOB_ABSENT}; +use busbar_contract::abi::mechanism::call::{ + AbiStr, Blob, InHead, OutHead, Outcome, Span, BLOB_ABSENT, +}; use busbar_contract::abi::mechanism::door::{ KindTailHead, Section, Statement, SECTION_CONSUMED, SECTION_DECLARING, }; @@ -55,31 +59,35 @@ use busbar_contract::abi::mechanism::lifecycle::{ }; use busbar_contract::abi::mechanism::ticket::{CompletionHandle, Ticket}; use busbar_contract::abi::plane::{ - ArriveIn, ArriveOut, BillableClass, OnPieceIn, OnPieceOut, OpClass, OutField, PlaneDriveIn, - PlaneDriveOut, PlaneOpenIn, PlaneOpenOut, PlaneRefreshOut, PlaneSnapshot, PlaneTail, ProjectIn, - ProjectOut, RefusalIn, RefusalOut, ServeIn, ServeOut, UnitCount, CANCEL_ABORTED, CLAIM_EXACT, - CLAIM_PROBE, EMIT_DONE, EMIT_TO_FAR_END, FROM_CALLER, FROM_FAR_END, FROM_KERNEL, - INGRESS_REQUEST_RESPONSE, INGRESS_RESPONSE_STREAM, PIECE_HAS_STATUS, PIECE_LAST, - PRINCIPAL_NONE, PRINCIPAL_REQUIRED, SHAPE_PIECEWISE, TAIL_FALLBACK, TAIL_PROBES, + ArriveIn, ArriveOut, BillableClass, DialectAuth, OnPieceIn, OnPieceOut, OpClass, OutField, + PlaneDriveIn, PlaneDriveOut, PlaneOpenIn, PlaneOpenOut, PlaneRefreshOut, PlaneSnapshot, + PlaneTail, ProjectIn, ProjectOut, RefusalIn, RefusalOut, ServeIn, ServeOut, UnitCount, + CANCEL_ABORTED, CANCEL_OK_PARTIAL, CLAIM_EXACT, CLAIM_PROBE, EMIT_DONE, EMIT_TO_FAR_END, + FROM_CALLER, FROM_FAR_END, FROM_KERNEL, INGRESS_REQUEST_RESPONSE, INGRESS_RESPONSE_STREAM, + PIECE_HAS_STATUS, PIECE_LAST, PRINCIPAL_NONE, PRINCIPAL_REQUIRED, REFUSAL_GATE, ROUTE_DIRECT, + ROUTE_POOL, SHAPE_PIECEWISE, SPAN_ABSENT, TAIL_FALLBACK, TAIL_PROBES, UNITS_FLOOR, UNITS_REPORTED, VERDICT_HARD, VERDICT_NONE, VERDICT_OK, VERDICT_RETRY, }; use busbar_contract::abi::plane::{PlaneCancelIn, PlaneCancelOut}; +use busbar_contract::abi::plane::{RecordWrite, AUDIT_DEGRADED, RECORD_AUDIT}; use busbar_contract::abi::sdk::door::{abi_str, statement}; use busbar_contract::abi::sdk::life::Refusal; use busbar_contract::abi::sdk::publish::{ClaimSpec, SnapshotSpec}; use busbar_contract::abi::sdk::{ open_failed, Generations, HostBuf, Instance, Lent, Out, Safe, SafeSlot, Services, }; +use busbar_contract::abi::transport::{FAULT_CALLER, FAULT_HARD, FAULT_NONE, FAULT_TRANSIENT}; use busbar_contract::ids::{MeterClassDecl, OpClassId}; use busbar_contract::plane::PlaneMeta; +use busbar_contract::upstream::Disposition; use serde_json::Value; use crate::dialect::DIALECTS; use crate::exchange::arrive::{self, envelope_for, Arrived, Declined}; use crate::exchange::attempt::{self, stream_intent, FarRequest}; -use crate::exchange::reply::{At, Piece, Reply, ReplyCtx, Units, Verdict}; +use crate::exchange::reply::{At, Fault, Piece, Reply, ReplyCtx, Units, Verdict}; use crate::exchange::shaping::{sections, Shaping}; -use crate::exchange::{handler_of, probe, refuse}; +use crate::exchange::{handler_of, probe, project, refuse}; use crate::LlmPlane; // ── the Statement ──────────────────────────────────────────────────────────────────────────────── @@ -92,6 +100,8 @@ const MAX_INFLIGHT: u32 = 1024; /// The human label. const LABEL: &str = "LLM"; +/// One registration of the `pools:` map. +const POOL_NOUN: &str = "pool"; /// The transport claim a far end is reached over. const TRANSPORT: &str = "http"; @@ -130,9 +140,19 @@ const fn op(i: usize) -> OpClass { OpClass { op: name, name } } +/// THE TOKEN CLASSES' CARD NAMES, in [`METER`]'s order: the classes 1.5.5's `rate_card:` prices +/// (`input`, `output`, `cache_read`, `cache_write`), which the previous release's plane declaration +/// ledgered its four token tiers under, so a door-served count is priced by the card as written. +pub const TOKEN_CLASSES: [&str; 4] = [ + busbar_contract::records::UNIT_INPUT, + busbar_contract::records::UNIT_OUTPUT, + busbar_contract::records::UNIT_CACHE_READ, + busbar_contract::records::UNIT_CACHE_WRITE, +]; + const fn billable(i: usize) -> BillableClass { BillableClass { - class: abi_str(METER[i].key.as_str()), + class: abi_str(TOKEN_CLASSES[i]), family: abi_str(METER[i].family), } } @@ -159,6 +179,14 @@ const fn open_class(k: usize) -> BillableClass { } } +/// THE FEE UNIT the plane counts: one per billable request (`busbar_contract::plane::PER_REQUEST`), +/// reported as a count of 1 exactly where 1.5.5 billed its flat request fee (a reply whose caller +/// status is a success) and never where 1.5.5 refunded it (owner #77, money-B1: the plane's report is +/// the ONE fee decider). It is the last of the tail's billable classes, and never usage. +pub const FEE_CLASS: &str = busbar_contract::plane::PER_REQUEST; +/// [`FEE_CLASS`]'s index in the tail's billable classes. +pub const FEE_CLASS_INDEX: u32 = (METER.len() + OPEN_CLASSES.len()) as u32; + const BILLABLE_CLASSES: &[BillableClass] = &[ billable(0), billable(1), @@ -179,26 +207,71 @@ const BILLABLE_CLASSES: &[BillableClass] = &[ open_class(12), open_class(13), open_class(14), + BillableClass { + class: abi_str(FEE_CLASS), + family: abi_str("request"), + }, ]; +/// The tail's fee units: [`FEE_CLASS`]. +const FEE_UNITS: &[AbiStr] = &[abi_str(FEE_CLASS)]; const _: () = assert!( DIALECTS.len() == DIALECT_NAMES.len() && OPS.len() == OP_CLASSES.len() - && METER.len() + OPEN_CLASSES.len() == BILLABLE_CLASSES.len(), + && METER.len() + OPEN_CLASSES.len() + 1 == BILLABLE_CLASSES.len(), "the tail states every dialect, op class and token class the plane declares" ); -/// The egress-auth schemes the dialects decorate a far-end request with ([`DIALECTS`]' -/// `egress_scheme`), each named once. -pub const EGRESS_SCHEMES: &[&str] = &[DIALECTS[0].egress_scheme, DIALECTS[3].egress_scheme]; +/// THE OUTBOUND AUTH STYLES THE PLANE'S MEMBERS ARE BOUND UNDER (the design's auth points): each +/// dialect's default ([`DIALECTS`]' `egress_style`), then the provider `auth:` overrides the +/// configuration grammar accepts (`api-key` is also a dialect default), each named once. One +/// outbound need per style ([`NEEDS`]): a member dials the need its resolved style names. +pub const EGRESS_STYLES: &[&str] = &[ + DIALECTS[1].egress_style, + DIALECTS[0].egress_style, + DIALECTS[2].egress_style, + DIALECTS[3].egress_style, + "jwt-bearer", + "oauth-client-credentials", +]; const _: () = assert!( - const_eq(DIALECTS[0].egress_scheme, DIALECTS[1].egress_scheme) - && const_eq(DIALECTS[0].egress_scheme, DIALECTS[2].egress_scheme) - && const_eq(DIALECTS[0].egress_scheme, DIALECTS[4].egress_scheme) - && const_eq(DIALECTS[0].egress_scheme, DIALECTS[5].egress_scheme) - && !const_eq(DIALECTS[0].egress_scheme, DIALECTS[3].egress_scheme), - "EGRESS_SCHEMES names every dialect's scheme, each once" + const_eq(DIALECTS[1].egress_style, DIALECTS[4].egress_style) + && const_eq(DIALECTS[1].egress_style, DIALECTS[5].egress_style) + && !const_eq(DIALECTS[0].egress_style, DIALECTS[1].egress_style) + && !const_eq(DIALECTS[2].egress_style, DIALECTS[1].egress_style) + && !const_eq(DIALECTS[3].egress_style, DIALECTS[1].egress_style), + "EGRESS_STYLES names every dialect's default style, each once" ); +const fn dialect_auth(i: usize) -> DialectAuth { + let params = DIALECTS[i].egress_params.as_bytes(); + DialectAuth { + dialect: i as u32, + _reserved: 0, + style: abi_str(DIALECTS[i].egress_style), + // The default style's parameters (ARCHITECT RULING 2026-10-03, Q-L6-AUTHPARAMS). + params: if params.is_empty() { + Blob::ABSENT + } else { + Blob { + ptr: params.as_ptr(), + len: params.len(), + fmt: busbar_contract::abi::mechanism::call::BLOB_JSON, + flags: 0, + } + }, + } +} + +/// THE TAIL'S `dialect_auth`: each dialect's default outbound style (ARCHITECT Q-L1-AUTH (A)). +const DIALECT_AUTH: &[DialectAuth] = &[ + dialect_auth(0), + dialect_auth(1), + dialect_auth(2), + dialect_auth(3), + dialect_auth(4), + dialect_auth(5), +]; + const fn const_eq(a: &str, b: &str) -> bool { let (a, b) = (a.as_bytes(), b.as_bytes()); if a.len() != b.len() { @@ -254,9 +327,20 @@ const fn need(auth: &'static str) -> Need { } /// THE PLANE'S NEEDS: outbound to a configured provider over the claim's transport, one per -/// egress-auth scheme a dialect decorates with ([`EGRESS_SCHEMES`]), as the decisions plane -/// declares its one. -pub const NEEDS: &[Need] = &[need(EGRESS_SCHEMES[0]), need(EGRESS_SCHEMES[1])]; +/// outbound style a member may be bound under ([`EGRESS_STYLES`]), each naming its style, as the +/// decisions plane's one need names its. +pub const NEEDS: &[Need] = &[ + need(EGRESS_STYLES[0]), + need(EGRESS_STYLES[1]), + need(EGRESS_STYLES[2]), + need(EGRESS_STYLES[3]), + need(EGRESS_STYLES[4]), + need(EGRESS_STYLES[5]), +]; + +/// THE RESOURCE A GRANT NAMES on this plane: a pool (a key's `allowed_pools`, the previous release's +/// `scope_kinds: ["pool"]`), so the kernel judges a route as named against the caller's grant. +const SCOPE_KINDS: &[AbiStr] = &[abi_str("pool")]; /// THE STATEMENT TAIL: the plane's static facts. pub const TAIL: &PlaneTail = &PlaneTail { @@ -271,26 +355,28 @@ pub const TAIL: &PlaneTail = &PlaneTail { _reserved: 0, scope: abi_str(sections::POOLS), label: abi_str(LABEL), - subject_noun: NONE, - admin_noun: NONE, + // What one registration of the `pools:` map is called, in a refusal naming it ("a pool may not + // be named `hooks`") and in the admin surface, as the previous release's declaration named it. + subject_noun: abi_str(POOL_NOUN), + admin_noun: abi_str(POOL_NOUN), audit_kind: NONE, signing_domain: NONE, signing_kid_prefix: NONE, cli_help: NONE, dialects: DIALECT_NAMES.as_ptr(), dialects_len: DIALECT_NAMES.len(), - dialect_auth: ptr::null(), - dialect_auth_len: 0, - scope_kinds: ptr::null(), - scope_kinds_len: 0, + dialect_auth: DIALECT_AUTH.as_ptr(), + dialect_auth_len: DIALECT_AUTH.len(), + scope_kinds: SCOPE_KINDS.as_ptr(), + scope_kinds_len: SCOPE_KINDS.len(), op_classes: OP_CLASSES.as_ptr(), op_classes_len: OP_CLASSES.len(), billable_classes: BILLABLE_CLASSES.as_ptr(), billable_classes_len: BILLABLE_CLASSES.len(), route_cost: ptr::null(), route_cost_len: 0, - fee_units: ptr::null(), - fee_units_len: 0, + fee_units: FEE_UNITS.as_ptr(), + fee_units_len: FEE_UNITS.len(), record_kinds: ptr::null(), record_kinds_len: 0, egress_targets: ptr::null(), @@ -354,10 +440,18 @@ struct Answer { units: Vec<UnitCount>, /// `VERDICT_*`. verdict: u32, + /// `FAULT_*`: the breaker's reading of this answer, written once, on its first window. + fault: u8, /// The caller's reply is complete. done: bool, + /// The unit's audit rows, `(action, resource)`, each written as a degraded `RECORD_AUDIT`: + /// what a TRANSLATE attempt could not carry (1.5.5's `egress.control_unrepresentable` rows). + audits: Vec<(Vec<u8>, Vec<u8>)>, } +/// The action of a dropped control's audit row (1.5.5's). +const DROPPED_CONTROL: &[u8] = b"egress.control_unrepresentable"; + impl Answer { /// The walk ends here: nothing for the far end, nothing more for the caller. fn hard() -> Self { @@ -401,6 +495,13 @@ struct UnitState { started: Option<u64>, /// The answer the driver's re-call is owed. pending: Option<Pending>, + /// `project` found the body unreadable: the unit's refusal reads the previous release's + /// unreadable-body sentence. + unreadable: bool, + /// The far end's last cumulative counts, as the unit last reported them. + reported: Vec<UnitCount>, + /// The caller was answered under a success status: the request's fee unit was incurred. + fee: bool, } impl UnitState { @@ -416,6 +517,9 @@ impl UnitState { reply: None, started: None, pending: None, + unreadable: false, + reported: Vec::new(), + fee: false, } } @@ -525,6 +629,21 @@ fn dialect_index(name: &str) -> u32 { .unwrap_or(0) } +/// THE ROUTE AN ARRIVAL NAMES (ARCHITECT Q-SW6 / Q-FL3, 2026-10-02): the model it asked for, as the +/// entry inside the plane's own sections, and what that entry is. The previous release resolved the +/// model as a pool first, then as a configured model (its by-model lane); the plane says which by its +/// own tables, and the kernel resolves the entry. A model that is neither is named as a direct entry +/// the kernel does not hold, so the kernel refuses it (`no_destination`, rendered as 1.5.5's 404). +#[must_use] +pub fn route_of<'a>(shaping: &Shaping, model: &'a str) -> (u8, &'a str) { + let class = if shaping.pools.contains_key(model) { + ROUTE_POOL + } else { + ROUTE_DIRECT + }; + (class, model) +} + fn op_class_index(arrived: &Arrived) -> u32 { OPS.iter() .position(|op| op.as_str() == arrived.operation.name()) @@ -551,10 +670,15 @@ pub fn counts(units: &Units) -> Vec<UnitCount> { units.cache_write, ]; let mut out = Vec::new(); - if tokens.iter().any(|&n| n != 0) { + let source = if units.floor { + UNITS_FLOOR + } else { + UNITS_REPORTED + }; + if units.stated || tokens.iter().any(|&n| n != 0) { out.extend(tokens.iter().zip(0u32..).map(|(&amount, class)| UnitCount { class, - source: UNITS_REPORTED, + source, amount, })); } @@ -578,6 +702,28 @@ pub fn counts(units: &Units) -> Vec<UnitCount> { out } +/// THE BREAKER'S READING of a judged answer (ARCHITECT BREAKER OUTCOME: one neutral vocabulary, +/// stated apart from the walk's verdict), as 1.5.5's classifier recorded it per disposition (v1.5.5 +/// `crates/busbar-llm/src/engine/attempt/classify.rs`): the caller's own bad input records nothing +/// against the destination; a transient failure (an `error_map` entry naming `rate_limit` among +/// them) counts toward the pool cell's trip; a refused credential or account trips every cell; a +/// request too large for the window fails over with nothing recorded. A 2xx that failed after its +/// head (a stream's terminal error, a cut body, an untranslatable answer, a failed generation) is +/// the compensating transient 1.5.5 recorded. +#[must_use] +pub fn breaker_fault(f: Option<&Fault>) -> u8 { + match f { + None => FAULT_NONE, + Some(Fault::Transient(_)) => FAULT_TRANSIENT, + Some(Fault::Judged { disposition, .. }) => match disposition { + Disposition::ClientFault => FAULT_CALLER, + Disposition::TransientUpstream => FAULT_TRANSIENT, + Disposition::HardDown => FAULT_HARD, + Disposition::ContextLength => FAULT_NONE, + }, + } +} + fn verdict(v: Verdict) -> u32 { match v { Verdict::None => VERDICT_NONE, @@ -587,8 +733,58 @@ fn verdict(v: Verdict) -> u32 { } } +/// THE REQUEST'S FEE UNIT on `answer` (owner #77, money-B1: the plane's report is the one fee +/// decider): incurred when the caller's reply opens under a success status, which is where 1.5.5 +/// kept its flat request fee (its finish refunded the fee for a non-2xx caller status, and only +/// then). From that answer on, every answer that carries counts carries the fee unit's 1 beside the +/// far end's last cumulative counts; an answer that carries none leaves the last report standing. +/// A probe is no billable request. +fn fee_unit(unit: &mut UnitState, answer: &mut Answer) { + if !answer.units.is_empty() { + unit.reported.clone_from(&answer.units); + } + let incurred = + !unit.fee && !unit.probe && !answer.to_far_end && (200..=299).contains(&answer.status); + unit.fee |= incurred; + if unit.fee && (incurred || !answer.units.is_empty()) { + answer.units.clone_from(&unit.reported); + answer.units.push(UnitCount { + class: FEE_CLASS_INDEX, + source: UNITS_REPORTED, + amount: 1, + }); + } +} + +/// THE UNIT'S STICKY-ROUTING KEY, as 1.5.5 derived it (v1.5.5 `crates/busbar/src/ingress/mod.rs` +/// `affinity_header_for`, `crates/busbar/src/proxy/engine/mod.rs` the affinity hash): the value of +/// the pool's affinity header (`header`, matched case-blind) where the caller sent one readable as +/// text, else the operation's body key (chat's non-empty `system`); `None` = no affinity. The kernel +/// hashes it and picks the member; the plane never does. +#[must_use] +pub fn affinity_key( + header: &str, + caller: &[(&[u8], &[u8])], + handler: Option<&dyn busbar_contract::codec::OperationHandler>, + body: Option<&Value>, +) -> Option<String> { + let visible = |v: &[u8]| v.iter().all(|b| *b == b'\t' || (0x20..0x7f).contains(b)); + caller + .iter() + .find(|(n, _)| n.eq_ignore_ascii_case(header.as_bytes())) + .map(|(_, v)| *v) + .filter(|v| visible(v)) + .and_then(|v| std::str::from_utf8(v).ok()) + .map(str::to_string) + .or_else(|| { + let (h, b) = (handler?, body?); + h.body_affinity_key(b).map(str::to_string) + }) +} + /// A reply piece as the caller's answer. fn to_caller(piece: Piece<'_>) -> Answer { + let fault = breaker_fault(piece.fault.as_ref()); let (status, fields) = piece .head .map_or((0, Vec::new()), |h| (u32::from(h.status), owned(&h.fields))); @@ -598,6 +794,7 @@ fn to_caller(piece: Piece<'_>) -> Answer { bytes: piece.bytes.into_owned(), units: counts(&piece.units), verdict: verdict(piece.verdict), + fault, done: piece.done, ..Answer::default() } @@ -657,6 +854,9 @@ fn attempt(unit: &mut UnitState, piece: &PieceIn<'_>) -> Answer { }; match built { Ok(request) => { + // A control the far end's dialect cannot carry, one degraded row each (`<control> on + // <dialect>`, the row 1.5.5 wrote as it translated the request). + let egress = unit.shaping.lane(&unit.member).map_or("", |l| l.dialect); let answer = Answer { to_far_end: true, request: Some(( @@ -664,6 +864,16 @@ fn attempt(unit: &mut UnitState, piece: &PieceIn<'_>) -> Answer { request.target.as_bytes().to_vec(), )), fields: owned(&request.fields), + audits: request + .dropped_controls + .iter() + .map(|c| { + ( + DROPPED_CONTROL.to_vec(), + format!("{c} on {egress}").into_bytes(), + ) + }) + .collect(), ..Answer::default() }; unit.request = Some(request); @@ -713,8 +923,21 @@ fn far_end(unit: &mut UnitState, piece: &PieceIn<'_>) -> Answer { let Some(reply) = unit.reply.as_mut() else { return Answer::hard(); }; - let fed = reply.feed(&ctx, piece.bytes, piece.last, at(piece.clock, unit.started)); - to_caller(fed) + let mut fed = reply.feed(&ctx, piece.bytes, piece.last, at(piece.clock, unit.started)); + // An answer member the caller's dialect has no form for, one degraded row each (`<path> from + // <dialect>`, the row 1.5.5 wrote as it delivered the translated answer). + let dropped = std::mem::take(&mut fed.dropped); + let mut answer = to_caller(fed); + answer.audits = dropped + .iter() + .map(|p| { + ( + DROPPED_CONTROL.to_vec(), + format!("{p} from {}", lane.dialect).into_bytes(), + ) + }) + .collect(); + answer } /// The plane's answer to one piece. @@ -744,12 +967,16 @@ fn settle( out: &mut Out<'_, OnPieceOut>, fields: &HostBuf<'_, OutField>, units: &HostBuf<'_, UnitCount>, + records: &HostBuf<'_, RecordWrite>, arena: &HostBuf<'_, u8>, ) -> bool { - let short = !(fields.fits() && units.fits() && arena.fits()); + let short = !(fields.fits() && units.fits() && records.fits() && arena.fits()); let (fw, fnd) = fields.settle(short); let (uw, und) = units.settle(short); + let (rw, rnd) = records.settle(short); let (aw, and) = arena.settle(short); + out.set(|o| &o.records_written, rw as u32); + out.set(|o| &o.records_needed, rnd as u32); out.set(|o| &o.fields_written, fw as u32); out.set(|o| &o.fields_needed, fnd as u32); out.set(|o| &o.units_written, uw as u32); @@ -771,8 +998,12 @@ fn deliver( return (Outcome::Fault, false); }; if !p.opened { - let (mut fields, mut units, mut arena) = - (input.fields_buf(), input.units_buf(), input.arena_buf()); + let (mut fields, mut units, mut records, mut arena) = ( + input.fields_buf(), + input.units_buf(), + input.records_buf(), + input.arena_buf(), + ); let request = p .answer .request @@ -785,7 +1016,15 @@ fn deliver( }); } units.extend(&p.answer.units); - if settle(out, &fields, &units, &arena) { + for (action, resource) in &p.answer.audits { + records.push(RecordWrite { + kind: AUDIT_DEGRADED, + op: RECORD_AUDIT, + key: arena.span(action), + value: arena.span(resource), + }); + } + if settle(out, &fields, &units, &records, &arena) { return (Outcome::Failed, false); } if let Some((verb, target)) = request { @@ -809,6 +1048,8 @@ fn deliver( out.set(|o| &o.emitted, n as u64); out.set(|o| &o.more, u32::from(more)); out.set(|o| &o.verdict, p.answer.verdict); + // Once per answer: a reading recorded on every window of a long answer would count it again. + out.set(|o| &o.fault, std::mem::take(&mut p.answer.fault)); out.set(|o| &o.flags, far | if done { EMIT_DONE } else { 0 }); if !more { unit.pending = None; @@ -910,12 +1151,21 @@ slot!( slot!( /// `cancel`: the unit on the cancelled ticket ends; nothing it owed is delivered. Cancel, PlaneCancelIn, PlaneCancelOut, |instance, input, mut out| { + let mut partial = false; if let Some(door) = instance.get() { if let Some(unit) = guard(&door.tickets).remove(&input.cancel.ticket) { - guard(&door.units).remove(&unit); + partial = guard(&door.units) + .remove(&unit) + .is_some_and(|u| u.reply.as_ref().is_some_and(Reply::partial)); } } - out.set(|o| &o.cancel.disposition, CANCEL_ABORTED); + // A caller that left a usable part of the answer leaves a PARTIAL unit: the units the + // answer stated so far bill (1.5.5's drop arm; item 367 for a buffered relay). Any other + // cancel bills nothing. + out.set( + |o| &o.cancel.disposition, + if partial { CANCEL_OK_PARTIAL } else { CANCEL_ABORTED }, + ); Outcome::Ready } ); @@ -970,6 +1220,22 @@ slot!( out.set(|o| &o.op_class, op_class_index(&arrived)); out.set(|o| &o.dialect, dialect_index(arrived.dialect)); out.set(|o| &o.principal_need, PRINCIPAL_REQUIRED); + let (class, entry) = route_of(&unit.shaping, &arrived.model); + out.route(class, entry); + let handler = handler_of(&arrived); + if handler.is_some_and(|h| stream_intent(h, arrived.parsed.as_ref()).wants_stream) { + out.stream(); + } + if class == ROUTE_POOL { + if let Some(key) = affinity_key( + unit.shaping.affinity_header(entry), + &unit.caller_fields(), + handler, + arrived.parsed.as_ref(), + ) { + out.affinity(&key); + } + } unit.arrived = Some(arrived); guard(&door.units).insert(given.unit, unit); Outcome::Ready @@ -1014,7 +1280,8 @@ slot!( passthrough: given.passthrough != 0, clock: if given.from == FROM_CALLER { None } else { door.clock() }, }; - let answer = answer(&mut unit, given.from, &piece); + let mut answer = answer(&mut unit, given.from, &piece); + fee_unit(&mut unit, &mut answer); unit.pending = Some(Pending { answer, sent: 0, @@ -1058,7 +1325,9 @@ slot!( // A model that resolved to no destination reads the previous release's not-found // sentence, which names the model the caller asked for; every other refusal reads // the kernel's own text. + let unreadable = held.as_ref().is_some_and(|u| u.unreadable); let text = match arrived { + _ if unreadable => project::UNREADABLE_BODY_MESSAGE.to_string(), Some(a) if given.reason == crate::refusal::reason::NO_DESTINATION => { refuse::model_not_found( &a.model, @@ -1069,13 +1338,12 @@ slot!( } _ => String::from_utf8_lossy(input.field(|i| &i.text).bytes()).into_owned(), }; - refuse::kernel_refusal( - envelope, - given.reason, - u16::try_from(given.status).unwrap_or(500), - &text, - given.retry_after_s, - ) + let status = u16::try_from(given.status).unwrap_or(500); + if given.cause == REFUSAL_GATE { + refuse::gate_refusal(envelope, status, &text) + } else { + refuse::kernel_refusal(envelope, given.reason, status, &text, given.retry_after_s) + } } }; let (mut reply, mut fields, mut arena) = @@ -1125,8 +1393,99 @@ slot!( ); slot!( - /// `project`: not bound yet; the driver's hook stage answers it at the llm flip. - Project, ProjectIn, ProjectOut, |_, _, _| { Outcome::Refused } + /// `project`: the hook view of the unit's request ([`crate::exchange::project`]), its strings in + /// the host's arena and its turns in the host's turn buffer; with a request-stage hook's + /// rewrite, the rewrite applied to the unit's request first (kept as the unit's request, so + /// every attempt is written from it), the rewritten body answered and THAT body projected. A + /// body the operation's reader refuses is REFUSED, and the unit's refusal then reads the + /// previous release's unreadable-body sentence. + Project, ProjectIn, ProjectOut, |instance, input, mut out| { + let Some(door) = instance.get() else { + return Outcome::Failed; + }; + let given = input.get(); + let mut units = guard(&door.units); + // A unit that never arrived (or arrived declined, or as a probe) has nothing to project. + let Some(arrived) = units.get_mut(&given.unit).and_then(|u| u.arrived.as_mut()) else { + return Outcome::Refused; + }; + let rewrite = input.field(|i| &i.rewrite).bytes(); + let rewritten = if rewrite.is_empty() { + None + } else { + project::apply_rewrite(arrived, rewrite) + }; + let view = match project::project(arrived) { + Ok(view) => view, + Err(project::Unreadable) => { + if let Some(unit) = units.get_mut(&given.unit) { + unit.unreadable = true; + } + return Outcome::Refused; + } + }; + let dialect = arrived.dialect; + let pool = arrived.model.clone(); + drop(units); + let (signals, mut arena, mut turns) = + (input.signals_buf(), input.arena_buf(), input.messages_buf()); + let pool = arena.span(pool.as_bytes()); + let dialect = arena.span(dialect.as_bytes()); + let system = view.system.as_deref().map(|s| arena.span(s.as_bytes())); + let end_user = view.end_user.as_deref().map(|s| arena.span(s.as_bytes())); + for (role, text) in &view.turns { + let role = arena.span(role.as_bytes()); + let text = arena.span(text.as_bytes()); + turns.push_turn(&arena, role, text); + } + let body = rewritten.as_deref().map(|b| arena.span(b)); + let short = !(signals.fits() && arena.fits() && turns.fits()); + let (sw, snd) = signals.settle(short); + let (aw, and) = arena.settle(short); + let (tw, tnd) = turns.settle(short); + out.set(|o| &o.signals_needed, snd as u32); + out.set(|o| &o.arena_written, aw as u64); + out.set(|o| &o.arena_needed, and as u64); + out.set(|o| &o.messages_needed, tnd as u32); + out.set(|o| &o.body, Span { offset: SPAN_ABSENT, len: 0 }); + if short { + // The driver re-calls once with the buffers this named, the same rewrite with it: + // applying it again to the request it already rewrote writes the same body. + return Outcome::Failed; + } + let mut flags = 0; + if view.max_tokens.is_some() { + flags |= REQUEST_HAS_MAX_TOKENS; + } + if view.has_tools { + flags |= REQUEST_HAS_TOOLS; + } + if view.stream { + flags |= REQUEST_STREAM; + } + out.host_str(|o| &o.view.pool, &arena, pool); + out.host_str(|o| &o.view.ingress_dialect, &arena, dialect); + out.set(|o| &o.view.message_count, view.turn_count as u64); + out.set(|o| &o.view.total_chars, view.text_chars as u64); + out.set(|o| &o.view.max_tokens, view.max_tokens.unwrap_or(0)); + out.set(|o| &o.view.flags, flags); + out.host_rows(|o| &o.view.signals, &signals); + out.set(|o| &o.view.signals_len, sw); + if let Some(system) = system { + out.host_str(|o| &o.prompt.system, &arena, system); + } + out.set(|o| &o.prompt.message_count, tw as u64); + out.host_rows(|o| &o.prompt.messages, &turns); + out.set(|o| &o.prompt.messages_len, tw); + if let Some(end_user) = end_user { + out.host_str(|o| &o.end_user, &arena, end_user); + } + out.set( + |o| &o.rewritten, + body.unwrap_or(Span { offset: SPAN_ABSENT, len: 0 }), + ); + Outcome::Ready + } ); busbar_contract::plugin_door! { diff --git a/crates/busbar-plane-llm/tests/conformance.rs b/crates/busbar-plane-llm/tests/conformance.rs index db8fbedff8..013f41521b 100644 --- a/crates/busbar-plane-llm/tests/conformance.rs +++ b/crates/busbar-plane-llm/tests/conformance.rs @@ -404,8 +404,9 @@ fn assert_the_planes_answers(t: &[String]) { "a same-dialect answer is the far end's bytes: {all}" ); assert!( - far.contains(r#"units=["0:7:true", "1:3:true", "2:0:true", "3:0:true"]"#), - "the far end's counts, in the tail's class order: {all}" + far.contains(r#"units=["0:7:true", "1:3:true", "2:0:true", "3:0:true", "19:1:true"]"#), + "the far end's counts, in the tail's class order, then the per-request fee unit a success \ + reply incurs, last after the 15 open classes (owner #77, money-B1; LEDGER-100): {all}" ); let narrow = line(t, "far_end narrow "); assert!( diff --git a/crates/busbar-plane-llm/tests/exchange_attempt.rs b/crates/busbar-plane-llm/tests/exchange_attempt.rs index 04ebdb9cca..b7a81b544e 100644 --- a/crates/busbar-plane-llm/tests/exchange_attempt.rs +++ b/crates/busbar-plane-llm/tests/exchange_attempt.rs @@ -16,6 +16,8 @@ fn shaping() -> Shaping { "providers": { "oai": { "protocol": "openai", "base_url": "https://api.example" }, "ant": { "protocol": "anthropic", "base_url": "https://anthropic.example" }, + "ant-key": { "protocol": "anthropic", "base_url": "https://az.example", "auth": "api-key" }, + "ant-bearer": { "protocol": "anthropic", "base_url": "https://b.example", "auth": "bearer" }, "goo": { "protocol": "gemini", "base_url": "https://gemini.example" }, "fixed": { "protocol": "openai", "base_url": "https://x.example", "path": "/custom/chat" } }, @@ -23,6 +25,8 @@ fn shaping() -> Shaping { "gpt": { "provider": "oai" }, "gpt-alias": { "provider": "oai", "upstream_model": "gpt-4o" }, "claude": { "provider": "ant", "default_max_tokens": 321 }, + "claude-key": { "provider": "ant-key", "default_max_tokens": 321 }, + "claude-bearer": { "provider": "ant-bearer", "default_max_tokens": 321 }, "gem": { "provider": "goo", "upstream_model": "gemini-pro" }, "fixed": { "provider": "fixed" } }, @@ -132,20 +136,21 @@ fn a_same_dialect_caller_s_fields_all_go_out_but_the_governed_ones() { assert_eq!( names, [ + "anthropic-version", + "content-type", "user-agent", "accept", - "content-type", - "anthropic-version", "anthropic-beta", "anthropic-beta", "x-client-trace" ], - "busbar's native defaults the caller did not send, then every caller field but the credential" + "busbar's own fields at their positions (the caller's values where it sent one), then every \ + other caller field but the credential, as 1.5.5's header map laid them" ); assert_eq!( field(&r, "user-agent"), [b"Anthropic/Python 0.39.0".as_slice()], - "a caller that sent no user-agent: the native client's (1.5.5's bytes)" + "a caller that sent no user-agent: the native client's (1.5.5's bytes, Q10)" ); assert_eq!(field(&r, "accept"), [b"application/json".as_slice()]); assert_eq!( @@ -224,6 +229,33 @@ fn a_same_dialect_unknown_body_member_goes_out() { assert_eq!(v["x_vendor_flag"], true, "the unknown member, unchanged"); } +/// A body-model caller that smuggles busbar's own array-stream router key into its body is not a +/// JSON-array caller (only a path-model arrival's URL makes one), and the key never reaches the far +/// end: 1.5.5 stripped every dialect's router key before every upstream call (v1.5.5 +/// `crates/busbar/src/proxy/wire.rs` `strip_router_shim_keys`; the retired legacy cover +/// `test_gemini_json_array_shim_ignored_for_body_model_ingress`). +#[test] +fn a_body_model_callers_smuggled_router_key_never_reaches_the_far_end() { + let h = head(&[("content-type", "application/json")]); + let a = arrived( + "/v1/chat/completions", + &h, + r#"{"model":"gpt","stream":true,"__busbar_gemini_json_array":true,"messages":[]}"#, + ); + assert!( + !a.path_model.as_ref().is_some_and(|p| p.json_array), + "a body key never makes a JSON-array caller" + ); + let r = build(&a, &h, &shaping(), "p", "gpt").expect("built"); + assert!(!r.pristine, "the router key is a governed member"); + let v: Value = busbar_plane_llm::codec::json::parse(&r.body).expect("json"); + assert!(v.get("__busbar_gemini_json_array").is_none(), "{v}"); + assert_eq!( + v["stream"], true, + "a body-model far end keeps the caller's stream" + ); +} + #[test] fn a_translated_route_drops_a_member_the_far_dialect_cannot_carry() { let h = head(&[("content-type", "application/json")]); @@ -436,3 +468,121 @@ fn the_providers_tenant_goes_out_and_the_callers_does_not() { assert_eq!(field(&r, "openai-organization"), [b"org-cfg".as_slice()]); assert_eq!(field(&r, "openai-project"), [b"proj-cfg".as_slice()]); } + +/// THE DIALECT'S STATIC FIELDS (ARCHITECT SD-3 (1), "anthropic S2-a": anthropic declares +/// `anthropic-version`, written verbatim; byte-identical upstream requests): a caller that sends no +/// field of its own still reaches an anthropic far end with the pinned version and the native +/// client's user-agent, same-dialect or translated; under an operator's `auth: api-key` override, +/// whose scheme declares no static field, none goes out; `auth: bearer` is the dialect's own scheme +/// and keeps it. A same-dialect caller's own value replaces it. +#[test] +fn no_client_header_leaves_egress_unchanged() { + let none = head(&[]); + let body = r#"{"model":"claude","max_tokens":5,"messages":[{"role":"user","content":"hi"}]}"#; + let a = arrived("/v1/messages", &none, body); + let r = build(&a, &none, &shaping(), "p", "claude").expect("built"); + assert_eq!(field(&r, "anthropic-version"), [b"2023-06-01".as_slice()]); + assert_eq!( + field(&r, "user-agent"), + [b"Anthropic/Python 0.39.0".as_slice()] + ); + let names: Vec<&str> = r.fields.iter().map(|(n, _)| n.as_str()).collect(); + assert_eq!( + names, + ["anthropic-version", "content-type", "user-agent", "accept"] + ); + + let openai = r#"{"model":"claude","messages":[{"role":"user","content":"hi"}]}"#; + let a = arrived("/v1/chat/completions", &none, openai); + let r = build(&a, &none, &shaping(), "p", "claude").expect("built"); + assert_eq!( + field(&r, "anthropic-version"), + [b"2023-06-01".as_slice()], + "a translated route into anthropic carries it too" + ); + + let a = arrived("/v1/messages", &none, &body.replace("claude", "claude-key")); + let r = build(&a, &none, &shaping(), "p", "claude-key").expect("built"); + assert!( + field(&r, "anthropic-version").is_empty(), + "an api-key override declares no static field" + ); + let a = arrived( + "/v1/messages", + &none, + &body.replace("claude", "claude-bearer"), + ); + let r = build(&a, &none, &shaping(), "p", "claude-bearer").expect("built"); + assert_eq!(field(&r, "anthropic-version"), [b"2023-06-01".as_slice()]); + + let own = head(&[("anthropic-version", "2024-01-01")]); + let a = arrived("/v1/messages", &own, body); + let r = build(&a, &own, &shaping(), "p", "claude").expect("built"); + assert_eq!( + field(&r, "anthropic-version"), + [b"2024-01-01".as_slice()], + "the same-dialect caller's own value wins" + ); + + let a = arrived( + "/v1/chat/completions", + &none, + &openai.replace("claude", "gpt"), + ); + let r = build(&a, &none, &shaping(), "p", "gpt").expect("built"); + assert!( + field(&r, "anthropic-version").is_empty(), + "openai declares none" + ); +} + +/// THE PREVIOUS RELEASE'S HEADER-MAP ORDER (oracle `egress.auth|cred-identity|*`, `route.529|*`): a +/// same-dialect caller's own fields lie over busbar's as a header map lays them, one position per +/// name: a name busbar writes (its static field, `content-type`, `user-agent`, `accept`) keeps +/// busbar's position and takes the caller's value, a new name is appended in the caller's order, +/// and a repeated name keeps every value at its position. +#[test] +fn a_same_dialect_callers_fields_keep_busbars_positions() { + let h = head(&[ + ("user-agent", "Anthropic/Python 0.39.0"), + ("accept", "application/json"), + ("anthropic-version", "2023-06-01"), + ("content-type", "application/json"), + ("x-trace", "a"), + ("x-trace", "b"), + ]); + let body = r#"{"model":"claude","max_tokens":5,"messages":[{"role":"user","content":"hi"}]}"#; + let a = arrived("/v1/messages", &h, body); + let r = build(&a, &h, &shaping(), "p", "claude").expect("built"); + let names: Vec<&str> = r.fields.iter().map(|(n, _)| n.as_str()).collect(); + assert_eq!( + names, + [ + "anthropic-version", + "content-type", + "user-agent", + "accept", + "x-trace", + "x-trace" + ] + ); + assert_eq!(field(&r, "x-trace"), [b"a".as_slice(), b"b".as_slice()]); +} + +/// THE `accept` FIELD IS BUSBAR'S OWN (1.5.5's bytes: the oracle's same-dialect cells, e.g. +/// `llm|openai|openai|request|ok`, carry `accept: application/json` though the caller sent `*/*`): +/// every far request carries the answer framing busbar reads the reply in, a same-dialect caller's +/// `accept` never replaces it, and the transport is left no default to fill. +#[test] +fn a_same_dialect_callers_accept_never_replaces_busbars_own() { + let h = head(&[("accept", "*/*"), ("x-trace", "a")]); + let body = r#"{"model":"gpt","messages":[{"role":"user","content":"hi"}]}"#; + let a = arrived("/v1/chat/completions", &h, body); + let r = build(&a, &h, &shaping(), "p", "gpt").expect("built"); + assert_eq!(field(&r, "accept"), [b"application/json".as_slice()]); + assert_eq!(field(&r, "x-trace"), [b"a".as_slice()]); + let streamed = r#"{"model":"gpt","stream":true,"messages":[{"role":"user","content":"hi"}]}"#; + let a = arrived("/v1/chat/completions", &h, streamed); + let r = build(&a, &h, &shaping(), "p", "gpt").expect("built"); + assert_eq!(field(&r, "accept"), [b"text/event-stream".as_slice()]); +} diff --git a/crates/busbar-plane-llm/tests/exchange_reply.rs b/crates/busbar-plane-llm/tests/exchange_reply.rs index 668a04ba79..b25809be59 100644 --- a/crates/busbar-plane-llm/tests/exchange_reply.rs +++ b/crates/busbar-plane-llm/tests/exchange_reply.rs @@ -659,6 +659,7 @@ fn lane(dialect: &'static str) -> Lane { prompt_caching: false, caps: Default::default(), error_map: Default::default(), + statics: &[], } } @@ -792,6 +793,54 @@ fn a_reply_judges_a_far_end_error() { } } +/// THE BREAKER'S READING beside the walk's verdict (ARCHITECT BREAKER OUTCOME): +/// a request too large for the window fails over with nothing recorded (RETRY, FAULT_NONE); an +/// `error_map` entry naming `rate_limit` fails over and counts toward the trip (RETRY, +/// FAULT_TRANSIENT); the caller's own bad input records nothing (HARD, FAULT_CALLER); a refused +/// credential trips every cell (HARD, FAULT_HARD) — 1.5.5's per-disposition records. +#[test] +fn a_judged_far_end_error_carries_the_breakers_reading_apart_from_its_verdict() { + use busbar_contract::abi::transport::{FAULT_CALLER, FAULT_HARD, FAULT_NONE, FAULT_TRANSIENT}; + use busbar_plane_llm::plane_door::breaker_fault; + let arrived = arrival("openai", false); + let mut lane = lane("openai"); + lane.error_map = map(&[("1302", "rate_limit")]); + let ctx = ReplyCtx { + arrived: &arrived, + lane: &lane, + intent: stream_intent(chat_handler("openai"), arrived.parsed.as_ref()), + passthrough: false, + }; + let error = |code: &str| { + format!(r#"{{"error":{{"message":"m","type":"invalid_request_error","code":"{code}"}}}}"#) + }; + for (status, code, verdict, reading) in [ + (400, "context_length_exceeded", Verdict::Retry, FAULT_NONE), + (400, "1302", Verdict::Retry, FAULT_TRANSIENT), + (400, "bad_field", Verdict::Hard, FAULT_CALLER), + (401, "invalid_api_key", Verdict::Hard, FAULT_HARD), + (503, "busy", Verdict::Retry, FAULT_TRANSIENT), + ] { + let mut reply = Reply::new(&ctx, status, JSON_HEAD); + let body = error(code); + let piece = reply.feed(&ctx, body.as_bytes(), true, AT); + assert!(piece.done, "{status} {code}"); + assert_eq!(piece.verdict, verdict, "{status} {code}"); + assert_eq!( + breaker_fault(piece.fault.as_ref()), + reading, + "{status} {code}" + ); + } + // RED: a 2xx that failed after its head is the compensating transient 1.5.5 recorded; an + // answer with no fault is no reading at all. + assert_eq!( + breaker_fault(Some(&Fault::Transient("upstream-generation-failed"))), + FAULT_TRANSIENT + ); + assert_eq!(breaker_fault(None), FAULT_NONE); +} + /// A cut: a stream after its first byte ends on the caller's error frame, `Hard`, the far end's /// transfer at fault; a whole answer cut is the caller's 502. #[test] diff --git a/crates/busbar-plane-llm/tests/exchange_user_agent.rs b/crates/busbar-plane-llm/tests/exchange_user_agent.rs new file mode 100644 index 0000000000..08579b8010 --- /dev/null +++ b/crates/busbar-plane-llm/tests/exchange_user_agent.rs @@ -0,0 +1,107 @@ +// SPDX-License-Identifier: Apache-2.0 +// Copyright (C) 2026 Busbar Inc and contributors + +//! OWNER RULING Q10 (LLM-UA): the far end's `user-agent`. On a same-dialect route the caller's +//! passes through unchanged, and a caller that sent none gets the dialect's declared native-SDK +//! fingerprint, as 1.5.5 sent. On a translated route the plane writes the far dialect's declared +//! fingerprint, and no caller value reaches the far end. + +use serde_json::json; + +use busbar_plane_llm::exchange::arrive::arrive; +use busbar_plane_llm::exchange::attempt::{build, FarRequest}; +use busbar_plane_llm::exchange::shaping::Shaping; + +fn shaping() -> Shaping { + Shaping::from_settings(&json!({ + "providers": { + "oai": { "protocol": "openai", "base_url": "https://api.example" }, + "ant": { "protocol": "anthropic", "base_url": "https://anthropic.example" }, + "goo": { "protocol": "gemini", "base_url": "https://gemini.example" } + }, + "models": { + "gpt": { "provider": "oai" }, + "claude": { "provider": "ant", "default_max_tokens": 321 }, + "gem": { "provider": "goo", "upstream_model": "gemini-pro" } + }, + "pools": { "p": { "members": ["gpt", "claude", "gem"] } } + })) + .expect("reads") +} + +const CHAT: &str = r#"{"model":"m","messages":[{"role":"user","content":"hi"}]}"#; +const MESSAGES: &str = + r#"{"model":"m","max_tokens":5,"messages":[{"role":"user","content":"hi"}]}"#; + +fn far(target: &str, body: &str, caller_ua: Option<&'static str>, member: &str) -> FarRequest { + let mut h: Vec<(&[u8], &[u8])> = + vec![(b"content-type".as_slice(), b"application/json".as_slice())]; + if let Some(ua) = caller_ua { + h.push((b"user-agent".as_slice(), ua.as_bytes())); + } + let a = arrive("POST", target, &h, body.as_bytes(), &()).expect("arrives"); + build(&a, &h, &shaping(), "p", member).expect("built") +} + +fn user_agent(r: &FarRequest) -> Vec<&[u8]> { + r.fields + .iter() + .filter(|(n, _)| n.eq_ignore_ascii_case("user-agent")) + .map(|(_, v)| v.as_slice()) + .collect() +} + +#[test] +fn a_same_dialect_far_end_sees_exactly_the_caller_s_user_agent() { + for (target, body, member) in [ + ("/v1/messages", MESSAGES, "claude"), + ("/v1/chat/completions", CHAT, "gpt"), + ] { + let r = far(target, body, Some("curl/8.5.0"), member); + assert_eq!(user_agent(&r), [b"curl/8.5.0".as_slice()], "{target}"); + } +} + +#[test] +fn a_same_dialect_caller_that_sent_no_user_agent_carries_the_1_5_5_fingerprint() { + for (target, body, member, fingerprint) in [ + ( + "/v1/messages", + MESSAGES, + "claude", + "Anthropic/Python 0.39.0", + ), + ("/v1/chat/completions", CHAT, "gpt", "OpenAI/Python 1.54.0"), + ] { + let r = far(target, body, None, member); + assert_eq!(user_agent(&r), [fingerprint.as_bytes()], "{target}"); + } +} + +#[test] +fn a_translated_far_end_sees_the_far_dialect_s_fingerprint_never_the_caller_s() { + for (target, body, member, fingerprint) in [ + ( + "/v1/chat/completions", + CHAT, + "claude", + "Anthropic/Python 0.39.0", + ), + ("/v1/messages", MESSAGES, "gpt", "OpenAI/Python 1.54.0"), + ( + "/v1/chat/completions", + CHAT, + "gem", + "google-genai-sdk/0.8.0 gl-python/3.11", + ), + ] { + for caller in [None, Some("curl/8.5.0")] { + let r = far(target, body, caller, member); + assert_eq!( + user_agent(&r), + [fingerprint.as_bytes()], + "{target} -> {member}, caller {caller:?}" + ); + } + } +} diff --git a/crates/busbar-plane-llm/tests/hook_absent_projection.rs b/crates/busbar-plane-llm/tests/hook_absent_projection.rs new file mode 100644 index 0000000000..3f3d9993ef --- /dev/null +++ b/crates/busbar-plane-llm/tests/hook_absent_projection.rs @@ -0,0 +1,46 @@ +// SPDX-License-Identifier: Apache-2.0 +// Copyright (C) 2026 Busbar Inc and contributors + +//! ABSENT SEMANTICS for the hook projection, on the plane that projects it (ported from the +//! predev-only `busbar-llm` test `hook_non_chat_projection_tests:: +//! absent_semantics_hold_for_opless_and_bodyless`, whose `read_hook_facts` is the plane's `project` +//! on the driver): an arrival the plane has no operation reader for, or one with no body, projects +//! the EMPTY view (no turn, no system text, no size signal) and is never refused as unreadable. + +use busbar_plane_llm::exchange::arrive::Arrived; +use busbar_plane_llm::exchange::project::{project, HookView}; + +fn arrived(dialect: &'static str, body: &[u8], parsed: Option<serde_json::Value>) -> Arrived { + Arrived { + dialect, + operation: busbar_contract::operation::OpVerb::EMBEDDINGS, + model: "m0".to_string(), + content_type: "application/json".to_string(), + body: body.to_vec(), + parsed, + path: "/v1/embeddings".to_string(), + query: None, + path_model: None, + } +} + +#[test] +fn absent_semantics_hold_for_opless_and_bodyless() { + // A JSON object body under a dialect with no operation reader: the empty view, not a refusal. + let v = serde_json::json!({"input": "x"}); + let view = project(&arrived( + "not-a-protocol", + &serde_json::to_vec(&v).expect("serializes"), + Some(v), + )) + .expect("an opless arrival is not unreadable"); + assert_eq!(view, HookView::default()); + // No body at all: the empty view. + let view = + project(&arrived("openai", b"", None)).expect("a bodyless arrival is not unreadable"); + assert_eq!(view.turns, Vec::<(String, String)>::new()); + assert_eq!( + (view.system, view.text_chars, view.turn_count), + (None, 0, 0) + ); +} diff --git a/crates/busbar-plane-llm/tests/hook_max_tokens_saturation_1_5_5.rs b/crates/busbar-plane-llm/tests/hook_max_tokens_saturation_1_5_5.rs new file mode 100644 index 0000000000..5c12df8fce --- /dev/null +++ b/crates/busbar-plane-llm/tests/hook_max_tokens_saturation_1_5_5.rs @@ -0,0 +1,60 @@ +// SPDX-License-Identifier: Apache-2.0 +// Copyright (C) 2026 Busbar Inc and contributors + +//! The size signal a hook sees for an output cap above `u32::MAX`, as 1.5.5 projected it +//! (HOOK-DRIFTS, lane-hook-drifts 7a0964496e: "1.5.5 behaviour wins"). +//! +//! In 1.5.5 the hook projection read the cap from the caller's body and saturated it +//! (`max_tokens_for`: `u32::try_from(n).unwrap_or(u32::MAX)`), under the dialect's own key +//! (`max_output_tokens` for the responses dialect, `max_tokens` for every other), and the v1.5.5 +//! hook tests pinned it: `max_tokens_saturates_not_wraps` expected `Some(u32::MAX)` for +//! `5_000_000_000`, and `max_tokens_signal_is_dialect_aware_for_responses` expected `Some(u32::MAX)` +//! for `u64::MAX`. The plane's hook projection (`project`) reads it so; the IR's cap is the far +//! end's, which a reader drops when it cannot carry it, and it is not the hook signal. + +use busbar_plane_llm::exchange::arrive::arrive; +use busbar_plane_llm::exchange::project::{max_tokens_for, project}; + +fn hook_max_tokens(target: &str, body: serde_json::Value) -> Option<u32> { + let bytes = serde_json::to_vec(&body).expect("body serializes"); + let fields: &[(&[u8], &[u8])] = &[(b"content-type", b"application/json")]; + let arrived = arrive("POST", target, fields, &bytes, &()).expect("the request arrives"); + project(&arrived).expect("the request reads").max_tokens +} + +#[test] +fn an_anthropic_cap_above_u32_saturates() { + let body = serde_json::json!({ + "model": "m0", + "max_tokens": 5_000_000_000u64, + "messages": [{"role": "user", "content": "hi"}] + }); + assert_eq!(hook_max_tokens("/v1/messages", body), Some(u32::MAX)); +} + +#[test] +fn an_openai_cap_above_u32_saturates() { + let body = serde_json::json!({ + "model": "m0", + "max_tokens": u64::MAX, + "messages": [{"role": "user", "content": "hi"}] + }); + assert_eq!( + hook_max_tokens("/v1/chat/completions", body), + Some(u32::MAX) + ); +} + +#[test] +fn the_cap_is_read_under_the_dialects_own_key() { + let responses = + serde_json::json!({"input": "hi", "max_tokens": 999, "max_output_tokens": 4096}); + assert_eq!(max_tokens_for(&responses, "responses"), Some(4096)); + let anthropic = serde_json::json!({"max_tokens": 512}); + assert_eq!(max_tokens_for(&anthropic, "anthropic"), Some(512)); + assert_eq!(max_tokens_for(&anthropic, "gemini"), Some(512)); + assert_eq!( + max_tokens_for(&serde_json::json!({"max_tokens": u64::MAX}), "anthropic"), + Some(u32::MAX) + ); +} diff --git a/crates/busbar-plane-llm/tests/on_driver/held.rs b/crates/busbar-plane-llm/tests/on_driver/held.rs new file mode 100644 index 0000000000..737861995d --- /dev/null +++ b/crates/busbar-plane-llm/tests/on_driver/held.rs @@ -0,0 +1,581 @@ +// SPDX-License-Identifier: Apache-2.0 +// Copyright (C) 2026 Busbar Inc and contributors + +//! The hook tests the legacy llm crate carries beyond the 1.5.5 set, held on the driver before that +//! crate is deleted: the access amendment a hook handed the prompt leaves (`hook_access_tests`), +//! the operation-general projection (`hook_non_chat_projection_tests`) and the in-band system turn +//! (`hook_system_turn_tests`), each under its name with its values. The two seat-order tests ride +//! the serving path's real admission and ledger, and are held there. + +use std::sync::{Arc, Mutex}; +use std::time::Duration; + +use busbar_contract::hooks::{ + Candidate, PolicyResult, RewriteReply, RoutingContext, RoutingDecision, RoutingPolicy, + RoutingRequest, TransformOutcome, +}; +use busbar_kernel::audit::amend::{node_recent, Access, AmendBody, Reader}; +use busbar_kernel::config::PolicyOnError; +use busbar_kernel::hooks::{FallbackHook, ResolvedPolicy}; +use serde_json::{json, Value}; + +use super::policies::capturing; +use crate::rig::{member, Answered, Hooks, Pool, Rig}; + +// ── the access amendment ───────────────────────────────────────────────────────────────────────── + +/// Every hook access the node journal holds for `name` (the journal is process-wide: every hook +/// here carries a name no other test uses). +fn accesses(name: &str) -> Vec<Access> { + node_recent() + .into_iter() + .filter_map(|a| match a.body { + AmendBody::Access(access) if access.reader == Reader::Hook && access.name == name => { + Some(access) + } + _ => None, + }) + .collect() +} + +/// A hook that abstains, rewrites nothing and swallows a tap: only its name matters. +struct Named { + name: &'static str, + fails: bool, +} + +#[async_trait::async_trait] +impl RoutingPolicy for Named { + async fn decide( + &self, + _: &RoutingRequest<'_>, + _: &[Candidate<'_>], + _: &RoutingContext<'_>, + _: Duration, + ) -> PolicyResult { + if self.fails { + return Err("deliberately broken".into()); + } + Ok(RoutingDecision::Abstain) + } + fn name(&self) -> &'static str { + self.name + } + async fn transform(&self, _: &RoutingRequest<'_>, _: Duration) -> TransformOutcome { + TransformOutcome::Abstain + } +} + +fn named(name: &'static str) -> Arc<dyn RoutingPolicy> { + Arc::new(Named { name, fails: false }) +} + +fn policy( + primary: Arc<dyn RoutingPolicy>, + send_prompt: bool, + send_user: bool, + on_error_chain: Vec<FallbackHook>, +) -> ResolvedPolicy { + ResolvedPolicy::Policy { + policy: primary, + on_error: PolicyOnError::Weighted, + on_error_chain, + timeout: Duration::from_millis(500), + send_prompt, + send_user, + on_empty: PolicyOnError::Reject, + } +} + +fn access_body() -> Value { + json!({ + "model": "m0", + "max_tokens": 16, + "system": "sys prompt", + "metadata": {"user_id": "alice"}, + "messages": [{"role": "user", "content": "hello"}] + }) +} + +fn one() -> Pool { + Pool { + name: "p", + members: vec![member("m0", "m0")], + } +} + +async fn decide(resolved: ResolvedPolicy) { + let rig = Rig::new( + one(), + None, + Hooks { + policy: Some(resolved), + ..Hooks::default() + }, + ); + rig.fire(&access_body()).await; +} + +#[tokio::test] +async fn a_decision_gate_handed_the_prompt_leaves_exactly_one_access_amendment() { + decide(policy( + named("access-decide-prompt"), + true, + true, + Vec::new(), + )) + .await; + let seen = accesses("access-decide-prompt"); + assert_eq!(seen.len(), 1, "one access per hand-over: {seen:?}"); + assert_eq!(seen[0].op_class.as_str(), "anthropic"); + assert_eq!( + seen[0].fields, + vec!["content".to_string(), "identity".to_string()], + "field NAMES that crossed, never their values" + ); +} + +#[tokio::test] +async fn a_decision_gate_handed_only_the_shape_leaves_no_access_amendment() { + decide(policy( + named("access-decide-shape"), + false, + true, + Vec::new(), + )) + .await; + assert!(accesses("access-decide-shape").is_empty()); +} + +#[tokio::test] +async fn an_on_error_fallback_handed_the_prompt_leaves_its_own_access_amendment() { + let primary: Arc<dyn RoutingPolicy> = Arc::new(Named { + name: "access-primary-fails", + fails: true, + }); + let fallback = FallbackHook { + policy: named("access-fallback-prompt"), + timeout: Duration::from_millis(500), + send_prompt: true, + send_user: false, + on_empty: PolicyOnError::Reject, + }; + decide(policy(primary, true, false, vec![fallback])).await; + assert_eq!(accesses("access-primary-fails").len(), 1); + let seen = accesses("access-fallback-prompt"); + assert_eq!( + seen.len(), + 1, + "the fallback was handed the prompt: {seen:?}" + ); + assert_eq!(seen[0].fields, vec!["content".to_string()]); +} + +#[tokio::test] +async fn an_on_error_fallback_without_the_prompt_grant_leaves_no_access_amendment() { + let primary: Arc<dyn RoutingPolicy> = Arc::new(Named { + name: "access-primary-fails-2", + fails: true, + }); + let fallback = FallbackHook { + policy: named("access-fallback-shape"), + timeout: Duration::from_millis(500), + send_prompt: false, + send_user: false, + on_empty: PolicyOnError::Reject, + }; + decide(policy(primary, true, false, vec![fallback])).await; + assert!(accesses("access-fallback-shape").is_empty()); +} + +#[tokio::test] +async fn a_rewrite_hook_handed_the_prompt_leaves_exactly_one_access_amendment() { + let rig = Rig::new( + one(), + None, + Hooks { + rewrites: vec![(Duration::from_millis(500), named("access-rewrite"))], + ..Hooks::default() + }, + ); + rig.fire(&access_body()).await; + let seen = accesses("access-rewrite"); + assert_eq!(seen.len(), 1, "one access per hand-over: {seen:?}"); + assert_eq!(seen[0].fields, vec!["content".to_string()]); +} + +#[tokio::test] +async fn a_global_tap_leaves_an_access_amendment_only_when_handed_the_prompt() { + let ms = Duration::from_millis(500); + let mut hooks = Hooks::default(); + hooks.taps.request = vec![ + (ms, true, named("access-tap-prompt"), Vec::new()), + (ms, false, named("access-tap-shape"), Vec::new()), + ]; + let rig = Rig::new(one(), None, hooks); + rig.fire(&json!({ + "model": "p", + "messages": [{"role": "user", "content": "hi"}], + "max_tokens": 10 + })) + .await; + assert_eq!(accesses("access-tap-prompt").len(), 1); + assert!(accesses("access-tap-shape").is_empty()); +} + +// ── the operation-general projection ───────────────────────────────────────────────────────────── + +/// The exact strings a `prompt: ro` hook is shown for `body` posted to `path`, flattened across +/// turns, and the size signal. +async fn gate_view(path: &str, content_type: &str, body: &[u8]) -> (String, usize) { + let (seen, policy) = capturing(true, false, None); + let rig = Rig::new( + Pool { + name: "p", + members: vec![member("m0", "m0").provider("oai")], + }, + None, + Hooks { + policy: Some(policy), + ..Hooks::default() + }, + ); + rig.fire_with( + &crate::common::TestUnits::passing(), + path, + &[("content-type", content_type)], + body, + ) + .await; + let captured = seen.lock().unwrap().clone(); + let c = captured.unwrap_or_else(|| panic!("the hook was shown the request at {path}")); + let (system, messages) = c.prompt.expect("a prompt: ro hook is shown the prompt"); + let mut out: Vec<String> = system.into_iter().collect(); + out.extend(messages.into_iter().map(|(_, t)| t)); + (out.join("\n"), c.total_chars) +} + +async fn json_view(path: &str, v: Value) -> (String, usize) { + gate_view( + path, + "application/json", + &serde_json::to_vec(&v).expect("body serializes"), + ) + .await +} + +#[tokio::test] +async fn embeddings_body_is_no_longer_gate_blind() { + let (view, chars) = json_view( + "/v1/embeddings", + json!({"model": "m0", "input": "SCREEN-THIS-INPUT"}), + ) + .await; + assert!(chars > 0, "embeddings request must not project empty"); + assert!(view.contains("SCREEN-THIS-INPUT")); +} + +#[tokio::test] +async fn image_body_is_no_longer_gate_blind() { + let (view, _) = json_view( + "/v1/images/generations", + json!({"model": "m0", "prompt": "SCREEN-THIS-PROMPT"}), + ) + .await; + assert!(view.contains("SCREEN-THIS-PROMPT")); +} + +#[tokio::test] +async fn speech_body_projects_input_and_instructions() { + // The OpenAI `/v1/audio/speech` request body as a client sends it. + let v: Value = serde_json::from_str(include_str!("speech_request.json")) + .expect("the speech request fixture is JSON"); + let (view, _) = json_view("/v1/audio/speech", v).await; + assert!(view.contains("SPEAK-THIS")); + assert!( + view.contains("STYLE-INSTRUCTIONS"), + "instructions must be screened" + ); +} + +#[tokio::test] +async fn moderation_body_projects_text_and_marks_image_url_opaque() { + let (view, _) = json_view( + "/v1/moderations", + json!({ + "model": "m0", + "input": [ + {"type": "text", "text": "SCREEN-THIS-TEXT"}, + {"type": "image_url", "image_url": {"url": "https://x.test/y.png"}} + ] + }), + ) + .await; + assert!(view.contains("SCREEN-THIS-TEXT")); + // The image URL is present-but-unscreenable, shown as the marker — not empty, not leaked. + assert!(view.contains(busbar_contract::ir::facts::OPAQUE_CONTENT_MARKER)); + assert!(!view.contains("x.test")); +} + +#[tokio::test] +async fn rerank_body_projects_query_and_documents() { + let (view, _) = json_view( + "/v2/rerank", + json!({"model": "m0", "query": "THE-QUERY", "documents": ["DOC-ONE", "DOC-TWO"]}), + ) + .await; + assert!(view.contains("THE-QUERY") && view.contains("DOC-ONE") && view.contains("DOC-TWO")); +} + +#[tokio::test] +async fn transcription_prompt_is_seen_through_the_byte_seam() { + let boundary = "----busbartestBOUNDARY"; + let body = format!( + "--{b}\r\nContent-Disposition: form-data; name=\"model\"\r\n\r\nm0\r\n\ + --{b}\r\nContent-Disposition: form-data; name=\"prompt\"\r\n\r\nSCREEN-THIS-PROMPT\r\n\ + --{b}\r\nContent-Disposition: form-data; name=\"file\"; filename=\"a.mp3\"\r\n\ + Content-Type: audio/mpeg\r\n\r\nRAWAUDIOBYTES\r\n\ + --{b}--\r\n", + b = boundary + ); + let ct = format!("multipart/form-data; boundary={boundary}"); + let (view, _) = gate_view("/v1/audio/transcriptions", &ct, body.as_bytes()).await; + assert!( + view.contains("SCREEN-THIS-PROMPT"), + "the multipart transcription prompt must be screenable through the byte reader" + ); +} + +// ── the in-band system turn ────────────────────────────────────────────────────────────────────── + +/// What a hook was shown. +#[derive(Debug, Clone, PartialEq)] +struct Seen { + system: Option<String>, + messages: Vec<(String, String)>, + message_count: usize, + total_chars: usize, +} + +/// A headroom-shaped compressor: echoes every turn as `{role, content}`, keeps a `system` turn and +/// the last turn verbatim, and replaces every other turn's text. +struct Compressor(Mutex<Option<Seen>>); + +#[async_trait::async_trait] +impl RoutingPolicy for Compressor { + async fn decide( + &self, + _: &RoutingRequest<'_>, + _: &[Candidate<'_>], + _: &RoutingContext<'_>, + _: Duration, + ) -> PolicyResult { + Ok(RoutingDecision::Abstain) + } + fn name(&self) -> &'static str { + "system-turn-compressor" + } + async fn transform(&self, req: &RoutingRequest<'_>, _: Duration) -> TransformOutcome { + let prompt = req.prompt.as_ref().expect("a rw hook is shown the prompt"); + let messages: Vec<(String, String)> = prompt + .messages + .iter() + .map(|(r, t)| (r.to_string(), t.to_string())) + .collect(); + *self.0.lock().unwrap() = Some(Seen { + system: prompt.system.as_deref().map(str::to_string), + messages: messages.clone(), + message_count: req.message_count, + total_chars: req.total_chars, + }); + let last = messages.len().saturating_sub(1); + TransformOutcome::Rewrite(RewriteReply { + messages: messages + .iter() + .enumerate() + .map(|(i, (role, text))| { + let text = if i == last || role == "system" { + text.clone() + } else { + "COMPRESSED".to_string() + }; + json!({"role": role, "content": text}) + }) + .collect(), + tools: vec![], + }) + } +} + +/// One rw hook over `v` posted to `path` in a same-dialect pool served by `provider`: what the +/// hook saw, and the body the far end was sent. +async fn rewrite(path: &str, provider: &'static str, v: Value) -> (Seen, Value) { + let hook = Arc::new(Compressor(Mutex::new(None))); + let rig = Rig::new( + Pool { + name: "p", + members: vec![member("m0", "m0").provider(provider)], + }, + None, + Hooks { + rewrites: vec![(Duration::from_millis(50), hook.clone())], + ..Hooks::default() + }, + ); + let answer: Answered = rig + .fire_with( + &crate::common::TestUnits::passing(), + path, + &[("content-type", "application/json")], + &serde_json::to_vec(&v).expect("body serializes"), + ) + .await; + let seen = hook.0.lock().unwrap().clone().expect("the hook fired"); + let sent = answer + .sent + .last() + .expect("the rewritten request was dispatched"); + ( + seen, + serde_json::from_slice(&sent.body).expect("the far end's body is JSON"), + ) +} + +fn turns(pairs: &[(&str, &str)]) -> Vec<(String, String)> { + pairs + .iter() + .map(|(r, t)| (r.to_string(), t.to_string())) + .collect() +} + +#[tokio::test] +async fn a_rewrite_keeps_the_in_band_system_turn_upstream() { + let (seen, sent) = rewrite( + "/v1/chat/completions", + "oai", + json!({ + "model": "m0", + "messages": [ + {"role": "system", "content": "OPERATOR SYSTEM PROMPT"}, + {"role": "user", "content": "a long earlier turn"}, + {"role": "assistant", "content": "an earlier answer"}, + {"role": "user", "content": "the ask"} + ] + }), + ) + .await; + assert_eq!( + seen, + Seen { + system: None, + messages: turns(&[ + ("system", "OPERATOR SYSTEM PROMPT"), + ("user", "a long earlier turn"), + ("assistant", "an earlier answer"), + ("user", "the ask"), + ]), + message_count: 4, + total_chars: 22 + 19 + 17 + 7, + }, + "the hook must be shown the 1.5.5 view" + ); + assert_eq!( + serde_json::to_string(&sent["messages"]).unwrap(), + r#"[{"content":"OPERATOR SYSTEM PROMPT","role":"system"},{"content":"COMPRESSED","role":"user"},{"content":"COMPRESSED","role":"assistant"},{"content":"the ask","role":"user"}]"#, + "the operator's system prompt must reach the provider, exactly as 1.5.5 sent it" + ); +} + +#[tokio::test] +async fn a_mid_conversation_developer_turn_keeps_its_place_and_role() { + let (seen, sent) = rewrite( + "/v1/chat/completions", + "oai", + json!({ + "model": "m0", + "messages": [ + {"role": "user", "content": "u1"}, + {"role": "developer", "content": "DEV"}, + {"role": "user", "content": "u2"} + ] + }), + ) + .await; + assert_eq!(seen.system, None); + assert_eq!( + seen.messages, + turns(&[("user", "u1"), ("developer", "DEV"), ("user", "u2")]) + ); + assert_eq!((seen.message_count, seen.total_chars), (3, 7)); + assert_eq!( + sent["messages"], + json!([ + {"role": "user", "content": "COMPRESSED"}, + {"role": "developer", "content": "COMPRESSED"}, + {"role": "user", "content": "u2"} + ]), + "1.5.5 shipped whatever the hook answered for the developer turn" + ); +} + +#[tokio::test] +async fn cohere_in_band_system_turn_is_a_turn() { + let (seen, sent) = rewrite( + "/v2/chat", + "coh", + json!({ + "model": "m0", + "messages": [ + {"role": "system", "content": "OPERATOR SYSTEM PROMPT"}, + {"role": "user", "content": "earlier"}, + {"role": "user", "content": "hi"} + ] + }), + ) + .await; + assert_eq!(seen.system, None); + assert_eq!( + seen.messages, + turns(&[ + ("system", "OPERATOR SYSTEM PROMPT"), + ("user", "earlier"), + ("user", "hi") + ]) + ); + assert_eq!(seen.message_count, 3); + assert_eq!(sent["messages"][0]["role"], "system"); + assert_eq!(sent["messages"][0]["content"], "OPERATOR SYSTEM PROMPT"); +} + +#[tokio::test] +async fn responses_instructions_are_system_and_a_system_item_is_a_turn() { + let (seen, policy) = capturing(true, false, None); + let rig = Rig::new( + one(), + None, + Hooks { + policy: Some(policy), + ..Hooks::default() + }, + ); + rig.fire_with( + &crate::common::TestUnits::passing(), + "/v1/responses", + &[("content-type", "application/json")], + &serde_json::to_vec(&json!({ + "model": "m0", + "instructions": "INSTR", + "input": [ + {"role": "system", "content": "S"}, + {"role": "user", "content": "hi"} + ] + })) + .unwrap(), + ) + .await; + let c = seen.lock().unwrap().clone().expect("the hook was shown it"); + let (system, messages) = c.prompt.expect("prompt shown"); + assert_eq!(system.as_deref(), Some("INSTR")); + assert_eq!(messages, turns(&[("system", "S"), ("user", "hi")])); + assert_eq!(c.message_count, 2); +} diff --git a/crates/busbar-plane-llm/tests/on_driver/mod.rs b/crates/busbar-plane-llm/tests/on_driver/mod.rs new file mode 100644 index 0000000000..84a302070a --- /dev/null +++ b/crates/busbar-plane-llm/tests/on_driver/mod.rs @@ -0,0 +1,24 @@ +// SPDX-License-Identifier: Apache-2.0 +// Copyright (C) 2026 Busbar Inc and contributors + +//! THE LLM PLANE'S DRIVER PROOFS: the 1.5.5 hook tests of llm origin, each under its 1.5.5 name and +//! asserting the values 1.5.5 asserted, kept here, in the plane's own crate, where the dialect names +//! they are written in belong (BUSBAR-1.6.0.md Law 5; ARCHITECT ruling on #464: the composition root +//! names zero plane or dialect instance vocabulary). +//! +//! They are proven through the plugin door: the composition root's `hook_parity_driver` suite links +//! this plane's door, loads it through the one loader, drives it on the kernel's plane driver with the +//! hook stage bound, and compiles these cases beside its rig (the root's manifest row +//! `[package.metadata.busbar.driver-proofs]`, which its `build.rs` turns into the one `#[path]` the +//! suite includes). The kernel and the loader are the root's to link, never a plane's, so the cases +//! are compiled into the root's test binary rather than this crate's: this directory has no +//! `main.rs`, and `cargo test -p busbar-plane-llm` does not build it. +//! +//! Development-only, like the switch it proves: compiled only under the plane's fold switch +//! (`llm-on-driver`). + +pub(crate) mod held; +pub(crate) mod policies; +pub(crate) mod projection; +pub(crate) mod seam; +pub(crate) mod wire; diff --git a/crates/busbar-plane-llm/tests/on_driver/policies.rs b/crates/busbar-plane-llm/tests/on_driver/policies.rs new file mode 100644 index 0000000000..c1ad66bbf6 --- /dev/null +++ b/crates/busbar-plane-llm/tests/on_driver/policies.rs @@ -0,0 +1,254 @@ +// SPDX-License-Identifier: Apache-2.0 +// Copyright (C) 2026 Busbar Inc and contributors + +//! The 1.5.5 hook tests' own in-process policies: a capturing policy, a canned gate, a rewriting +//! gate, an erroring policy and a capturing tap. + +use std::sync::{Arc, Mutex}; +use std::time::Duration; + +use busbar_contract::hooks::{ + Candidate, PolicyResult, RewriteReply, RoutingContext, RoutingDecision, RoutingPolicy, + RoutingRequest, TransformOutcome, +}; +use busbar_kernel::config::PolicyOnError; +use busbar_kernel::hooks::{ResolvedPolicy, TapEntry}; +use serde_json::Value; + +/// The prompt as the policy saw it: (flattened system, [(role, text)]). +pub type SeenPrompt = (Option<String>, Vec<(String, String)>); +/// The identity as the policy saw it: (key_id, key_name, end-user). +pub type SeenIdentity = (Option<String>, Option<String>, Option<String>); + +/// What the policy saw through the seam. +#[derive(Clone, Default, Debug)] +pub struct CapturedReq { + pub prompt: Option<SeenPrompt>, + pub identity: Option<SeenIdentity>, + pub max_tokens: Option<u32>, + pub message_count: usize, + pub total_chars: usize, + pub request_id: u64, +} + +/// A policy that records the projection it was handed and returns a fixed decision. +pub struct CapturingPolicy { + pub seen: Arc<Mutex<Option<CapturedReq>>>, + pub reject: Option<(u16, String)>, +} + +#[async_trait::async_trait] +impl RoutingPolicy for CapturingPolicy { + async fn decide( + &self, + req: &RoutingRequest<'_>, + _candidates: &[Candidate<'_>], + _ctx: &RoutingContext<'_>, + _budget: Duration, + ) -> PolicyResult { + *self.seen.lock().unwrap() = Some(CapturedReq { + prompt: req.prompt.as_ref().map(|p| { + ( + p.system.as_deref().map(str::to_string), + p.messages + .iter() + .map(|(r, t)| (r.to_string(), t.to_string())) + .collect(), + ) + }), + identity: req + .identity + .as_ref() + .map(|i| (i.key_id.clone(), i.key_name.clone(), i.user.clone())), + max_tokens: req.max_tokens, + message_count: req.message_count, + total_chars: req.total_chars, + request_id: req.request_id, + }); + Ok(match &self.reject { + Some((status, message)) => RoutingDecision::Reject { + status: *status, + message: message.clone(), + }, + None => RoutingDecision::Abstain, + }) + } + fn name(&self) -> &'static str { + "capture" + } +} + +/// A capturing policy as a resolved hook, with the grants given. +pub fn capturing( + send_prompt: bool, + send_user: bool, + reject: Option<(u16, String)>, +) -> (Arc<Mutex<Option<CapturedReq>>>, ResolvedPolicy) { + let seen = Arc::new(Mutex::new(None)); + ( + seen.clone(), + ResolvedPolicy::Policy { + policy: Arc::new(CapturingPolicy { seen, reject }), + on_error: PolicyOnError::default(), + on_error_chain: Vec::new(), + timeout: Duration::from_millis(500), + send_prompt, + send_user, + on_empty: PolicyOnError::Reject, + }, + ) +} + +/// A canned decision. +pub enum Canned { + Order(Vec<usize>), + Restrict(Vec<String>), + Reject(u16, &'static str), +} + +/// A gate returning a fixed decision. +pub struct CannedGate { + pub canned: Canned, + pub name: &'static str, +} + +#[async_trait::async_trait] +impl RoutingPolicy for CannedGate { + async fn decide( + &self, + _req: &RoutingRequest<'_>, + _candidates: &[Candidate<'_>], + _ctx: &RoutingContext<'_>, + _budget: Duration, + ) -> PolicyResult { + Ok(match &self.canned { + Canned::Order(order) => RoutingDecision::Prefer(order.clone()), + Canned::Restrict(tags) => RoutingDecision::Restrict { + tags_any: tags.clone(), + }, + Canned::Reject(status, message) => RoutingDecision::Reject { + status: *status, + message: (*message).to_string(), + }, + }) + } + fn name(&self) -> &'static str { + self.name + } +} + +/// A canned decision as a resolved gate (shape-only, fail-closed defaults). +pub fn canned_gate(canned: Canned, name: &'static str) -> ResolvedPolicy { + ResolvedPolicy::Policy { + policy: Arc::new(CannedGate { canned, name }), + on_error: PolicyOnError::default(), + on_error_chain: Vec::new(), + timeout: Duration::from_millis(500), + send_prompt: false, + send_user: false, + on_empty: PolicyOnError::Reject, + } +} + +/// A rewrite gate: abstains as a decision, rewrites the messages on transform. +pub struct RewritingGate(pub Vec<Value>); + +#[async_trait::async_trait] +impl RoutingPolicy for RewritingGate { + async fn decide( + &self, + _req: &RoutingRequest<'_>, + _candidates: &[Candidate<'_>], + _ctx: &RoutingContext<'_>, + _budget: Duration, + ) -> PolicyResult { + Ok(RoutingDecision::Abstain) + } + fn name(&self) -> &'static str { + "rewriter" + } + async fn transform(&self, _req: &RoutingRequest<'_>, _budget: Duration) -> TransformOutcome { + TransformOutcome::Rewrite(RewriteReply { + messages: self.0.clone(), + tools: vec![], + }) + } +} + +/// A rewrite gate replacing the turns with one user turn. +pub fn rewriting(content: &str) -> (Duration, Arc<dyn RoutingPolicy>) { + ( + Duration::from_millis(500), + Arc::new(RewritingGate(vec![ + serde_json::json!({"role": "user", "content": content}), + ])), + ) +} + +/// A policy whose decide always errors. +pub struct ErroringPolicy; + +#[async_trait::async_trait] +impl RoutingPolicy for ErroringPolicy { + async fn decide( + &self, + _req: &RoutingRequest<'_>, + _candidates: &[Candidate<'_>], + _ctx: &RoutingContext<'_>, + _budget: Duration, + ) -> PolicyResult { + Err("deliberately broken".into()) + } + fn name(&self) -> &'static str { + "erroring" + } +} + +/// A tap whose `notify` records the delivered projection. +pub struct CaptureTap { + pub last: Mutex<Option<Vec<u8>>>, +} + +#[async_trait::async_trait] +impl RoutingPolicy for CaptureTap { + async fn decide( + &self, + _req: &RoutingRequest<'_>, + _cands: &[Candidate<'_>], + _ctx: &RoutingContext<'_>, + _budget: Duration, + ) -> PolicyResult { + Ok(RoutingDecision::Abstain) + } + fn name(&self) -> &'static str { + "capture-tap" + } + async fn notify( + &self, + tap: Arc<busbar_contract::abi::host::hook::NotifyFrame>, + _budget: Duration, + ) { + let projection = serde_json::to_vec(&tap.projection_json()).expect("the tap's JSON"); + *self.last.lock().unwrap() = Some(projection); + } +} + +/// An in-process tap capture and its tap entry. +pub fn webhook_tap() -> (Arc<CaptureTap>, TapEntry) { + let cap = Arc::new(CaptureTap { + last: Mutex::new(None), + }); + let policy: Arc<dyn RoutingPolicy> = cap.clone(); + (cap, (Duration::from_millis(500), false, policy, Vec::new())) +} + +/// The tap's delivered projection (taps are detached tasks). +pub async fn wait_for_tap_body(cap: &CaptureTap) -> Value { + for _ in 0..200 { + if let Some(body) = cap.last.lock().unwrap().clone() { + return serde_json::from_slice(&body).expect("tap payload is JSON"); + } + tokio::time::sleep(Duration::from_millis(10)).await; + } + panic!("tap was never delivered"); +} diff --git a/crates/busbar-plane-llm/tests/on_driver/projection.rs b/crates/busbar-plane-llm/tests/on_driver/projection.rs new file mode 100644 index 0000000000..a32584188a --- /dev/null +++ b/crates/busbar-plane-llm/tests/on_driver/projection.rs @@ -0,0 +1,766 @@ +// SPDX-License-Identifier: Apache-2.0 +// Copyright (C) 2026 Busbar Inc and contributors + +//! `v1.5.5` `crates/busbar/src/proxy/tests/hook_opt_in_projection_tests.rs`, on the driver: what a +//! `prompt: ro` hook is shown of a request — the prompt view, the size signals, the end user, the +//! reasoning shapes and the opaque-content marker — as the llm plane's `project` writes it and the +//! driver hands it to the hook. Each test keeps its 1.5.5 name and the values 1.5.5 asserted. +//! +//! What a crossing does not carry is not asserted: whether a string was borrowed from the body +//! (every string crosses into the host's arena), and the system-only character count (the hook +//! wire never carried it; where a 1.5.5 test pinned it, the request carries no turn, so the total +//! it IS shown equals it). + +use serde_json::{json, Value}; + +use super::policies::{capturing, CapturedReq}; +use crate::rig::{member, Hooks, Pool, Rig}; + +/// The fixed non-content marker shown in place of opaque provider-encrypted reasoning. +const MARKER: &str = "[busbar:redacted_reasoning]"; + +/// Where each dialect's request is posted, and what its body needs beside the content. +fn target(dialect: &str, mut v: Value) -> (&'static str, Value) { + let obj = v.as_object_mut().expect("a JSON object body"); + match dialect { + "anthropic" => { + obj.entry("model").or_insert(json!("m0")); + obj.entry("max_tokens").or_insert(json!(16)); + ("/v1/messages", v) + } + "openai" => { + obj.entry("model").or_insert(json!("m0")); + ("/v1/chat/completions", v) + } + "responses" => { + obj.entry("model").or_insert(json!("m0")); + ("/v1/responses", v) + } + "gemini" => ("/v1beta/models/m0:generateContent", v), + "bedrock" => ("/model/m0/converse", v), + "cohere" => { + obj.entry("model").or_insert(json!("m0")); + ("/v2/chat", v) + } + other => panic!("no target for {other}"), + } +} + +/// What a `prompt: ro` base policy is shown of `v` posted in `dialect`; `None` when the request +/// never reached the hooks. +async fn shown(dialect: &str, v: Value) -> Option<CapturedReq> { + let (seen, policy) = capturing(true, true, None); + let rig = Rig::new( + Pool { + name: "p", + members: vec![member("m0", "m0")], + }, + None, + Hooks { + policy: Some(policy), + ..Hooks::default() + }, + ); + let (path, body) = target(dialect, v); + let answer = rig + .fire_with( + &crate::common::TestUnits::passing(), + path, + if dialect == "anthropic" { + &[ + ("content-type", "application/json"), + ("anthropic-version", "2023-06-01"), + ] + } else { + &[("content-type", "application/json")] + }, + &serde_json::to_vec(&body).expect("body serializes"), + ) + .await; + let captured = seen.lock().unwrap().clone(); + if captured.is_none() { + eprintln!( + "{dialect} {path}: {} {} {:?}", + answer.status, + answer.text(), + answer.outcome + ); + } + captured +} + +/// [`shown`], when the request must reach the hooks. +async fn seen(dialect: &str, v: Value) -> CapturedReq { + shown(dialect, v) + .await + .unwrap_or_else(|| panic!("the hook was shown the {dialect} request")) +} + +fn prompt(c: &CapturedReq) -> (Option<String>, Vec<(String, String)>) { + c.prompt + .clone() + .expect("a prompt: ro hook is shown the prompt") +} + +fn turns(pairs: &[(&str, &str)]) -> Vec<(String, String)> { + pairs + .iter() + .map(|(r, t)| (r.to_string(), t.to_string())) + .collect() +} + +#[tokio::test] +async fn prompt_projection_flattens_string_and_block_content() { + let c = seen( + "anthropic", + json!({ + "system": "be brief", + "messages": [ + {"role": "user", "content": "hello"}, + {"role": "assistant", "content": [ + {"type": "text", "text": "part one"}, + {"type": "image", "source": {"type": "base64", "media_type": "image/png", "data": "AAAA"}}, + {"type": "text", "text": "part two"} + ]} + ] + }), + ) + .await; + let (system, messages) = prompt(&c); + assert_eq!(system.as_deref(), Some("be brief")); + assert_eq!( + messages, + turns(&[("user", "hello"), ("assistant", "part one\npart two")]) + ); +} + +#[tokio::test] +async fn prompt_projection_system_blocks_and_absent() { + let c = seen( + "anthropic", + json!({ + "system": [{"type": "text", "text": "sys a"}, {"type": "text", "text": "sys b"}], + "messages": [{"role": "user", "content": "hi"}] + }), + ) + .await; + assert_eq!(prompt(&c).0.as_deref(), Some("sys a\nsys b")); + + let c = seen( + "anthropic", + json!({"messages": [{"role": "user", "content": "hi"}]}), + ) + .await; + assert_eq!(prompt(&c).0, None); +} + +#[tokio::test] +async fn prompt_projection_keeps_empty_entries_aligned() { + let c = seen( + "anthropic", + json!({ + "messages": [ + {"role": "user", "content": [ + {"type": "image", "source": {"type": "base64", "media_type": "image/png", "data": "AAAA"}} + ]}, + {"role": "assistant", "content": "second turn"} + ] + }), + ) + .await; + let (_, messages) = prompt(&c); + assert_eq!(messages.len(), 2, "media-only entries must not vanish"); + assert_eq!(messages[0].0, "user"); + assert_eq!(messages[0].1, "", "media-only turn reads as empty text"); + assert_eq!( + messages[1], + ("assistant".to_string(), "second turn".to_string()) + ); + assert_eq!(c.message_count, 2); +} + +#[tokio::test] +async fn system_text_chars_counts_block_arrays() { + // No turn carries text, so the total a hook is shown is the system's own count. + let c = seen( + "anthropic", + json!({ + "system": [{"type": "text", "text": "abcde"}, {"type": "text", "text": "fgh"}], + "messages": [{"role": "user", "content": ""}] + }), + ) + .await; + assert_eq!(c.total_chars, 8); + assert_eq!(prompt(&c).0.as_deref(), Some("abcde\nfgh")); + + let c = seen( + "anthropic", + json!({"system": "plain", "messages": [{"role": "user", "content": ""}]}), + ) + .await; + assert_eq!(c.total_chars, 5); +} + +#[tokio::test] +async fn prompt_projection_reads_gemini_contents() { + let c = seen( + "gemini", + json!({ + "systemInstruction": {"parts": [{"text": "be brief"}]}, + "contents": [ + {"role": "user", "parts": [{"text": "hello"}]}, + {"role": "model", "parts": [ + {"text": "part one"}, + {"inlineData": {"mimeType": "image/png", "data": "AAAA"}}, + {"text": "part two"} + ]} + ] + }), + ) + .await; + let (system, messages) = prompt(&c); + assert_eq!(system.as_deref(), Some("be brief")); + assert_eq!( + messages, + turns(&[("user", "hello"), ("assistant", "part one\npart two")]) + ); + assert_eq!(c.message_count, 2); + assert_eq!(c.total_chars, 8 + 5 + 16); +} + +#[tokio::test] +async fn prompt_projection_reads_responses_input() { + let c = seen( + "responses", + json!({ + "instructions": "be brief", + "input": [ + {"role": "user", "content": "hello"}, + {"role": "assistant", "content": [{"type": "output_text", "text": "hi there"}]} + ] + }), + ) + .await; + let (system, messages) = prompt(&c); + assert_eq!(system.as_deref(), Some("be brief")); + assert_eq!( + messages, + turns(&[("user", "hello"), ("assistant", "hi there")]) + ); + assert_eq!(c.message_count, 2); + + // A bare-string `input` is ONE implicit user turn. + let c = seen("responses", json!({"input": "just a question"})).await; + assert_eq!(prompt(&c).1, turns(&[("user", "just a question")])); + assert_eq!(c.message_count, 1); + assert_eq!(c.total_chars, 15); + + // Top-level typed items carry text at the item root, roles inferred from `type`. + let c = seen( + "responses", + json!({ + "input": [ + {"type": "input_text", "text": "hello"}, + {"type": "output_text", "text": "hi back"} + ] + }), + ) + .await; + assert_eq!( + prompt(&c).1, + turns(&[("user", "hello"), ("assistant", "hi back")]), + "top-level input_text/output_text items must project with inferred roles, not blank" + ); + assert_eq!(c.message_count, 2); + assert_eq!(c.total_chars, 12); +} + +#[tokio::test] +async fn max_tokens_signal_is_dialect_aware_for_responses() { + // Responses ingress: only `max_output_tokens` is present. + let c = seen( + "responses", + json!({"input": "hi", "max_output_tokens": 4096}), + ) + .await; + assert_eq!(c.max_tokens, Some(4096)); + // A stray `max_tokens` on a responses body is NOT the signal. + let c = seen( + "responses", + json!({"input": "hi", "max_tokens": 999, "max_output_tokens": 4096}), + ) + .await; + assert_eq!(c.max_tokens, Some(4096)); + // Every other dialect reads `max_tokens`. + let c = seen( + "anthropic", + json!({"messages": [{"role": "user", "content": "hi"}], "max_tokens": 512}), + ) + .await; + assert_eq!(c.max_tokens, Some(512)); + let c = seen( + "gemini", + json!({"contents": [{"role": "user", "parts": [{"text": "hi"}]}], "max_tokens": 512}), + ) + .await; + assert_eq!(c.max_tokens, Some(512)); + // An absurd cap saturates rather than wrapping. + let c = seen( + "openai", + json!({"messages": [{"role": "user", "content": "hi"}], "max_tokens": u64::MAX}), + ) + .await; + assert_eq!(c.max_tokens, Some(u32::MAX)); +} + +#[tokio::test] +async fn prompt_projection_reads_bedrock_messages() { + let c = seen( + "bedrock", + json!({ + "system": [{"text": "sys"}], + "messages": [ + {"role": "user", "content": [{"text": "hello"}, {"text": "again"}]} + ] + }), + ) + .await; + let (system, messages) = prompt(&c); + assert_eq!(system.as_deref(), Some("sys")); + assert_eq!(messages[0].1, "hello\nagain"); + assert_eq!(c.message_count, 1); + assert_eq!(c.total_chars, 3 + 10); +} + +fn user_of(c: &CapturedReq) -> Option<String> { + c.identity.clone().and_then(|(_, _, u)| u) +} + +#[tokio::test] +async fn body_end_user_reads_both_dialects() { + let hi = json!([{"role": "user", "content": "hi"}]); + let c = seen("openai", json!({"user": "alice", "messages": hi})).await; + assert_eq!(user_of(&c).as_deref(), Some("alice")); + let c = seen( + "anthropic", + json!({"metadata": {"user_id": "bob"}, "messages": hi}), + ) + .await; + assert_eq!(user_of(&c).as_deref(), Some("bob")); + let c = seen("anthropic", json!({"messages": hi})).await; + assert_eq!(user_of(&c), None); +} + +/// One assistant turn carrying `block`, in `dialect`'s own conversation shape. +fn assistant_turn(dialect: &str, block: Value) -> Value { + match dialect { + "responses" => json!({"input": [block]}), + "gemini" => json!({"contents": [{"role": "model", "parts": [block]}]}), + _ => json!({"messages": [{"role": "assistant", "content": [block]}]}), + } +} + +#[tokio::test] +async fn prompt_projection_sees_anthropic_thinking_text() { + let c = seen( + "anthropic", + assistant_turn( + "anthropic", + json!({"type": "thinking", "thinking": "SMUGGLED", "signature": "sig"}), + ), + ) + .await; + assert_eq!(prompt(&c).1, turns(&[("assistant", "SMUGGLED")])); +} + +#[tokio::test] +async fn prompt_projection_sees_bedrock_reasoning_text() { + let c = seen( + "bedrock", + assistant_turn( + "bedrock", + json!({"reasoningContent": {"reasoningText": {"text": "SMUGGLED", "signature": "sig-xyz"}}}), + ), + ) + .await; + let (_, messages) = prompt(&c); + assert_eq!(messages, turns(&[("assistant", "SMUGGLED")])); + assert!(!messages[0].1.contains("sig-xyz")); +} + +#[tokio::test] +async fn prompt_projection_sees_responses_reasoning_summary_only() { + let c = seen( + "responses", + assistant_turn( + "responses", + json!({"type": "reasoning", "summary": [{"type": "summary_text", "text": "SMUGGLED"}]}), + ), + ) + .await; + let (_, messages) = prompt(&c); + assert_eq!(messages.len(), 1); + assert_eq!(messages[0].1, "SMUGGLED"); +} + +#[tokio::test] +async fn prompt_projection_marks_anthropic_redacted_thinking() { + let c = seen( + "anthropic", + assistant_turn( + "anthropic", + json!({"type": "redacted_thinking", "data": "OPAQUE_CIPHERTEXT_BYTES"}), + ), + ) + .await; + let (_, messages) = prompt(&c); + assert_eq!(messages, turns(&[("assistant", MARKER)])); + assert!(!messages[0].1.contains("OPAQUE_CIPHERTEXT_BYTES")); +} + +#[tokio::test] +async fn prompt_projection_marks_bedrock_redacted_content() { + let c = seen( + "bedrock", + assistant_turn( + "bedrock", + json!({"reasoningContent": {"redactedContent": "OPAQUE_CIPHERTEXT_BYTES"}}), + ), + ) + .await; + let (_, messages) = prompt(&c); + assert_eq!(messages, turns(&[("assistant", MARKER)])); + assert!(!messages[0].1.contains("OPAQUE_CIPHERTEXT_BYTES")); +} + +#[tokio::test] +async fn prompt_projection_marks_responses_encrypted_content_only_reasoning() { + let c = seen( + "responses", + assistant_turn( + "responses", + json!({"type": "reasoning", "encrypted_content": "OPAQUE_BLOB_XYZ"}), + ), + ) + .await; + let (_, messages) = prompt(&c); + assert_eq!(messages, turns(&[("assistant", MARKER)])); + assert!(!messages[0].1.contains("OPAQUE_BLOB_XYZ")); +} + +#[tokio::test] +async fn prompt_projection_and_total_chars_mark_responses_reasoning_with_empty_content_array() { + let c = seen( + "responses", + assistant_turn( + "responses", + json!({"type": "reasoning", "content": [], "encrypted_content": "OPAQUE_BLOB_XYZ"}), + ), + ) + .await; + let (_, messages) = prompt(&c); + assert_eq!(messages, turns(&[("assistant", MARKER)])); + assert!(!messages[0].1.contains("OPAQUE_BLOB_XYZ")); + assert_eq!( + c.total_chars, + MARKER.chars().count(), + "the size signal counts the marker's length, not silently 0" + ); +} + +#[tokio::test] +async fn block_text_responses_reasoning_rejects_malformed_encrypted_content() { + // A malformed blob is not a real opaque blob: an item carrying nothing else projects no + // marker (the reader drops it, as it never ships it). + for blob in [json!(""), json!(123)] { + let c = seen( + "responses", + json!({"input": [ + {"type": "reasoning", "encrypted_content": blob}, + {"role": "user", "content": "q"} + ]}), + ) + .await; + assert!( + prompt(&c).1.iter().all(|(_, t)| !t.contains(MARKER)), + "{blob}: a malformed encrypted_content must not read as opaque content" + ); + } +} + +#[tokio::test] +async fn prompt_projection_responses_reasoning_prefers_text_over_encrypted_content() { + let c = seen( + "responses", + assistant_turn( + "responses", + json!({ + "type": "reasoning", + "content": [{"type": "reasoning_text", "text": "VISIBLE"}], + "encrypted_content": "ENC_BLOB_123" + }), + ), + ) + .await; + let (_, messages) = prompt(&c); + assert_eq!(messages.len(), 1); + assert_eq!(messages[0].1, "VISIBLE"); + assert!(!messages[0].1.contains(MARKER)); + assert!(!messages[0].1.contains("ENC_BLOB_123")); +} + +#[tokio::test] +async fn responses_single_part_reasoning_text_borrows() { + let c = seen( + "responses", + assistant_turn( + "responses", + json!({"type": "reasoning", "content": [{"type": "reasoning_text", "text": "one part"}]}), + ), + ) + .await; + assert_eq!(prompt(&c).1[0].1, "one part"); + let c = seen( + "responses", + assistant_turn( + "responses", + json!({"type": "reasoning", "summary": [{"type": "summary_text", "text": "just a summary"}]}), + ), + ) + .await; + assert_eq!(prompt(&c).1[0].1, "just a summary"); +} + +#[tokio::test] +async fn responses_multi_part_reasoning_text_concatenates() { + let c = seen( + "responses", + assistant_turn( + "responses", + json!({ + "type": "reasoning", + "content": [ + {"type": "reasoning_text", "text": "first "}, + {"type": "reasoning_text", "text": "second "} + ], + "summary": [{"type": "summary_text", "text": "third"}] + }), + ), + ) + .await; + assert_eq!(prompt(&c).1[0].1, "first second third"); +} + +#[tokio::test] +async fn prompt_projection_attributes_responses_reasoning_to_assistant() { + let c = seen( + "responses", + assistant_turn( + "responses", + json!({"type": "reasoning", "content": [{"type": "reasoning_text", "text": "chain of thought"}]}), + ), + ) + .await; + let (_, messages) = prompt(&c); + assert_eq!(messages.len(), 1); + assert_eq!(messages[0].0, "assistant"); +} + +#[tokio::test] +async fn prompt_projection_mixed_text_and_thinking_turn_joins_both() { + let c = seen( + "anthropic", + json!({ + "messages": [ + {"role": "assistant", "content": [ + {"type": "text", "text": "visible answer"}, + {"type": "thinking", "thinking": "SMUGGLED", "signature": "sig"} + ]} + ] + }), + ) + .await; + assert_eq!(prompt(&c).1[0].1, "visible answer\nSMUGGLED"); +} + +#[tokio::test] +async fn prompt_projection_gemini_thought_part_still_projects() { + let c = seen( + "gemini", + json!({ + "contents": [ + {"role": "model", "parts": [ + {"text": "the answer", "thought": false}, + {"text": "reasoning about it", "thought": true, "thoughtSignature": "sig"} + ]} + ] + }), + ) + .await; + assert_eq!(prompt(&c).1[0].1, "the answer\nreasoning about it"); +} + +#[tokio::test] +async fn total_text_chars_counts_reasoning_text() { + let c = seen( + "anthropic", + assistant_turn( + "anthropic", + json!({"type": "thinking", "thinking": "12345"}), + ), + ) + .await; + assert_eq!(c.total_chars, 5); + let c = seen( + "bedrock", + assistant_turn( + "bedrock", + json!({"reasoningContent": {"reasoningText": {"text": "1234567"}}}), + ), + ) + .await; + assert_eq!(c.total_chars, 7); + let c = seen( + "responses", + assistant_turn( + "responses", + json!({"type": "reasoning", "summary": [{"type": "summary_text", "text": "123"}]}), + ), + ) + .await; + assert_eq!(c.total_chars, 3); +} + +#[tokio::test] +async fn size_signal_and_projection_agree_on_reasoning() { + let c = seen( + "anthropic", + json!({ + "messages": [ + {"role": "assistant", "content": [ + {"type": "text", "text": "visible"}, + {"type": "thinking", "thinking": "hidden reasoning"} + ]}, + {"role": "assistant", "content": [ + {"type": "redacted_thinking", "data": "opaque"} + ]} + ] + }), + ) + .await; + let projected: usize = prompt(&c).1.iter().map(|(_, t)| t.chars().count()).sum(); + // One separator newline per block boundary within a turn; the size signal counts text only. + assert_eq!(projected, c.total_chars + 1); +} + +#[tokio::test] +async fn every_known_protocol_has_a_declared_reasoning_wire_shape() { + // (dialect, a reasoning block in its own shape, the text it must show, or the marker). + let rows: Vec<(&str, Option<Value>, Option<&str>)> = vec![ + ( + "anthropic", + Some(json!({"type": "thinking", "thinking": "W"})), + Some("W"), + ), + ( + "anthropic", + Some(json!({"type": "redacted_thinking", "data": "b64"})), + None, + ), + ( + "bedrock", + Some(json!({"reasoningContent": {"reasoningText": {"text": "W"}}})), + Some("W"), + ), + ( + "bedrock", + Some(json!({"reasoningContent": {"redactedContent": "b64"}})), + None, + ), + ( + "responses", + Some(json!({"type": "reasoning", "summary": [{"type": "summary_text", "text": "W"}]})), + Some("W"), + ), + ( + "responses", + Some(json!({"type": "reasoning", "encrypted_content": "OPAQUE_BLOB"})), + None, + ), + // No dialect-specific reasoning shape: the generic text probe covers it. + ("gemini", None, None), + ("openai", None, None), + ("cohere", None, None), + ]; + let dialects = crate::rig::dialects(); + let dialects: Vec<&str> = dialects.iter().map(String::as_str).collect(); + for d in &dialects { + assert!( + rows.iter().any(|(p, _, _)| p == d), + "dialect '{d}' has no row in this witness table — declare its reasoning wire shape" + ); + } + assert_eq!( + rows.iter() + .map(|(p, _, _)| *p) + .collect::<std::collections::HashSet<_>>() + .len(), + dialects.len(), + "witness table covers a different dialect set than the plane's" + ); + for (dialect, sample, text) in rows { + let Some(sample) = sample else { continue }; + let c = seen(dialect, assistant_turn(dialect, sample)).await; + let shown = prompt(&c).1; + let turn = &shown.last().expect("the reasoning turn is shown").1; + match text { + Some(needle) => assert!(turn.contains(needle), "{dialect}: {turn:?}"), + None => assert_eq!(turn, MARKER, "{dialect}"), + } + } +} + +#[tokio::test] +async fn apply_rewrite_to_body_echoes_redacted_marker_as_visible_text() { + let body = json!({ + "model": "m0", + "max_tokens": 16, + "messages": [ + {"role": "assistant", "content": [ + {"type": "redacted_thinking", "data": "OPAQUE_CIPHERTEXT_BYTES"} + ]} + ] + }); + let c = seen("anthropic", body.clone()).await; + assert_eq!(prompt(&c).1[0].1, MARKER); + + // A hook that echoes exactly what it was projected (the common "pass through" rewrite shape). + let echo = std::sync::Arc::new(super::policies::RewritingGate(vec![json!({ + "role": "assistant", + "content": MARKER, + })])); + let rig = Rig::new( + Pool { + name: "p", + members: vec![member("m0", "m0")], + }, + None, + Hooks { + rewrites: vec![(std::time::Duration::from_millis(500), echo)], + ..Hooks::default() + }, + ); + let answer = rig.fire(&body).await; + let sent: Value = serde_json::from_slice( + &answer + .sent + .last() + .expect("the rewritten request was dispatched") + .body, + ) + .expect("the far end's body is JSON"); + let msgs = sent["messages"].as_array().expect("messages array"); + assert_eq!(msgs.len(), 1); + assert_eq!(msgs[0]["content"].as_str(), Some(MARKER)); + assert!(!sent.to_string().contains("OPAQUE_CIPHERTEXT_BYTES")); +} diff --git a/crates/busbar-plane-llm/tests/on_driver/seam.rs b/crates/busbar-plane-llm/tests/on_driver/seam.rs new file mode 100644 index 0000000000..b10da639dd --- /dev/null +++ b/crates/busbar-plane-llm/tests/on_driver/seam.rs @@ -0,0 +1,1021 @@ +// SPDX-License-Identifier: Apache-2.0 +// Copyright (C) 2026 Busbar Inc and contributors + +//! `v1.5.5` `crates/busbar/src/proxy/tests/hook_seam_tests.rs`, on the driver: the opt-in grants +//! (what a hook is shown), the decision gates and the base policy (reject, restrict, order and +//! their reconcile), the on-error chain, the rewrite chain, the stage taps, the reject envelopes +//! and the correlation id. Each test keeps its 1.5.5 name and the values 1.5.5 asserted. + +use std::sync::{Arc, Mutex}; +use std::time::Duration; + +use busbar_kernel::config::PolicyOnError; +use busbar_kernel::hooks::{FallbackHook, ResolvedPolicy}; +use busbar_kernel::plane_driver::{CallerKey, Constraint, Restrict}; +use serde_json::{json, Value}; + +use super::policies::{ + canned_gate, capturing, rewriting, wait_for_tap_body, webhook_tap, Canned, CannedGate, + ErroringPolicy, +}; +use crate::rig::{member, Callers, Hooks, Pool, Rig}; + +/// The principal the kernel's passing steps verify. +const PRINCIPAL: &str = "acct:battery"; + +fn pool(members: Vec<crate::rig::Member>) -> Pool { + Pool { name: "p", members } +} + +/// One live member. +fn one() -> Pool { + pool(vec![member("m0", "m0")]) +} + +/// One dead member. +fn one_dead() -> Pool { + pool(vec![member("m0", "m0").dead()]) +} + +fn body() -> Value { + // The end-user id spelled the way this dialect spells it (`metadata.user_id` on Anthropic), + // and the `user` key beside it, as a client may send both. + json!({ + "model": "m0", + "max_tokens": 16, + "system": "sys prompt", + "user": "alice", + "metadata": {"user_id": "alice"}, + "messages": [{"role": "user", "content": "hello"}] + }) +} + +fn chat_body() -> Value { + json!({ + "model": "p", + "messages": [{"role": "user", "content": "hi"}], + "max_tokens": 10 + }) +} + +/// The previous release's `run`: a one-member pool whose base route policy is a capturing +/// policy with these grants and this decision, one unit of `v`; what it saw and the answer. +async fn run( + send_prompt: bool, + send_user: bool, + reject: Option<(u16, String)>, + v: Value, +) -> (crate::rig::Answered, Option<super::policies::CapturedReq>) { + let (seen, policy) = capturing(send_prompt, send_user, reject); + let rig = Rig::new( + one(), + None, + Hooks { + policy: Some(policy), + ..Hooks::default() + }, + ); + let answer = rig.fire(&v).await; + let captured = seen.lock().unwrap().clone(); + (answer, captured) +} + +fn gates(gates: Vec<(u16, ResolvedPolicy)>) -> Hooks { + Hooks { + gates, + ..Hooks::default() + } +} + +#[tokio::test] +async fn default_flags_project_nothing() { + let (answer, captured) = run(false, false, None, body()).await; + let captured = captured.expect("policy must have been called"); + assert!(captured.prompt.is_none(), "send_prompt off ⇒ no prompt"); + assert!(captured.identity.is_none(), "send_user off ⇒ no identity"); + assert_eq!(answer.status, 200, "abstain ⇒ the walk's own pick serves"); +} + +#[tokio::test] +async fn global_gate_reject_short_circuits_the_request() { + let (_, gate) = capturing(false, false, Some((451, "blocked by global policy".into()))); + let rig = Rig::new(one_dead(), None, gates(vec![(0, gate)])); + let answer = rig.fire(&chat_body()).await; + assert_eq!( + answer.status, 451, + "a global reject gate must short-circuit with its clamped status, before any dispatch" + ); + assert!(answer.sent.is_empty(), "no far end was reached"); +} + +#[tokio::test] +async fn global_gate_abstain_does_not_reject() { + let (_, gate) = capturing(false, false, None); + let rig = Rig::new(one_dead(), None, gates(vec![(0, gate)])); + let answer = rig.fire(&chat_body()).await; + assert_ne!( + answer.status, 451, + "an abstaining global gate must not reject the request" + ); + assert_eq!(answer.sent.len(), 1, "the request proceeded to the walk"); +} + +#[test] +fn enforce_restricts_reapplies_compliance_tags_across_pools() { + // Fallback pool "fb": member 0 carries `baa`, member 1 carries nothing. + let tags: Vec<Vec<String>> = vec![vec!["baa".to_string()], vec![]]; + let members = || tags.iter().enumerate().map(|(i, t)| (i, t.as_slice())); + let restrict = |tag: &str, on_empty: PolicyOnError, name: &'static str| Constraint { + restricts: vec![Restrict { + tags_any: vec![tag.to_string()], + on_empty, + name, + }], + ..Constraint::default() + }; + + // A required `baa` restrict narrows the fallback pool to ONLY the tagged member. + assert_eq!( + restrict("baa", PolicyOnError::Reject, "baa-gate") + .enforce(members()) + .unwrap(), + vec![0], + "only the baa-tagged fallback lane may survive a required restrict" + ); + // A required restrict with NO matching fallback member fails CLOSED, never spills. + assert!( + restrict("hipaa", PolicyOnError::Reject, "hipaa-gate") + .enforce(members()) + .is_err(), + "a required restrict with no eligible fallback lane must fail closed, not spill" + ); + // A `weighted` restrict with no match is an advisory escape — candidates pass unchanged. + assert_eq!( + restrict("hipaa", PolicyOnError::Weighted, "hipaa-advisory") + .enforce(members()) + .unwrap() + .len(), + 2, + "a weighted restrict with no eligible lane escapes (candidates unchanged)" + ); + // No active restrict → identity. + assert_eq!(Constraint::default().enforce(members()).unwrap().len(), 2); +} + +#[tokio::test] +async fn base_policy_restrict_persists_across_fallback_pool_hop() { + let primary = pool(vec![member("primary", "primary").tags(&["baa"]).dead()]); + let fallback = Pool { + name: "fb", + members: vec![member("fbmember", "fbmember")], + }; + let rig = Rig::new( + primary, + Some(fallback), + Hooks { + policy: Some(canned_gate( + Canned::Restrict(vec!["baa".to_string()]), + "compliance", + )), + ..Hooks::default() + }, + ); + let answer = rig.fire(&chat_body()).await; + assert_eq!( + answer.status, 503, + "a base-policy compliance restrict must fail closed at the fallback boundary" + ); + assert!( + answer.text().contains("restriction"), + "the 503 must be the compliance fail-closed, not a generic transport 503: {}", + answer.text() + ); + assert!( + answer.sent.iter().all(|r| r.member != "fbmember"), + "the ineligible fallback member is never served" + ); +} + +#[tokio::test] +async fn completion_tap_fires_synthetic_rejected_by_auth() { + let (cap, tap) = webhook_tap(); + let mut hooks = Hooks::default(); + hooks.taps.response = vec![tap]; + let rig = Rig::new(one_dead(), None, hooks); + let answer = rig + .fire_unauthenticated("/v1/messages", &serde_json::to_vec(&chat_body()).unwrap()) + .await; + assert_eq!(answer.status, 401); + let payload = wait_for_tap_body(&cap).await; + assert_eq!(payload["stage"]["at"], "response"); + assert_eq!(payload["stage"]["outcome"], "rejected_by_auth"); + assert_eq!(payload["stage"]["status"], 401); +} + +#[tokio::test] +async fn completion_tap_status_is_protocol_native_gemini_400() { + let (cap, tap) = webhook_tap(); + let mut hooks = Hooks::default(); + hooks.taps.response = vec![tap]; + let rig = Rig::new(one_dead(), None, hooks); + // Gemini ingress path + bad key → the served response is HTTP 400 (not 401). + let answer = rig + .fire_unauthenticated( + "/v1beta/models/gemini-x:generateContent", + br#"{"contents":[{"role":"user","parts":[{"text":"hi"}]}]}"#, + ) + .await; + assert_eq!(answer.status, 400, "gemini bad-key is native 400"); + let payload = wait_for_tap_body(&cap).await; + assert_eq!(payload["stage"]["outcome"], "rejected_by_auth"); + assert_eq!( + payload["stage"]["status"], 400, + "the tap status must match the client-visible native status, not a hardcoded 401" + ); +} + +#[tokio::test] +async fn completion_tap_fires_synthetic_rejected_by_gate() { + let (cap, tap) = webhook_tap(); + let mut hooks = gates(vec![( + 0, + canned_gate(Canned::Reject(451, "denied"), "denier"), + )]); + hooks.taps.response = vec![tap]; + let rig = Rig::new(one_dead(), None, hooks); + let answer = rig.fire(&chat_body()).await; + assert_eq!(answer.status, 451); + let payload = wait_for_tap_body(&cap).await; + assert_eq!(payload["stage"]["at"], "response"); + assert_eq!(payload["stage"]["outcome"], "rejected_by_gate"); + assert_eq!(payload["stage"]["status"], 451); +} + +#[tokio::test] +async fn completion_tap_reports_ok_outcome() { + let (cap, tap) = webhook_tap(); + let mut hooks = Hooks::default(); + hooks.taps.response = vec![tap]; + let rig = Rig::new(pool(vec![member("m0", "served")]), None, hooks); + let answer = rig.fire(&chat_body()).await; + assert_eq!(answer.status, 200); + let payload = wait_for_tap_body(&cap).await; + assert_eq!(payload["stage"]["at"], "response"); + assert_eq!(payload["stage"]["outcome"], "ok"); + assert_eq!(payload["stage"]["status"], 200); +} + +#[tokio::test] +async fn attempt_tap_carries_attempt_story() { + let (cap, tap) = webhook_tap(); + let mut hooks = Hooks::default(); + hooks.taps.routing = vec![tap]; + let rig = Rig::new(pool(vec![member("m0", "served")]), None, hooks); + let answer = rig.fire(&chat_body()).await; + assert_eq!(answer.status, 200); + let payload = wait_for_tap_body(&cap).await; + assert_eq!(payload["stage"]["at"], "routing"); + assert_eq!(payload["stage"]["attempt_number"], 1); + assert_eq!(payload["stage"]["model"], "m0"); + assert!( + payload["stage"].get("previous_failure").is_none(), + "no failure precedes the first attempt" + ); +} + +#[tokio::test] +async fn route_tap_reports_surviving_candidates() { + let (cap, tap) = webhook_tap(); + let mut hooks = Hooks::default(); + hooks.taps.candidate = vec![tap]; + let rig = Rig::new(pool(vec![member("m0", "served")]), None, hooks); + let answer = rig.fire(&chat_body()).await; + assert_eq!(answer.status, 200); + let payload = wait_for_tap_body(&cap).await; + assert_eq!(payload["stage"]["at"], "candidate"); + assert_eq!(payload["stage"]["remaining_candidates"], 1); +} + +/// The body the far end was sent. +fn sent_body(answer: &crate::rig::Answered) -> Value { + let last = answer + .sent + .last() + .expect("upstream must have been dispatched"); + serde_json::from_slice(&last.body).expect("the far end's body is JSON") +} + +#[tokio::test] +async fn same_protocol_passthrough_carries_global_rewrite() { + let rig = Rig::new( + one(), + None, + Hooks { + rewrites: vec![rewriting("COMPRESSED")], + ..Hooks::default() + }, + ); + let answer = rig.fire(&chat_body()).await; + assert_eq!(answer.status, 200); + assert_eq!( + sent_body(&answer)["messages"][0]["content"], + "COMPRESSED", + "the upstream must see the REWRITTEN body on the same-protocol fast path" + ); +} + +#[tokio::test] +async fn pool_scoped_rw_gate_rewrites_the_body() { + let rig = Rig::new( + one(), + None, + Hooks { + pool_rewrites: vec![rewriting("POOL-COMPRESSED")], + ..Hooks::default() + }, + ); + let answer = rig.fire(&chat_body()).await; + assert_eq!(answer.status, 200); + assert_eq!( + sent_body(&answer)["messages"][0]["content"], + "POOL-COMPRESSED", + "a pool-scoped rw gate must rewrite the dispatched body" + ); +} + +#[tokio::test] +async fn on_error_fallback_hook_fires_and_decides() { + let gate = ResolvedPolicy::Policy { + policy: Arc::new(ErroringPolicy), + on_error: PolicyOnError::Weighted, + on_error_chain: vec![FallbackHook { + policy: Arc::new(CannedGate { + canned: Canned::Reject(451, "fallback says no"), + name: "backup", + }), + timeout: Duration::from_millis(500), + send_prompt: false, + send_user: false, + on_empty: PolicyOnError::Reject, + }], + timeout: Duration::from_millis(50), + send_prompt: false, + send_user: false, + on_empty: PolicyOnError::Reject, + }; + let rig = Rig::new(one_dead(), None, gates(vec![(0, gate)])); + let answer = rig.fire(&chat_body()).await; + assert_eq!( + answer.status, 451, + "the failed gate's fallback hook must fire and its reject must be honored" + ); +} + +#[tokio::test] +async fn on_error_chain_exhausted_applies_terminal() { + let gate = ResolvedPolicy::Policy { + policy: Arc::new(ErroringPolicy), + on_error: PolicyOnError::Reject, + on_error_chain: vec![FallbackHook { + policy: Arc::new(ErroringPolicy), + timeout: Duration::from_millis(50), + send_prompt: false, + send_user: false, + on_empty: PolicyOnError::Reject, + }], + timeout: Duration::from_millis(50), + send_prompt: false, + send_user: false, + on_empty: PolicyOnError::Reject, + }; + let rig = Rig::new(one_dead(), None, gates(vec![(0, gate)])); + let answer = rig.fire(&chat_body()).await; + assert_eq!( + answer.status, 503, + "an exhausted chain must land on the fail-closed reject terminal" + ); + assert!(answer.sent.is_empty(), "refused before any dispatch"); +} + +#[tokio::test] +async fn on_error_reject_terminal_short_circuits_before_a_live_lane_ever_dispatches() { + let gate = ResolvedPolicy::Policy { + policy: Arc::new(ErroringPolicy), + on_error: PolicyOnError::Reject, + on_error_chain: vec![], + timeout: Duration::from_millis(50), + send_prompt: false, + send_user: false, + on_empty: PolicyOnError::Reject, + }; + let rig = Rig::new( + pool(vec![member("m0", "live-lane")]), + None, + gates(vec![(0, gate)]), + ); + let answer = rig.fire(&chat_body()).await; + assert_eq!( + answer.status, 503, + "an errored gate terminating in `on_error: reject` must fail closed BEFORE dispatch — a \ + live, healthy lane must never actually serve this request" + ); + assert!(answer.sent.is_empty()); +} + +#[tokio::test] +async fn global_request_stage_tap_fires_on_a_real_dispatched_request() { + let (cap, tap) = webhook_tap(); + let mut hooks = Hooks::default(); + hooks.taps.request = vec![tap]; + let rig = Rig::new(pool(vec![member("m0", "live-lane")]), None, hooks); + let answer = rig.fire(&chat_body()).await; + assert_eq!( + answer.status, 200, + "the request must still dispatch normally" + ); + let payload = wait_for_tap_body(&cap).await; + assert_eq!( + payload["op"], "notify", + "the global request-stage tap must receive the real notify wire envelope: {payload}" + ); +} + +#[tokio::test] +async fn pool_gate_reject_fires_from_pool_runtime_gates() { + let rig = Rig::new( + one_dead(), + None, + Hooks { + pool_gates: vec![( + 0, + canned_gate(Canned::Reject(451, "pool gate says no"), "pg"), + )], + ..Hooks::default() + }, + ); + let answer = rig.fire(&chat_body()).await; + assert_eq!( + answer.status, 451, + "a pool gate must fire in the phase-2 reconcile" + ); +} + +#[tokio::test] +async fn reject_priority_tie_break_surfaces_lowest_priority() { + // Deliberately stated out of order: the stable sort must put 1 before 5. + let rig = Rig::new( + one_dead(), + None, + gates(vec![ + ( + 5, + canned_gate(Canned::Reject(451, "high priority number"), "late"), + ), + ( + 1, + canned_gate(Canned::Reject(452, "low priority number"), "early"), + ), + ]), + ); + let answer = rig.fire(&chat_body()).await; + assert_eq!( + answer.status, 452, + "the lowest-priority rejecting gate must supply the surfacing status" + ); +} + +#[tokio::test] +async fn multi_restrict_disjoint_intersection_fails_closed() { + let rig = Rig::new( + pool(vec![ + member("eu-lane", "eu-lane").tags(&["eu"]).dead(), + member("baa-lane", "baa-lane").tags(&["baa"]).dead(), + ]), + None, + gates(vec![ + (0, canned_gate(Canned::Restrict(vec!["eu".into()]), "geo")), + ( + 1, + canned_gate(Canned::Restrict(vec!["baa".into()]), "hipaa"), + ), + ]), + ); + let answer = rig.fire(&chat_body()).await; + assert_eq!( + answer.status, 503, + "disjoint concurrent restricts must intersect to empty and fail closed" + ); +} + +#[tokio::test] +async fn multi_restrict_intersection_dispatches_only_the_survivor() { + let rig = Rig::new( + pool(vec![ + member("eu-only", "eu-only").tags(&["eu"]).dead(), + member("eu-baa", "both").tags(&["eu", "baa"]), + ]), + None, + gates(vec![ + (0, canned_gate(Canned::Restrict(vec!["eu".into()]), "geo")), + ( + 1, + canned_gate(Canned::Restrict(vec!["baa".into()]), "hipaa"), + ), + ]), + ); + let answer = rig.fire(&chat_body()).await; + assert_eq!(answer.status, 200); + assert_eq!( + answer.model(), + "both", + "dispatch must stay inside the restrict intersection" + ); +} + +#[tokio::test] +async fn stale_order_filtered_against_post_restrict_set() { + let rig = Rig::new( + pool(vec![ + member("excluded", "excluded").tags(&["a"]).dead(), + member("kept-lane", "kept").tags(&["b"]), + ]), + None, + gates(vec![ + (0, canned_gate(Canned::Order(vec![0]), "orderer")), + ( + 1, + canned_gate(Canned::Restrict(vec!["b".into()]), "restrictor"), + ), + ]), + ); + let answer = rig.fire(&chat_body()).await; + assert_eq!(answer.status, 200); + assert_eq!( + answer.model(), + "kept", + "a t0 order naming only restricted-out members must abstain to the surviving set" + ); +} + +#[tokio::test] +async fn order_last_in_chain_wins() { + let rig = Rig::new( + pool(vec![member("lane-a", "alpha"), member("lane-b", "beta")]), + None, + gates(vec![ + (0, canned_gate(Canned::Order(vec![0, 1]), "first")), + (1, canned_gate(Canned::Order(vec![1, 0]), "second")), + ]), + ); + let answer = rig.fire(&chat_body()).await; + assert_eq!(answer.status, 200); + assert_eq!( + answer.model(), + "beta", + "the LAST ordering gate in the chain must win the reconcile" + ); +} + +#[tokio::test] +async fn last_order_gate_filtered_to_empty_abstains_to_base_not_to_a_lower_gate() { + let rig = Rig::new( + pool(vec![ + member("base-lane", "base").tags(&["keep"]), + member("low-lane", "low").tags(&["keep"]), + member("excluded-lane", "excluded").dead(), + ]), + None, + gates(vec![ + ( + 0, + canned_gate(Canned::Restrict(vec!["keep".into()]), "restrictor"), + ), + (10, canned_gate(Canned::Order(vec![1]), "low-orderer")), + (20, canned_gate(Canned::Order(vec![2]), "high-orderer")), + ]), + ); + let answer = rig.fire(&chat_body()).await; + assert_eq!(answer.status, 200); + assert_eq!( + answer.model(), + "base", + "a filtered-to-empty highest-priority order must abstain to the pool's base ordering, \ + not fall through to a lower-priority gate's stale order" + ); +} + +#[tokio::test] +async fn global_gate_order_arm_is_honored() { + let rig = Rig::new( + pool(vec![member("lane-a", "alpha"), member("lane-b", "beta")]), + None, + gates(vec![(0, canned_gate(Canned::Order(vec![1]), "prefer-b"))]), + ); + let answer = rig.fire(&chat_body()).await; + assert_eq!(answer.status, 200); + assert_eq!( + answer.model(), + "beta", + "a global ordering gate must steer dispatch (the ORDER arm is live)" + ); +} + +#[tokio::test] +async fn send_prompt_projects_content_only() { + let (_, captured) = run(true, false, None, body()).await; + let captured = captured.expect("policy must have been called"); + let (system, messages) = captured.prompt.expect("send_prompt on ⇒ prompt present"); + assert_eq!(system.as_deref(), Some("sys prompt")); + assert_eq!(messages, vec![("user".to_string(), "hello".to_string())]); + assert!(captured.identity.is_none(), "send_user off ⇒ no identity"); +} + +#[tokio::test] +async fn send_user_projects_identity_only() { + let (_, captured) = run(false, true, None, body()).await; + let captured = captured.expect("policy must have been called"); + assert!(captured.prompt.is_none(), "send_prompt off ⇒ no prompt"); + let (key_id, key_name, user) = captured.identity.expect("send_user on ⇒ identity present"); + assert_eq!(key_id, None, "no governance ⇒ no key id"); + assert_eq!(key_name, None, "no governance ⇒ no key name"); + assert_eq!(user.as_deref(), Some("alice")); +} + +#[tokio::test] +async fn reject_decision_maps_to_reject_request_outcome() { + let (answer, captured) = run(true, false, Some((451, "PII detected".into())), body()).await; + assert!(captured.is_some(), "the policy decided"); + assert_eq!(answer.status, 451); + assert_eq!(answer.json()["error"]["message"], "PII detected"); + assert!( + answer.sent.is_empty(), + "a rejection is a decision: nothing dispatched" + ); +} + +#[tokio::test] +async fn max_tokens_saturates_not_wraps() { + let v = json!({ + "model": "m0", + "max_tokens": 5_000_000_000u64, + "messages": [{"role": "user", "content": "hi"}] + }); + let (_, captured) = run(false, false, None, v).await; + let captured = captured.expect("policy must have been called"); + // 1.5.5: an absurd caller cap still signals "the largest cap there is". + assert_eq!(captured.max_tokens, Some(u32::MAX)); + assert_ne!( + captured.max_tokens, + Some(5_000_000_000u64 as u32), + "the one thing that must never happen is a wrap to a small number" + ); +} + +/// A rig whose verified caller has `key`, and whose base policy is a `send_user` capture. +async fn identity_seen(key: Option<CallerKey>) -> super::policies::CapturedReq { + let (seen, policy) = capturing(false, true, None); + let mut callers = Callers::default(); + if let Some(key) = key { + callers.keys.insert(PRINCIPAL.to_string(), key); + } + let rig = Rig::new( + one(), + None, + Hooks { + policy: Some(policy), + callers, + ..Hooks::default() + }, + ); + rig.fire(&body()).await; + let captured = seen.lock().unwrap().clone(); + captured.expect("policy called") +} + +#[tokio::test] +async fn send_user_projects_governance_key_identity() { + let secret = "bb_sk_secret-material-never-projected"; + let captured = identity_seen(Some(CallerKey { + id: "vk_123".to_string(), + name: "sales-team".to_string(), + })) + .await; + let (key_id, key_name, user) = captured.identity.expect("identity present"); + assert_eq!(key_id.as_deref(), Some("vk_123")); + assert_eq!(key_name.as_deref(), Some("sales-team")); + assert_eq!(user.as_deref(), Some("alice")); + // The secret NEVER rides the projection, under any configuration. + assert_ne!(key_id.as_deref(), Some(secret)); + assert_ne!(key_name.as_deref(), Some(secret)); +} + +#[tokio::test] +async fn send_user_falls_back_to_synthesized_group_key_identity() { + // The key the identity step resolved for a group/SSO principal: its id/name carry the + // principal. + let captured = identity_seen(Some(CallerKey { + id: "eng-oncall".to_string(), + name: "eng-oncall".to_string(), + })) + .await; + let (key_id, key_name, _user) = captured.identity.expect("identity present"); + assert_eq!( + key_id.as_deref(), + Some("eng-oncall"), + "a group principal's synthesized key id must project, not fall through to None" + ); + assert_eq!(key_name.as_deref(), Some("eng-oncall")); +} + +#[tokio::test] +async fn send_user_prefers_resolved_key_over_disabled_legacy_lookup() { + // The identity a hook sees is the key of the principal the kernel VERIFIED (the authoritative + // admission), never a second lookup of the raw token. + let captured = identity_seen(Some(CallerKey { + id: "synthesized-principal".to_string(), + name: "synthesized-principal".to_string(), + })) + .await; + let (key_id, key_name, _user) = captured.identity.expect("identity present"); + assert_eq!( + key_id.as_deref(), + Some("synthesized-principal"), + "must resolve to the key auth actually authorized (the synthesized key), \ + not the disabled legacy key a raw `lookup` would hit" + ); + assert_eq!(key_name.as_deref(), Some("synthesized-principal")); +} + +#[tokio::test] +async fn forward_with_pool_keyed_threads_group_key_to_pool_policy() { + let captured = identity_seen(Some(CallerKey { + id: "eng-oncall".to_string(), + name: "eng-oncall".to_string(), + })) + .await; + let (key_id, _, _) = captured.identity.expect("identity present"); + assert_eq!( + key_id.as_deref(), + Some("eng-oncall"), + "the resolved key must reach a pool's send_user policy" + ); +} + +#[tokio::test] +async fn reject_status_and_message_resanitized_at_the_seam() { + let (answer, _) = run( + false, + false, + Some((500, "evil\r\ninjected\u{202E}spoof".into())), + body(), + ) + .await; + assert_eq!(answer.status, 403); + assert_eq!( + answer.json()["error"]["message"], + "evilinjectedspoof", + "seam must re-sanitize" + ); +} + +/// One unit refused by a gate rejecting with `status` and `message`, at `target` in `dialect`. +async fn gate_refusal( + status: u16, + message: &'static str, + target: &str, + fields: &[(&str, &str)], + body: &[u8], +) -> crate::rig::Answered { + let rig = Rig::new( + one_dead(), + None, + gates(vec![( + 0, + canned_gate(Canned::Reject(status, message), "guard"), + )]), + ); + rig.fire_with(&crate::common::TestUnits::passing(), target, fields, body) + .await +} + +const ANTHROPIC: &[(&str, &str)] = &[ + ("content-type", "application/json"), + ("anthropic-version", "2023-06-01"), +]; + +#[tokio::test] +async fn reject_408_maps_to_anthropic_timeout_envelope() { + let answer = gate_refusal( + 408, + "guardrail: request deadline", + "/v1/messages", + ANTHROPIC, + &serde_json::to_vec(&chat_body()).unwrap(), + ) + .await; + assert_eq!(answer.status, 408); + let v = answer.json(); + assert_eq!(v["error"]["type"], "timeout_error"); + assert_eq!(v["error"]["message"], "guardrail: request deadline"); +} + +#[tokio::test] +async fn reject_rides_the_full_forward_path() { + let (seen, policy) = capturing(false, false, Some((451, "PII detected".into()))); + let rig = Rig::new( + Pool { + name: "pa", + members: vec![member("m0", "m0")], + }, + None, + Hooks { + policy: Some(policy), + ..Hooks::default() + }, + ); + let answer = rig + .fire(&json!({ + "model": "pa", + "messages": [{"role": "user", "content": "hi"}], + "max_tokens": 5 + })) + .await; + assert_eq!( + answer.status, 451, + "the hook's status must reach the caller" + ); + let v = answer.json(); + assert_eq!(v["type"], "error"); + assert_eq!(v["error"]["type"], "invalid_request_error"); + assert_eq!(v["error"]["message"], "PII detected"); + assert!( + seen.lock().unwrap().is_some(), + "the policy must actually have decided" + ); +} + +/// The gate's status picks the refusal's kind, rendered in the caller's dialect (here Anthropic's +/// `error.type`): 401, 403, 404, 408 and 429 their own kinds, every other 4xx an invalid request. +#[tokio::test] +async fn reject_kind_mapping_matches_status_semantics() { + for (status, kind) in [ + (401, "authentication_error"), + (403, "permission_error"), + (404, "not_found_error"), + (408, "timeout_error"), + (429, "rate_limit_error"), + (400, "invalid_request_error"), + (422, "invalid_request_error"), + (451, "invalid_request_error"), + (499, "invalid_request_error"), + ] { + let answer = gate_refusal( + status, + "guardrail", + "/v1/messages", + ANTHROPIC, + &serde_json::to_vec(&chat_body()).unwrap(), + ) + .await; + assert_eq!(answer.status, u32::from(status), "{status}"); + assert_eq!(answer.json()["error"]["type"], kind, "{status}"); + } +} + +#[tokio::test] +async fn reject_message_reaches_anthropic_error_body() { + let answer = gate_refusal( + 451, + "PII detected in message 3", + "/v1/messages", + ANTHROPIC, + &serde_json::to_vec(&chat_body()).unwrap(), + ) + .await; + assert_eq!(answer.status, 451); + let v = answer.json(); + assert_eq!(v["type"], "error"); + assert_eq!( + v["error"]["message"], "PII detected in message 3", + "the hook's sanitized reject message must reach the client verbatim" + ); +} + +#[tokio::test] +async fn reject_produces_bedrock_native_envelope() { + let answer = gate_refusal( + 429, + "quota guardrail: try later", + "/model/m0/converse", + &[("content-type", "application/json")], + br#"{"messages":[{"role":"user","content":[{"text":"hi"}]}]}"#, + ) + .await; + assert_eq!(answer.status, 429); + assert!( + answer + .fields + .iter() + .any(|(n, _)| n.eq_ignore_ascii_case(b"x-amzn-errortype")), + "Bedrock error envelope must carry x-amzn-errortype" + ); + assert_eq!( + answer.json()["message"], + "quota guardrail: try later", + "Bedrock error body carries the message field" + ); +} + +#[test] +fn request_id_counter_is_unique_and_monotonic_across_sequential_requests() { + // The counter the driver's binder stamps each unit from is the process's one counter + // (`App::next_request_id`). + let app = busbar_kernel::test_support::TestApp::new().build(); + let a = app.next_request_id(); + let b = app.next_request_id(); + let c = app.next_request_id(); + assert!(b > a, "the counter must strictly increase: {a} then {b}"); + assert!(c > b, "the counter must strictly increase: {b} then {c}"); + assert_eq!(b, a + 1, "a plain fetch_add(1): no gaps between requests"); + assert_eq!(c, a + 2); + let app2 = busbar_kernel::test_support::TestApp::new().build(); + let d = app2.next_request_id(); + assert_ne!( + d, a, + "two independently-seeded counters must not coincide (boot entropy seed)" + ); +} + +#[tokio::test] +async fn same_request_id_joins_gate_decision_and_completion_tap() { + let (cap, tap) = webhook_tap(); + let (seen, gate) = capturing(false, false, None); + let mut hooks = gates(vec![(0, gate)]); + hooks.taps.response = vec![tap]; + let rig = Rig::new(pool(vec![member("m0", "served")]), None, hooks); + let answer = rig.fire(&chat_body()).await; + assert_eq!(answer.status, 200); + let gate_request_id = seen + .lock() + .unwrap() + .as_ref() + .expect("the gate ran and captured the RoutingRequest") + .request_id; + let payload = wait_for_tap_body(&cap).await; + let tap_request_id = payload["request"]["request_id"] + .as_u64() + .expect("response tap payload carries request_id as a plain integer"); + assert_eq!( + gate_request_id, tap_request_id, + "the pre-forward routing decision and the post-response response tap for the SAME \ + request must carry the IDENTICAL request_id — that identity is the join-key contract" + ); +} + +/// A layer capturing the value recorded onto a native `u64` field named `request_id`. +#[derive(Clone, Default)] +struct RequestIdSpanCapture(Arc<Mutex<Option<u64>>>); + +impl<S: tracing::Subscriber> tracing_subscriber::Layer<S> for RequestIdSpanCapture { + fn on_record( + &self, + _id: &tracing::span::Id, + values: &tracing::span::Record<'_>, + _ctx: tracing_subscriber::layer::Context<'_, S>, + ) { + struct Vis<'a>(&'a Mutex<Option<u64>>); + impl tracing::field::Visit for Vis<'_> { + fn record_u64(&mut self, field: &tracing::field::Field, value: u64) { + if field.name() == "request_id" { + *self.0.lock().unwrap() = Some(value); + } + } + fn record_debug(&mut self, _: &tracing::field::Field, _: &dyn std::fmt::Debug) {} + } + values.record(&mut Vis(&self.0)); + } +} + +#[tokio::test] +async fn request_id_is_recorded_as_native_u64_tracing_field() { + let (cap, tap) = webhook_tap(); + let mut hooks = Hooks::default(); + hooks.taps.response = vec![tap]; + let rig = Rig::new(pool(vec![member("m0", "served")]), None, hooks); + let seen = Arc::new(Mutex::new(None)); + use tracing_subscriber::layer::SubscriberExt as _; + let subscriber = tracing_subscriber::registry().with(RequestIdSpanCapture(seen.clone())); + let _guard = tracing::subscriber::set_default(subscriber); + tracing::callsite::rebuild_interest_cache(); + let answer = rig.fire(&chat_body()).await; + assert_eq!(answer.status, 200); + let payload = wait_for_tap_body(&cap).await; + let tap_id = payload["request"]["request_id"] + .as_u64() + .expect("the tap carries the id"); + assert_eq!( + *seen.lock().unwrap(), + Some(tap_id), + "the unit's span records the SAME id the response tap carries, as a native u64" + ); +} diff --git a/crates/busbar-plane-llm/tests/on_driver/speech_request.json b/crates/busbar-plane-llm/tests/on_driver/speech_request.json new file mode 100644 index 0000000000..5be3d38816 --- /dev/null +++ b/crates/busbar-plane-llm/tests/on_driver/speech_request.json @@ -0,0 +1,6 @@ +{ + "model": "gpt-4o-mini-tts", + "input": "SPEAK-THIS", + "voice": "alloy", + "instructions": "STYLE-INSTRUCTIONS" +} diff --git a/crates/busbar-plane-llm/tests/on_driver/wire.rs b/crates/busbar-plane-llm/tests/on_driver/wire.rs new file mode 100644 index 0000000000..4b52c3ade8 --- /dev/null +++ b/crates/busbar-plane-llm/tests/on_driver/wire.rs @@ -0,0 +1,66 @@ +// SPDX-License-Identifier: Apache-2.0 +// Copyright (C) 2026 Busbar Inc and contributors + +//! WHAT THE ROOT'S RIG SPEAKS TO THIS PLANE: the settings a case's pools become (one far end per +//! dialect, every member a model), each far end's answer, and the one plain request the rig's `fire` +//! posts (an Anthropic Messages request). The rig itself names no plane and no dialect; these are +//! this plane's words, so they live here. + +use serde_json::{json, Value}; + +use crate::rig::{Member, Pool}; + +/// The far end a member is served by unless its case says otherwise: the Anthropic one. +pub const DEFAULT_PROVIDER: &str = "ant"; + +/// The target of the rig's plain unit: Anthropic Messages. +pub const FIRE_TARGET: &str = "/v1/messages"; + +/// The request fields of the rig's plain unit. +pub const FIRE_FIELDS: &[(&str, &str)] = &[ + ("content-type", "application/json"), + ("anthropic-version", "2023-06-01"), +]; + +/// The plane's settings for these pools: one far end per protocol, every member a model. +pub fn settings(pools: &[&Pool]) -> Vec<u8> { + let mut models = serde_json::Map::new(); + let mut sections = serde_json::Map::new(); + for p in pools { + let names: Vec<Value> = p.members.iter().map(|m| json!(m.model)).collect(); + sections.insert(p.name.to_string(), json!({ "members": names })); + for m in &p.members { + models.insert(m.model.to_string(), json!({ "provider": m.provider })); + } + } + serde_json::to_vec(&json!({ + "providers": { + "ant": { "protocol": "anthropic", "base_url": "https://anthropic.example" }, + "oai": { "protocol": "openai", "base_url": "https://openai.example" }, + "brk": { "protocol": "bedrock", "base_url": "https://bedrock.example" }, + "goo": { "protocol": "gemini", "base_url": "https://gemini.example" }, + "coh": { "protocol": "cohere", "base_url": "https://cohere.example" }, + }, + "models": models, + "pools": sections, + })) + .expect("settings serialize") +} + +/// What a member's far end answers one attempt, in its far end's dialect. +pub fn answer_of(m: &Member) -> Value { + match m.provider { + "oai" => json!({ + "id": "chatcmpl-1", "object": "chat.completion", "model": m.label, + "choices": [{"index": 0, "message": {"role": "assistant", "content": "hi"}, + "finish_reason": "stop"}], + "usage": {"prompt_tokens": 1, "completion_tokens": 1, "total_tokens": 2} + }), + _ => json!({ + "id": "msg_1", "type": "message", "role": "assistant", "model": m.label, + "content": [{"type": "text", "text": "hi"}], + "stop_reason": "end_turn", + "usage": {"input_tokens": 1, "output_tokens": 1} + }), + } +} diff --git a/crates/busbar-plane-llm/tests/on_driver/words.txt b/crates/busbar-plane-llm/tests/on_driver/words.txt new file mode 100644 index 0000000000..a660d5f5fb --- /dev/null +++ b/crates/busbar-plane-llm/tests/on_driver/words.txt @@ -0,0 +1,14 @@ +# THE LLM PLANE'S DOOR WORDS, read as data by the composition root's pools-door cells under the fold +# switch (crates/busbar src/root/tests/serve_hook_seats.rs `word`, and the door cells of +# serve_tests.rs): `key = value`, one per line, `#` a comment. The root names no plane and no dialect +# (ARCHITECT ruling, #464), so the dialects those cells speak are this plane's words, kept here. +# +# The dialect the cells' members and callers speak: the caller's chat completions, and each member's +# far end, unless a cell names another. +dialect = openai +# A far dialect that cannot carry a control the caller set (`logit_bias`), so a TRANSLATE attempt +# writes 1.5.5's `egress.control_unrepresentable` rows (`<control> on <dialect>`, +# `<path> from <dialect>`), and that dialect's answer: a message whose usage carries a member the +# caller's dialect has no form for. +untranslatable_dialect = anthropic +untranslatable_answer = {"id":"msg_1","type":"message","role":"assistant","model":"m0","content":[{"type":"text","text":"hi"}],"stop_reason":"end_turn","stop_sequence":null,"usage":{"input_tokens":1,"output_tokens":1,"output_tokens_details":{"reasoning_tokens":0}}} diff --git a/crates/busbar-plane-llm/tests/plane_door.rs b/crates/busbar-plane-llm/tests/plane_door.rs index 2feb5a2459..aa3b53e919 100644 --- a/crates/busbar-plane-llm/tests/plane_door.rs +++ b/crates/busbar-plane-llm/tests/plane_door.rs @@ -10,8 +10,8 @@ use busbar_contract::abi::plane::{TAIL_FALLBACK, TAIL_PROBES, UNITS_REPORTED}; use busbar_plane_llm::dialect::DIALECTS; use busbar_plane_llm::exchange::reply::Units; use busbar_plane_llm::plane_door::{ - claims, counts, read_settings, DENY_RESPONSE_HEADERS, EGRESS_SCHEMES, NEEDS, OPEN_CLASSES, - STATEMENT, TAIL, VERSION, + claims, counts, read_settings, route_of, DENY_RESPONSE_HEADERS, EGRESS_STYLES, NEEDS, + OPEN_CLASSES, STATEMENT, TAIL, VERSION, }; #[test] @@ -37,8 +37,8 @@ fn the_tail_is_one_the_contract_accepts() { assert_eq!(TAIL.dialects_len, DIALECTS.len()); assert_eq!(TAIL.op_classes_len, 7); assert_eq!( - TAIL.billable_classes_len, 19, - "tokens in, out, cache read, cache write, then the open classes" + TAIL.billable_classes_len, 20, + "tokens in, out, cache read, cache write, the open classes, then the per-request fee unit" ); // Every reported count's class (owner LEDGER-100), `search_units` first and in its own family. assert_eq!( @@ -80,24 +80,53 @@ fn the_tail_is_one_the_contract_accepts() { 4 + k ); } + // THE FEE UNIT (owner #77, money-B1): one per billable request, the last billable class. + assert_eq!(TAIL.fee_units_len, 1); + // SAFETY: the tail's `'static` fee-unit list of `fee_units_len` entries. + let fee = unsafe { *TAIL.fee_units }; + // SAFETY: a `'static` str the door states (abi_str). + let fee = unsafe { std::slice::from_raw_parts(fee.ptr, fee.len) }; + assert_eq!(fee, busbar_contract::plane::PER_REQUEST.as_bytes()); + assert_eq!( + busbar_plane_llm::plane_door::FEE_CLASS_INDEX as usize, + TAIL.billable_classes_len - 1 + ); } +/// ARCHITECT Q-L1-AUTH (A): every dialect states its default outbound style in the tail, and every +/// style a member may be bound under (each dialect's default, the provider `auth:` overrides) has +/// exactly one outbound need naming it. #[test] -fn every_dialects_egress_scheme_has_its_one_outbound_need() { +fn every_dialects_default_style_is_stated_and_every_style_has_its_one_outbound_need() { assert_eq!(check_needs(NEEDS), Ok(())); assert_eq!(STATEMENT.needs_len, NEEDS.len()); - for d in DIALECTS { + assert_eq!(TAIL.dialect_auth_len, DIALECTS.len()); + for (i, d) in DIALECTS.iter().enumerate() { + // SAFETY: the tail's `'static` dialect_auth list of `dialect_auth_len` entries. + let stated = unsafe { *TAIL.dialect_auth.add(i) }; + assert_eq!(stated.dialect as usize, i); + // SAFETY: a `'static` str the door states (abi_str). + let style = unsafe { std::slice::from_raw_parts(stated.style.ptr, stated.style.len) }; + assert_eq!(style, d.egress_style.as_bytes(), "{}", d.name); assert_eq!( - EGRESS_SCHEMES + EGRESS_STYLES .iter() - .filter(|s| **s == d.egress_scheme) + .filter(|s| **s == d.egress_style) .count(), 1, "{}", d.name ); } - assert_eq!(NEEDS.len(), EGRESS_SCHEMES.len()); + for style in [ + "api-key", + "bearer", + "jwt-bearer", + "oauth-client-credentials", + ] { + assert!(EGRESS_STYLES.contains(&style), "{style}"); + } + assert_eq!(NEEDS.len(), EGRESS_STYLES.len()); } #[test] @@ -295,6 +324,66 @@ fn red_no_dialects_governed_response_field_leaks_through_a_need() { } } +/// THE ROUTE AN ARRIVAL NAMES (ARCHITECT Q-SW6 / Q-FL3, 2026-10-02): its model, verbatim, as a pool +/// when the plane's `pools` names it, else as a direct entry, the previous release's order (a pool +/// first, then a configured model). A model that is neither is still named, as a direct entry the +/// kernel does not hold, so the kernel refuses it (`no_destination`, 1.5.5's 404). +#[test] +fn an_arrival_routes_over_its_model_a_pool_first_then_a_direct_entry() { + use busbar_contract::abi::plane::{ROUTE_DIRECT, ROUTE_POOL}; + let shaping = read_settings( + br#"{"providers":{"ant":{"protocol":"anthropic","base_url":"https://anthropic.example"}}, + "models":{"claude":{"provider":"ant"},"both":{"provider":"ant"}}, + "pools":{"p":{"members":["claude"]},"both":{"members":["claude"]}}}"#, + ) + .expect("a well-formed generation reads"); + assert_eq!(route_of(&shaping, "p"), (ROUTE_POOL, "p")); + assert_eq!(route_of(&shaping, "claude"), (ROUTE_DIRECT, "claude")); + assert_eq!(route_of(&shaping, "both"), (ROUTE_POOL, "both")); + assert_eq!(route_of(&shaping, "nope"), (ROUTE_DIRECT, "nope")); +} + +/// THE STICKY-ROUTING KEY (ARCHITECT Q1 ArriveOut, 2026-10-05), as 1.5.5 derived it: the pool's +/// `affinity.header_name` (else `x-session-id`), matched case-blind, wins over chat's non-empty body +/// `system`; a header value that is not text is no key; neither is no affinity. +#[test] +fn the_sticky_key_is_the_pools_header_else_the_chat_bodys_system() { + use busbar_plane_llm::plane_door::affinity_key; + let shaping = read_settings( + br#"{"providers":{"ant":{"protocol":"anthropic","base_url":"https://anthropic.example"}}, + "models":{"claude":{"provider":"ant"}}, + "pools":{"p":{"members":["claude"]}, + "u":{"members":["claude"],"affinity":{"mode":"session","header_name":"x-user-id"}}}}"#, + ) + .expect("a well-formed generation reads"); + assert_eq!(shaping.affinity_header("p"), "x-session-id"); + assert_eq!(shaping.affinity_header("u"), "x-user-id"); + let chat = busbar_plane_llm::codec::DECLS + .iter() + .find(|d| d.name == "anthropic") + .and_then(|d| d.handler) + .and_then(|h| h.operation_handler(busbar_contract::operation::OpVerb::CHAT)); + let body = serde_json::json!({"model": "p", "system": "be brief", "messages": []}); + let session: &[(&[u8], &[u8])] = &[(b"X-Session-Id", b"s-1")]; + assert_eq!( + affinity_key("x-session-id", session, chat, Some(&body)), + Some("s-1".to_string()) + ); + assert_eq!( + affinity_key("x-user-id", session, chat, Some(&body)), + Some("be brief".to_string()), + "another pool's header is not this pool's" + ); + let unreadable: &[(&[u8], &[u8])] = &[(b"x-session-id", b"s\x01")]; + assert_eq!( + affinity_key("x-session-id", unreadable, chat, Some(&body)), + Some("be brief".to_string()) + ); + let plain = serde_json::json!({"model": "p", "system": "", "messages": []}); + assert_eq!(affinity_key("x-session-id", &[], chat, Some(&plain)), None); + assert_eq!(affinity_key("x-session-id", &[], None, Some(&body)), None); +} + /// EVERY DECLARED OPEN CLASS REACHES THE DURABLE BOOK (owner LEDGER-100): one count of each open /// class becomes exactly one reported `UnitCount` at that class's index in the tail, so no reported /// count is dropped with the "class the plane does not state" WARN. diff --git a/crates/busbar-plane-mcp/src/door_tasks.rs b/crates/busbar-plane-mcp/src/door_tasks.rs index 516fe5023b..4237233b8e 100644 --- a/crates/busbar-plane-mcp/src/door_tasks.rs +++ b/crates/busbar-plane-mcp/src/door_tasks.rs @@ -1804,6 +1804,16 @@ fn resolve( return Resolved::Task(Box::new(task.clone())); } } + // A TERMINAL STATE THIS INSTANCE HOLDS AND STILL OWES THE HOST (a continuation the kernel + // refused, its settle owed to the next create's sweep) is newer than the live row the host + // keeps until then: the instance's own word answers, as the served engine's runner answered a + // call it could not carry at once. The host's live row would otherwise overwrite it. + if let Some(task) = held + .as_ref() + .filter(|t| t.unsettled && t.status().is_terminal()) + { + return Resolved::Task(Box::new(task.clone())); + } // A host that lists none of the plane's records: what the instance holds answers. let unlisted = |held: Option<Task>| held.map_or(Resolved::Unknown, |t| Resolved::Task(Box::new(t))); diff --git a/crates/busbar-plane-mcp/src/tool_door.rs b/crates/busbar-plane-mcp/src/tool_door.rs index baa5905b69..7bc9873c83 100644 --- a/crates/busbar-plane-mcp/src/tool_door.rs +++ b/crates/busbar-plane-mcp/src/tool_door.rs @@ -463,6 +463,10 @@ slot!( /// The most units the instance keeps state for at once; past it, a new arrival is refused. pub const MAX_UNITS: usize = 4096; +/// The open breaker's reason as the kernel spells it (`RefusalCode::BreakerOpen`'s word): the words +/// this door says a walk's exhaustion in where it repeats them, as predev's door said it. +const BREAKER_OPEN_WORD: &str = "breaker_open"; + /// The most upstreams busbar remembers a negotiated revision for, and their bytes: forgetting one /// costs one renegotiation. const UPSTREAMS_KEPT: usize = 1024; @@ -3291,7 +3295,17 @@ slot!( _ => None, }) }); - let text = input.field(|i| &i.text).bytes(); + // THE WORDS the refusal is said in: the kernel's text, but for the walk's exhaustion (an open + // breaker, a walk out of members or out of time) the reason's own word, as predev's door said + // it, wherever this door repeats the words (its generic error, a task's failure): the + // kernel's text there is the walk's own sentence, which predev never handed this door. + let text: &[u8] = if given.cause != REFUSAL_ARRIVE + && given.reason == busbar_contract::abi::plane::RefusalCode::BreakerOpen.code() + { + BREAKER_OPEN_WORD.as_bytes() + } else { + input.field(|i| &i.text).bytes() + }; // The tool a refused `tools/call` named, where its arrival read one. let called = instance.get().and_then(|plane| { plane.units.with(&given.unit, |unit| { diff --git a/crates/busbar/Cargo.toml b/crates/busbar/Cargo.toml index d9ed4e298c..a691ba2340 100644 --- a/crates/busbar/Cargo.toml +++ b/crates/busbar/Cargo.toml @@ -275,7 +275,7 @@ plane-decisions = ["dep:busbar-plane-decisions"] # loader's one load beside every dropped-in plane door, the same table as its dropped-in cdylib # (#2, #30; THE DESIGN section 11.4). It carries `proto-llm`, whose row stays linked and answers # every arrival the serve path does not hand the door. -llm-on-driver = ["proto-llm", "dep:busbar-plane-llm"] +llm-on-driver = ["proto-llm", "dep:busbar-plane-llm", "auth-sigv4", "auth-oauth"] # THE MCP PLANE'S DOOR ROW, carried by `plane-mcp` and never set on its own (the mcp fold's switch # is gone: the plane is flipped onto the driver, BUSBAR-1.6.0.md Part 3 section 12). The plane's # memory-ABI door (`busbar_plane_mcp::plane_door::door`) is a `plane-door` row, bound through the @@ -486,6 +486,16 @@ auth-admin-tokens = "busbar-auth-admin-tokens" plane-a2a = "busbar-a2a" plane-streaming = "busbar-voice" +# THE DRIVER PROOFS of a development-only fold switch (a `<plane>-on-driver` feature, BUSBAR-1.6.0.md +# Part 3 section 12 "The switch"): cargo feature -> the plane crate whose `tests/on_driver/` holds the +# plane-origin hook cases (`mod.rs`) and the words the root's door cells read as data (`words.txt`). +# Test builds only: `build.rs` emits `$OUT_DIR/driver_proofs.rs` (the `#[path]` that +# `tests/hook_parity_driver` compiles beside its rig) and `$OUT_DIR/driver_words.rs` +# (`DRIVER_WORDS`), so the composition root's sources name neither the plane nor its dialects: the +# cases are written in the plane's own words and live in the plane's crate (ARCHITECT ruling, #464). +[package.metadata.busbar.driver-proofs] +llm-on-driver = "busbar-plane-llm" + [package.metadata.busbar.root-units] # The node's own row, keyed to the node (not a feature): it compiles in every build, so its card # repricer and book binding run in every build (ARCHITECT Q1). diff --git a/crates/busbar/build.rs b/crates/busbar/build.rs index c6575cbd2d..af389df585 100644 --- a/crates/busbar/build.rs +++ b/crates/busbar/build.rs @@ -182,6 +182,34 @@ fn main() { std::path::PathBuf::from(env::var("OUT_DIR").expect("OUT_DIR")).join("linked_planes.rs"); std::fs::write(out, linked_planes_source(&manifest, &enabled)).expect("write linked_planes.rs"); + // THE LINKED PLANE DOORS FOR THE INTEGRATION TESTS (tests only include it): each enabled linked + // row on the `plane-door` axis, its memory-ABI `door` resolved as the root resolves its entry, + // so a test finds a door by what its Statement states without naming a plane crate. + let out = std::path::PathBuf::from(env::var("OUT_DIR").expect("OUT_DIR")) + .join("linked_plane_doors.rs"); + std::fs::write(out, linked_plane_doors_source(&manifest, &enabled)) + .expect("write linked_plane_doors.rs"); + // `linked_fold_on_driver`: a development-only fold switch is on (a `<plane>-on-driver` feature, + // BUSBAR-1.6.0.md Part 3 section 12 "The switch"), so its proofs on the driver compile without a + // test spelling the plane the switch carries. + println!("cargo::rustc-check-cfg=cfg(linked_fold_on_driver)"); + if declared_features(&manifest) + .iter() + .any(|f| f.ends_with("-on-driver") && enabled(f)) + { + println!("cargo::rustc-cfg=linked_fold_on_driver"); + } + + // THE DRIVER PROOFS OF THE FOLD SWITCH (tests only include them): the enabled + // `[package.metadata.busbar.driver-proofs]` row's plane-owned cases (`mod proofs`, which the + // `hook_parity_driver` suite compiles beside its rig) and the words its door cells read as data + // (`DRIVER_WORDS`), both at the row's crate's own `tests/on_driver/`, so neither the suite nor + // the root's door cells spell the plane under proof or its dialects. + let (proofs, words) = driver_proofs_source(&manifest, &enabled); + let out_dir = std::path::PathBuf::from(env::var("OUT_DIR").expect("OUT_DIR")); + std::fs::write(out_dir.join("driver_proofs.rs"), proofs).expect("write driver_proofs.rs"); + std::fs::write(out_dir.join("driver_words.rs"), words).expect("write driver_words.rs"); + // THE LINKED WIRES FOR THE INTEGRATION TESTS (tests only include it): each linked transport row's // entry (`KEY`, `COMPOSES_OVER`, `SESSION`, `build`), the rows this build leaves unlinked, and the same // bottom-up fold the boot seal runs — so a test addresses a wire by the key its row declares and @@ -357,6 +385,55 @@ fn node_plane_source(manifest: &str, enabled: &dyn Fn(&str) -> bool) -> String { out } +/// `(proofs, words)` for the one enabled `[package.metadata.busbar.driver-proofs]` row (cargo feature +/// -> the plane crate whose fold switch it is): `proofs` is `pub(crate) mod proofs;` at that crate's +/// `tests/on_driver/mod.rs`, `words` is `DRIVER_WORDS`, that directory's `words.txt`. The crate's +/// directory is its `[dependencies]` row's `path`. No row enabled: no module and empty words; more +/// than one: a compile error in the one suite that includes them, since a build proves one switch. +fn driver_proofs_source(manifest: &str, enabled: &dyn Fn(&str) -> bool) -> (String, String) { + let head = + "// @generated by build.rs from Cargo.toml `[package.metadata.busbar.driver-proofs]`.\n"; + let rows: Vec<(String, String)> = + metadata_map(manifest, "package.metadata.busbar.driver-proofs") + .into_iter() + .filter(|(feature, _)| enabled(feature)) + .collect(); + let mut proofs = String::from(head); + let mut words = String::from(head); + match rows.as_slice() { + [] => words.push_str("#[allow(dead_code)]\npub(crate) const DRIVER_WORDS: &str = \"\";\n"), + [(feature, krate)] => { + let dependency = metadata_value(manifest, "dependencies", krate).unwrap_or_else(|| { + panic!("driver-proofs row `{feature}`: `{krate}` is no dependency") + }); + let path = dependency + .split_once("path") + .and_then(|(_, rest)| rest.split('"').nth(1)) + .unwrap_or_else(|| { + panic!("driver-proofs row `{feature}`: `{krate}` has no `path`") + }); + let at = std::path::Path::new(&env::var("CARGO_MANIFEST_DIR").expect("manifest dir")) + .join(path) + .join("tests/on_driver"); + proofs.push_str(&format!( + "#[path = {:?}]\npub(crate) mod proofs;\n", + at.join("mod.rs").display().to_string() + )); + words.push_str(&format!( + "#[allow(dead_code)]\npub(crate) const DRIVER_WORDS: &str = include_str!({:?});\n", + at.join("words.txt").display().to_string() + )); + } + _ => { + proofs.push_str( + "compile_error!(\"more than one driver-proofs row is enabled; a build proves one fold switch\");\n", + ); + words.push_str("#[allow(dead_code)]\npub(crate) const DRIVER_WORDS: &str = \"\";\n"); + } + } + (proofs, words) +} + /// `static SERVED_WITNESSES` over `<crate>::testkit::SERVED` for every /// `[package.metadata.busbar.served-witness]` row whose cargo feature is enabled, in manifest order. /// A build with the feature off has no row, exactly as it has no served leg for that plane. @@ -439,6 +516,36 @@ fn linked_planes_source(manifest: &str, enabled: &dyn Fn(&str) -> bool) -> Strin out } +/// `static LINKED_PLANE_DOORS`: every enabled `[package.metadata.busbar.linked]` row carrying the +/// `plane-door` axis, its door at its entry (`linked-entry` by row, then by crate, else +/// `<crate>::linked`), in manifest order. An entry inside the root itself is no test's to reach. +fn linked_plane_doors_source(manifest: &str, enabled: &dyn Fn(&str) -> bool) -> String { + let axes = metadata_map(manifest, "package.metadata.busbar.linked-axes"); + let entries = metadata_map(manifest, "package.metadata.busbar.linked-entry"); + let mut out = String::from( + "// @generated by build.rs from Cargo.toml `[package.metadata.busbar.linked]`.\n\ + pub(crate) static LINKED_PLANE_DOORS: &[::busbar_contract::abi::mechanism::door::DoorFn] = &[\n", + ); + for (feature, krate) in metadata_map(manifest, "package.metadata.busbar.linked") { + let is_door = axes + .iter() + .any(|(f, a)| *f == feature && a.split_whitespace().any(|x| x == "plane-door")); + if !(enabled(&feature) && is_door) { + continue; + } + let entry = entries + .iter() + .find(|(k, _)| *k == feature) + .or_else(|| entries.iter().find(|(k, _)| *k == krate)) + .map_or_else(|| format!("{}::linked", ident(&krate)), |(_, e)| e.clone()); + if !entry.starts_with("crate::") { + out.push_str(&format!(" ::{entry}::door,\n")); + } + } + out.push_str("];\n"); + out +} + /// `(source, off)`: `static LINKED_TRANSPORTS` over every `[package.metadata.busbar.linked]` row /// carrying the `transport` axis that this build links (its feature on, or its crate a required /// dependency — the rule `linked_source` links by), each row's entry resolved as the root resolves it diff --git a/crates/busbar/src/main.rs b/crates/busbar/src/main.rs index 9b724954b1..f2957d59f2 100644 --- a/crates/busbar/src/main.rs +++ b/crates/busbar/src/main.rs @@ -793,6 +793,12 @@ async fn run(data_workers: usize, late_services: std::sync::Arc<root::serve::Lat &cfg.providers, &config_validate::MetadataPosture::global(&cfg), ); + // The kernel-owned sections a door plane's Statement may name beside its own (spec Part 1 §4), + // as the configuration resolved them, captured before `cfg` is consumed. + let door_kernel_sections = root::door_steps::kernel_sections(&cfg); + // The model-serving pools the plane serving the `pools` map walks, with every bound the + // configuration states (`root::model_egress`). + let door_model_pools = root::model_egress::ModelPools::of(&cfg); // The unified `pools:` a named-definition carrier's members resolved to, by the carrier's // section key: each door plane's section carries its own pools (DoorPools), captured before // `cfg` is consumed. @@ -802,6 +808,10 @@ async fn run(data_workers: usize, late_services: std::sync::Arc<root::serve::Lat #[cfg(feature = "root-admin")] let breaker_policy = root::adapters::BreakerPolicy::from_pools(&cfg.pools); + // What the door planes are sealed over, projected from every configuration as it is built + // (`root::door_steps::door_config`), so a config apply refreshes them onto the new generation. + busbar_kernel::appbuild::set_config_projection(root::door_steps::door_config); + // The configured store's name, captured before `cfg` is consumed: the boot book's journal is // kept in that store, and a store that cannot keep it is refused by name. let store_module = cfg @@ -881,31 +891,62 @@ async fn run(data_workers: usize, late_services: std::sync::Arc<root::serve::Lat // deployment's providers, the auth plugins that serve its style (the build's own rows, then // the plugins directory's) and the one connector their needs were declared on. let door_auths = root::door_steps::process_auths(); + // The kernel's own App (its lane store, its telemetry bank), through its swap handle once it exists (a config apply + // rebuilds the store), the boot App's until then. + let door_live_handle: std::sync::Arc< + std::sync::OnceLock<std::sync::Arc<busbar_kernel::state::AppHandle>>, + > = std::sync::Arc::default(); + let door_models = root::model_egress::ModelServing { + pools: door_model_pools, + lanes: app + .engine_tables_view() + .model_indices() + .into_iter() + .map(|(model, lane)| (model.to_string(), lane)) + .collect(), + app: { + let (live, boot) = ( + std::sync::Arc::clone(&door_live_handle), + std::sync::Arc::clone(&app), + ); + std::sync::Arc::new(move || { + live.get() + .map_or_else(|| std::sync::Arc::clone(&boot), |h| h.load()) + }) + }, + }; let door_reach = root::door_steps::DoorReach { providers: &door_providers, secrets: &*app.secret_resolver, - auths: door_auths, + auths: std::sync::Arc::clone(&door_auths), conns: Arc::clone(root::connector::the()) as Arc<dyn busbar_contract::conn::PollConns>, stream_ceiling_secs: busbar_kernel::config::limits::installed().map_or( busbar_kernel::config::limits::DEFAULT_UPSTREAM_REQUEST_TIMEOUT_SECS, |l| l.upstream_request_timeout_secs, ), + models: Some(&door_models), upgrades: root::serve::upgrade_carriers(LINKED.transports), }; - // The kernel's own App through its swap handle once it exists (a config apply replaces the - // generation a unit's hooks are read off), the boot App's until then. - let door_live_handle: std::sync::Arc< - std::sync::OnceLock<std::sync::Arc<busbar_kernel::state::AppHandle>>, - > = std::sync::Arc::default(); let served = root::serve::compose_served( app.governance.clone(), root::boot::door_planes(), &root::dispatch::dispatcher(), &late_services, - &root::serve::with_pools(deploy.door_sections(), &door_pools), + &root::serve::with_pools( + { + let mut sections = deploy.door_sections(); + for (key, value) in door_kernel_sections { + sections.entry(key).or_insert(value); + } + sections + }, + &door_pools, + ), deploy.public_url.as_deref(), &door_reach, Some({ + // The hooks read the CURRENT generation's host: the App's swap handle once it exists, + // the boot App's until then. let (live, boot) = ( std::sync::Arc::clone(&door_live_handle), std::sync::Arc::clone(&app), @@ -946,7 +987,15 @@ async fn run(data_workers: usize, late_services: std::sync::Arc<root::serve::Lat // first config swap, retiring these boot probers (their `Weak` fails to upgrade) exactly as the old // `Weak<App>` did when the boot snapshot drained. // Built only when some linked entry re-anchors work on it: a build with none holds no host. - let on_host = LINKED.on_host; + // While a plane's fold switch is on, the plane serving the `pools` map runs its health probes + // through its door as the K7 probe unit (`root::serve`'s probe target over the plane's sealed + // egress), so the legacy row's probers are not spawned beside them: one prober per lane, as + // 1.5.5 ran. + let on_host = if cfg!(linked_fold_on_driver) { + &[][..] + } else { + LINKED.on_host + }; let boot_host = (!on_host.is_empty()).then(|| busbar_kernel::plane_host::engine_host(&app)); if let Some(host) = &boot_host { for spawn in on_host { @@ -985,6 +1034,8 @@ async fn run(data_workers: usize, late_services: std::sync::Arc<root::serve::Lat // Every config apply refreshes each served door plane onto the generation it installed // (ARCHITECT Q-DEL-A2A-APPLY), bound on the handle once the routers are built. let door_appliers = served.appliers(); + #[cfg(linked_axis_node)] + let door_post = served.post.clone(); // THE LINE CARRIER (SEAM-S1, `root::serve::lines`): with the stdio serve mode asked for and a // served plane claiming the stdio transport, the served planes are kept for the process's own // stdin/stdout, and the data router is built with none of them (it is never bound in this mode). @@ -1010,7 +1061,69 @@ async fn run(data_workers: usize, late_services: std::sync::Arc<root::serve::Lat ); credential_handle.set(std::sync::Arc::clone(&app_handle)); let _ = door_live_handle.set(std::sync::Arc::clone(&app_handle)); - app_handle.on_apply(Box::new(move |app| door_appliers.apply(app))); + app_handle.on_apply(Box::new({ + let door_appliers = door_appliers.clone(); + move |app| door_appliers.apply(app) + })); + // A CONFIG APPLY REFRESHES THE PLANE SERVING THE `pools` MAP onto the new generation + // (`DoorAppliers::refresh_models`): the swapped-in App's projected configuration, its lane + // table and its secret seam. A build with no node composes no door plane's money, so it has + // none to refresh. + #[cfg(linked_axis_node)] + if let Some(post) = door_post { + let (auths, live) = ( + std::sync::Arc::clone(&door_auths), + std::sync::Arc::clone(&door_live_handle), + ); + let upgrades = root::serve::upgrade_carriers(LINKED.transports); + app_handle.on_apply(Box::new(move |next| { + let Some(cfg) = next + .config_projection + .as_ref() + .and_then(|p| p.downcast_ref::<root::door_steps::DoorConfig>()) + else { + return; + }; + let models = root::model_egress::ModelServing { + pools: cfg.model_pools.clone(), + lanes: next + .engine_tables_view() + .model_indices() + .into_iter() + .map(|(model, lane)| (model.to_string(), lane)) + .collect(), + app: { + let (live, fallback) = + (std::sync::Arc::clone(&live), std::sync::Arc::clone(next)); + std::sync::Arc::new(move || { + live.get() + .map_or_else(|| std::sync::Arc::clone(&fallback), |h| h.load()) + }) + }, + }; + let reach = root::door_steps::DoorReach { + providers: &cfg.providers, + secrets: &*next.secret_resolver, + auths: std::sync::Arc::clone(&auths), + conns: Arc::clone(root::connector::the()) + as Arc<dyn busbar_contract::conn::PollConns>, + stream_ceiling_secs: busbar_kernel::config::limits::installed().map_or( + busbar_kernel::config::limits::DEFAULT_UPSTREAM_REQUEST_TIMEOUT_SECS, + |l| l.upstream_request_timeout_secs, + ), + models: Some(&models), + upgrades: upgrades.clone(), + }; + door_appliers.refresh_models( + &cfg.sections, + &root::serve::DoorEgress { + reach: &reach, + journal: std::sync::Arc::clone(&post) + as std::sync::Arc<dyn busbar_kernel_egress::ports::Journal>, + }, + ); + })); + } // Every opened plugin instance's log sink follows the applied `plugins.logs` (THE DESIGN §11.2). app_handle.on_apply(Box::new(|app| { root::boot::follow_plugin_logs(root::boot::plugin_logs(), &app.plugins_cfg); diff --git a/crates/busbar/src/root/door_steps.rs b/crates/busbar/src/root/door_steps.rs index fb50372f56..17f1cd61b5 100644 --- a/crates/busbar/src/root/door_steps.rs +++ b/crates/busbar/src/root/door_steps.rs @@ -309,8 +309,13 @@ fn members(pool: &serde_yaml::Value) -> Vec<(String, u32)> { /// THE KEY A DOOR PLANE'S ENTRY IS PRICED, METERED AND ROUTED UNDER (ARCHITECT Q-FL3: "keys the /// kernel's state by (plane key, model entry)"; #42/#47 one card per plane): `"<plane>\u{1f}<entry>"`. +/// The plane that owns the flat card (the empty plane key: 1.5.5's `rate_card:` and its lanes, +/// `busbar_kernel_ledger::cost::split_plane_lane`) keys its entries bare, as 1.5.5 did. #[must_use] pub fn plane_lane(plane: &str, entry: &str) -> String { + if plane.is_empty() { + return entry.to_string(); + } format!("{plane}{PLANE_LANE_SEP}{entry}") } @@ -328,6 +333,39 @@ pub fn egress_pool(plane: &str, routed: &Routed) -> String { } } +/// The pools and key the destination guard reads, for the plane serving the `pools` map. +struct DoorPoolView<'a> { + pools: &'a DoorPools, + key: Option<&'a VirtualKey>, + app: &'a App, +} + +impl busbar_kernel::door::PoolView for DoorPoolView<'_> { + fn has_key(&self) -> bool { + self.key.is_some() + } + + fn key_is_scoped(&self) -> bool { + self.key.is_some_and(|k| k.allowed_scopes.is_some()) + } + + fn pool_allowed(&self, pool: &str) -> bool { + self.key.is_none_or(|k| k.scope_allowed("pool", pool)) + } + + fn on_exhausted_fallback(&self, pool: &str) -> Option<String> { + self.pools.fallback(pool).map(str::to_owned) + } + + fn is_configured(&self, name: &str) -> bool { + self.pools.pools().contains_key(name) || self.pools.entries().iter().any(|e| e == name) + } + + fn is_unpriced(&self, name: &str) -> bool { + self.app.cost.model_unpriced(name) + } +} + // ── admission ──────────────────────────────────────────────────────────────────────────────────── /// HOW A UNIT IS ADMITTED (ARCHITECT P3 (a), 2026-10-02). @@ -419,6 +457,10 @@ pub struct DoorFacts { /// Each need's response-head rule, in Statement need order: what of a far end's head reaches /// the plane on that need. pub keeps: Vec<busbar_kernel::plane_driver::ResponseKeep>, + /// The plane's dialect names, by the index `arrive` answers, and each provider's dialect: an + /// attempt whose far end speaks another dialect than the caller's is counted as a translation + /// (`busbar_translations_total`), as the previous release counted it. Empty = none counted. + pub translations: (Arc<[String]>, Arc<BTreeMap<String, String>>), /// The plane's declared breaker fact (`declares.breaker`, ARCHITECT Q4): whether a transient /// failure below the trip threshold benches a member's cell. `None`: it declares none, and /// every cell keeps the host's default. @@ -442,11 +484,16 @@ struct DoorUnit { counted: bool, /// Its caller asked for the answer streamed (`ROUTE_STREAM`). stream: bool, + /// The walk's affinity position its `arrive`'s sticky key names. + affinity: Option<u64>, routed: Option<Routed>, /// The governance book's grant: its in-flight holds, released when the unit's steps drop. grant: Option<AdmitGrant>, /// Whether the unit's money facts were opened (it was charged). charged: bool, + /// The words of the refusal one of its steps raised, where it has its own (its Retry-After + /// seconds; its message). + refused: (Option<u32>, Option<String>), /// The key the unit's record was written under on the host's unit records, once authenticate /// passed; it is struck when these steps drop. recorded: Option<u64>, @@ -573,6 +620,12 @@ impl<'s> DoorSteps<'s> { self.lock().stream } + /// The walk's affinity position the unit's `arrive` named, where it stated a sticky key. + #[must_use] + pub fn affinity(&self) -> Option<u64> { + self.lock().affinity + } + /// The key of the plane the unit is of. #[must_use] pub fn plane(&self) -> &str { @@ -626,7 +679,15 @@ impl<'s> DoorSteps<'s> { let units = self.expected_units(&expected); let grant = gov .try_admit_estimated(&self.app.cost, key, &pool, self.arrived, &models, &units) - .map_err(|blocked| refusal_for(&blocked))?; + .map_err(|blocked| { + // The plane serving the `pools` map answers a blocked admission in the previous + // release's words (`busbar_kernel::ingress::limit_refusal`). + if self.facts.plane.is_empty() { + self.lock().refused.1 = + Some(busbar_kernel::ingress::limit_refusal("", &blocked).2); + } + refusal_for(&blocked) + })?; money.open( ctx.key, UnitMoney { @@ -639,6 +700,9 @@ impl<'s> DoorSteps<'s> { classes: Arc::clone(&self.facts.classes), arrived: self.arrived, mode: exhaustion_of(&self.app, key), + // THE PLANE'S REPORT IS THE ONE FEE DECIDER (owner #77, money-B1): every plane's + // fee is its reported fee units (#47); the plane serving the `pools` map reports + // its per-request unit exactly where 1.5.5 kept its flat fee. fee: FeeRefund::PlaneFeeUnits(Arc::clone(&self.facts.fee_units)), charge: grant.charge().clone(), }, @@ -684,6 +748,19 @@ impl DriverSteps for DoorSteps<'_> { self.lock().expected = units.to_vec(); } + fn refused_words(&self) -> (Option<u32>, Option<String>) { + self.lock().refused.clone() + } + + fn attempting(&self, _ctx: &UnitCtx, dialect: u32, provider: &str) { + let (names, providers) = &self.facts.translations; + if let (Some(from), Some(to)) = (names.get(dialect as usize), providers.get(provider)) { + if from != to { + busbar_kernel::telemetry::translation(from, to); + } + } + } + fn refusal_words(&self, reason: ReasonCode) -> Option<Vec<u8>> { // A unit routed by scope that no one entry reached: the entries that reached it (none, or // several), so the plane words which (abi/plane `ROUTE_SCOPE`). @@ -702,6 +779,10 @@ impl DriverSteps for DoorSteps<'_> { u.counted = flags & busbar_contract::abi::plane::ROUTE_COUNTED != 0; u.stream = flags & busbar_contract::abi::plane::ROUTE_STREAM != 0; } + + fn affinity(&self, _ctx: &UnitCtx, hash: u64) { + self.lock().affinity = Some(hash); + } } impl Units for DoorSteps<'_> { @@ -800,6 +881,26 @@ impl Units for DoorSteps<'_> { // An unknown route seals nothing and is refused at admission, after its grant was judged // (1.5.5's order); the empty set is an answer at this step. let named = self.lock().named.clone(); + // THE PLANE SERVING THE `pools` MAP judges the route as named with the previous release's + // destination guard (`busbar_kernel::door::destination_guard`): the key's pool grant, its + // fallback pools' grants, then a rate card's coverage of the name, refused in 1.5.5's words. + if self.facts.plane.is_empty() { + if let Some(name) = named + .as_ref() + .and_then(|(_, n)| n.as_deref()) + .and_then(|n| std::str::from_utf8(n).ok()) + { + let view = DoorPoolView { + pools: self.pools, + key: self.key.as_deref(), + app: &self.app, + }; + if let Err(refusal) = busbar_kernel::door::destination_guard(&view, name) { + self.lock().refused.1 = Some(refusal.message()); + return SeatVerdict::refuse(token, Refusal::new(refusal.reason())); + } + } + } let routed = named.and_then(|(class, n)| self.pools.resolve(class, n.as_deref())); let sealed: Vec<VerifiedDestination> = routed .iter() @@ -879,6 +980,7 @@ impl Units for DoorSteps<'_> { Admission::Keyed if !local || counted || estimated => { if let Some(key) = self.key.clone() { if let Err(refusal) = self.charge(ctx, &key) { + self.lock().refused.0 = refusal.retry_after_secs(); return SeatVerdict::refuse(token, refusal); } } @@ -1022,6 +1124,7 @@ pub fn door_facts( .collect(), audit_kind: OpClassId::new(audit_kind), keeps, + translations: (Arc::from(Vec::new()), Arc::default()), bench_below_trip_threshold: None, stream_ceiling_secs: 0, } @@ -1152,6 +1255,206 @@ pub fn compose_egress( }) } +// ── one configuration generation, as the door planes are sealed over it ───────────────────────── + +/// WHAT A DOOR PLANE IS SEALED OVER, for ONE configuration generation (spec Part 1 line 569: one +/// validated `{section: value}` object per `open`/`refresh`): the kernel-owned sections it reads +/// ([`kernel_sections`]), the providers its members are reached through ([`provider_routes`]) and +/// the model-serving pools' bounds ([`crate::root::model_egress::ModelPools`]). Projected from each +/// resolved configuration as it is built (`busbar_kernel::appbuild::set_config_projection`), and +/// read back when that generation is swapped in. +#[derive(Debug)] +pub struct DoorConfig { + /// The kernel-owned sections. + pub sections: BTreeMap<&'static str, serde_yaml::Value>, + /// The providers, by name. + pub providers: BTreeMap<String, ProviderRoute>, + /// The model-serving pools' bounds. + pub model_pools: crate::root::model_egress::ModelPools, +} + +/// The [`DoorConfig`] of `cfg` (the root's `ConfigProjection`). +#[must_use] +pub fn door_config(cfg: &busbar_kernel::config::RootCfg) -> Arc<dyn std::any::Any + Send + Sync> { + Arc::new(DoorConfig { + sections: kernel_sections(cfg), + providers: provider_routes( + &cfg.providers, + &busbar_kernel::config_validate::MetadataPosture::global(cfg), + ), + model_pools: crate::root::model_egress::ModelPools::of(cfg), + }) +} + +// ── the kernel-owned sections a door plane reads ──────────────────────────────────────────────── + +/// THE KERNEL-OWNED SECTIONS A DOOR PLANE MAY READ, as the configuration validated and resolved +/// them (spec Part 1 §4: "A plane's settings reach it as ONE validated JSON object `{section: +/// value}` per open/refresh, reserved keys and secrets stripped"; "A plane receives, at open, the +/// dialect fields of the providers it references and resolves model -> dialect itself"): `providers` +/// (catalog-merged, each provider's dialect fields: no `base_url`, no credential reference, no +/// health or token endpoint), `models`, `pools` (each pool's members and its fallback pool) and +/// `limits` (the output-cap default and the effort budgets). The previous release kept these in the +/// kernel; they are the root's to hand a door plane whose Statement names them. `pools` is present +/// whenever `models` is: a deployment with models and no pools routes to its models directly. +#[must_use] +pub fn kernel_sections( + cfg: &busbar_kernel::config::RootCfg, +) -> BTreeMap<&'static str, serde_yaml::Value> { + use serde_json::{json, Map, Value}; + fn opt(m: &mut Map<String, Value>, key: &str, v: Option<Value>) { + if let Some(v) = v { + m.insert(key.to_string(), v); + } + } + fn output_key(k: busbar_kernel::ir::lane_caps::MaxOutputKeyCfg) -> &'static str { + use busbar_kernel::ir::lane_caps::MaxOutputKeyCfg; + match k { + MaxOutputKeyCfg::MaxTokens => "max_tokens", + MaxOutputKeyCfg::MaxCompletionTokens => "max_completion_tokens", + } + } + let mut out = BTreeMap::new(); + let mut providers = Map::new(); + for (name, p) in &cfg.providers { + let mut m = Map::new(); + m.insert("protocol".into(), json!(p.protocol)); + opt(&mut m, "path", p.path.as_ref().map(|v| json!(v))); + opt(&mut m, "path_base", p.path_base.as_ref().map(|v| json!(v))); + opt( + &mut m, + "organization", + p.organization.as_ref().map(|v| json!(v)), + ); + opt(&mut m, "project", p.project.as_ref().map(|v| json!(v))); + m.insert("error_map".into(), json!(p.error_map)); + opt(&mut m, "auth", p.auth.map(style_word).map(|v| json!(v))); + opt( + &mut m, + "max_output_key", + p.max_output_key.map(output_key).map(|v| json!(v)), + ); + opt( + &mut m, + "anthropic_adaptive_thinking", + p.anthropic_adaptive_thinking.map(|v| json!(v)), + ); + opt( + &mut m, + "native_structured_output", + p.native_structured_output.map(|v| json!(v)), + ); + let rules: Vec<Value> = p + .model_capabilities + .iter() + .map(|r| { + let mut c = Map::new(); + c.insert("models".into(), json!(r.models)); + opt( + &mut c, + "max_output_key", + r.max_output_key.map(output_key).map(|v| json!(v)), + ); + opt( + &mut c, + "anthropic_adaptive_thinking", + r.anthropic_adaptive_thinking.map(|v| json!(v)), + ); + opt( + &mut c, + "native_structured_output", + r.native_structured_output.map(|v| json!(v)), + ); + opt(&mut c, "reasoning_none", r.reasoning_none.map(|v| json!(v))); + opt( + &mut c, + "thinking_always_on", + r.thinking_always_on.map(|v| json!(v)), + ); + Value::Object(c) + }) + .collect(); + if !rules.is_empty() { + m.insert("model_capabilities".into(), Value::Array(rules)); + } + providers.insert(name.clone(), Value::Object(m)); + } + let mut models = Map::new(); + for (name, model) in &cfg.models { + let mut m = Map::new(); + m.insert(MODEL_PROVIDER_KEY.into(), json!(model.provider)); + opt( + &mut m, + "upstream_model", + model.upstream_model.as_ref().map(|v| json!(v)), + ); + opt( + &mut m, + "default_max_tokens", + model.default_max_tokens.map(|v| json!(v)), + ); + opt(&mut m, "reasoning", model.reasoning.map(|v| json!(v))); + opt( + &mut m, + "prompt_caching", + model.prompt_caching.map(|v| json!(v)), + ); + models.insert(name.clone(), Value::Object(m)); + } + let mut pools = Map::new(); + for (name, pool) in &cfg.pools { + let members: Vec<Value> = pool + .members + .iter() + .map(|member| { + let mut m = Map::new(); + m.insert("model".into(), json!(member.model)); + opt(&mut m, "reasoning", member.reasoning.map(|v| json!(v))); + opt(&mut m, "context_max", member.context_max.map(|v| json!(v))); + Value::Object(m) + }) + .collect(); + let mut p = Map::new(); + p.insert(POOL_MEMBERS_KEY.into(), Value::Array(members)); + if let Some(busbar_kernel::config::pools::OnExhaustedCfg::FallbackPool(fallback)) = + &pool.on_exhausted + { + p.insert( + ON_EXHAUSTED_KEY.into(), + json!({ FALLBACK_POOL_KEY: fallback }), + ); + } + // The header a pool's session affinity reads (1.5.5's `affinity.header_name`): the plane + // reads the caller's key from it, and the kernel routes on that key (ARCHITECT Q1). + if let Some(header) = pool.affinity.as_ref().and_then(|a| a.header_name.as_ref()) { + p.insert("affinity".into(), json!({ "header_name": header })); + } + pools.insert(name.clone(), Value::Object(p)); + } + let limits = json!({ + "default_max_tokens": cfg.limits.default_max_tokens, + "reasoning_effort_budgets": { + "minimal": cfg.limits.reasoning_effort_budgets.minimal, + "low": cfg.limits.reasoning_effort_budgets.low, + "medium": cfg.limits.reasoning_effort_budgets.medium, + "high": cfg.limits.reasoning_effort_budgets.high, + }, + }); + let yaml = |v: Value| serde_yaml::to_value(v).unwrap_or(serde_yaml::Value::Null); + if !providers.is_empty() { + out.insert( + busbar_contract::plugin::Kind::Transport.root(), + yaml(Value::Object(providers)), + ); + } + if !models.is_empty() || !pools.is_empty() { + out.insert(RESERVED_MODELS_KEY, yaml(Value::Object(models))); + out.insert(RESERVED_POOLS_KEY, yaml(Value::Object(pools))); + } + out.insert("limits", yaml(limits)); + out +} + // ── the members' routes (THE DESIGN §6 steps 2-3) ────────────────────────────────────────────── /// ONE `providers:` ENTRY as a door plane's member reaches it (THE DESIGN §6 step 2; #50, #51): the @@ -1639,6 +1942,9 @@ pub struct DoorReach<'a> { pub conns: Arc<dyn busbar_contract::conn::PollConns>, /// Whole seconds. pub stream_ceiling_secs: u64, + /// The model-serving pools the plane serving the `pools` map walks + /// ([`crate::root::model_egress`]); `None` where none is composed. + pub models: Option<&'a crate::root::model_egress::ModelServing>, /// The linked claims that open at an upgrade (`crate::root::serve::upgrade_carriers`): /// a need over one dials its member's base URL in its own scheme ([`spelled_for`]). pub upgrades: Vec<&'static str>, @@ -1818,6 +2124,8 @@ pub fn member_routes( name: String, provider: &'p ProviderRoute, style: String, + /// The dialect default style's parameters, when the style is that default. + params: &'static [u8], } let mut resolved = Vec::new(); let mut registered = BTreeMap::new(); @@ -1888,10 +2196,17 @@ pub fn member_routes( served .dialect_auth .iter() - .find(|(d, _)| *d == at) - .map(|(_, style)| (*style).to_string()) + .find(|(d, _, _)| *d == at) + .map(|(_, style, params)| ((*style).to_string(), *params)) }); - let style = provider.style.clone().or(default).ok_or_else(|| { + // The provider's own `auth:` binds its style alone; the dialect's default carries its + // declared parameters (Q-L6-AUTHPARAMS). + let (style, params) = match (&provider.style, default) { + (Some(style), _) => (Some(style.clone()), &[][..]), + (None, Some((style, params))) => (Some(style), params), + (None, None) => (None, &[][..]), + }; + let style = style.ok_or_else(|| { format!( "member '{entry}': provider '{name}' states no `auth:` and its plane declares no \ default style for the dialect '{dialect}'" @@ -1902,6 +2217,7 @@ pub fn member_routes( name: name.to_string(), provider, style, + params, }); } let needs: Vec<ReadNeed> = served @@ -2023,6 +2339,8 @@ pub fn member_routes( r.entry, r.style ) })?; + let settings = sealed_settings(r.params, &r.provider.base_url, &settings) + .map_err(|e| format!("member '{}': {e}", r.entry))?; let handle = auth .open_outbound(&r.style, &credential, &settings) .map_err(|e| format!("provider '{}' {e}", r.name))?; @@ -2070,6 +2388,100 @@ pub fn member_routes( Ok(routes) } +/// THE SETTINGS A MEMBER'S BINDING IS OPENED WITH (ARCHITECT RULING 2026-10-03, Q-L6-AUTHPARAMS; +/// ruling 2026-09-28: the kernel resolves the dialect's declared parameters at seal into +/// `OpenOutboundIn::settings`): the dialect's `params` object, each value written +/// `{"host_label_after": [labels], "default": word, "unread": warning}` resolved from the provider's +/// `base_url` host ([`host_label`]; `default` when no label reads, its `unread` warning then handed +/// on as `<key>_unread` for the binding to log with each use), then the provider's own settings +/// over it. +/// +/// # Errors +/// +/// Parameters that are not a JSON object, or a host directive that is malformed. +pub fn sealed_settings( + params: &[u8], + base_url: &str, + provider: &serde_json::Value, +) -> Result<serde_json::Value, String> { + if params.is_empty() { + return Ok(provider.clone()); + } + let serde_json::Value::Object(mut sealed) = serde_json::from_slice(params) + .map_err(|e| format!("its dialect's auth parameters are not JSON: {e}"))? + else { + return Err("its dialect's auth parameters are not a JSON object".to_string()); + }; + let mut unread: Vec<(String, serde_json::Value)> = Vec::new(); + for (key, value) in &mut sealed { + let Some(directive) = value.get("host_label_after") else { + continue; + }; + let labels: Vec<&str> = directive + .as_array() + .map(|a| a.iter().filter_map(serde_json::Value::as_str).collect()) + .unwrap_or_default(); + let default = value + .get("default") + .and_then(serde_json::Value::as_str) + .ok_or_else(|| format!("its dialect's auth parameter `{key}` names no default"))? + .to_string(); + let resolved = match host_label(base_url_host(base_url), &labels) { + Some(label) => label.to_string(), + None => { + // The operator warning the dialect states for a host its rule cannot read, handed + // to the binding as `<key>_unread`: the plugin logs it where 1.5.5 did, with each + // use of the default. + if let Some(words) = value.get("unread").filter(|w| w.is_string()) { + unread.push((format!("{key}_unread"), words.clone())); + } + default + } + }; + *value = serde_json::Value::String(resolved); + } + sealed.extend(unread); + if let serde_json::Value::Object(own) = provider { + for (key, value) in own { + sealed.insert(key.clone(), value.clone()); + } + } + Ok(serde_json::Value::Object(sealed)) +} + +/// The host of `base_url`: after its scheme, before its port, path, query or fragment. +fn base_url_host(base_url: &str) -> &str { + let rest = base_url.split_once("://").map_or(base_url, |(_, r)| r); + let authority = rest.split(['/', '?', '#']).next().unwrap_or(""); + let authority = authority.rsplit_once('@').map_or(authority, |(_, a)| a); + authority.split(':').next().unwrap_or("") +} + +/// The dotted label of `host` right after the first of `after`, when it reads as a dashed name +/// ending in a number (three parts or more, every part before the last alphabetic, the last all +/// digits: the shape the dialect's own rule accepted, 1.5.5's region derivation). +fn host_label<'h>(host: &'h str, after: &[&str]) -> Option<&'h str> { + let shaped = |label: &str| { + let parts: Vec<&str> = label.split('-').collect(); + let Some((last, leading)) = parts.split_last() else { + return false; + }; + parts.len() >= 3 + && parts.iter().all(|p| !p.is_empty()) + && last.bytes().all(|b| b.is_ascii_digit()) + && leading + .iter() + .all(|p| p.bytes().all(|b| b.is_ascii_alphabetic())) + }; + let labels: Vec<&str> = host.split('.').collect(); + labels.iter().enumerate().find_map(|(i, label)| { + if !after.contains(label) { + return None; + } + labels.get(i + 1).copied().filter(|next| shaped(next)) + }) +} + /// The `auth:` style whose credential names its own token endpoint: a service account's /// `token_uri` (RFC 7523; `jwt-bearer`, 1.5.5's `egress_auth/jwt_bearer.rs`). const SERVICE_ACCOUNT_STYLE: &str = "jwt-bearer"; diff --git a/crates/busbar/src/root/egress_ports.rs b/crates/busbar/src/root/egress_ports.rs index 967efc6076..efe86ffbe4 100644 --- a/crates/busbar/src/root/egress_ports.rs +++ b/crates/busbar/src/root/egress_ports.rs @@ -58,6 +58,10 @@ impl Clock for NodeClock { self.origin.elapsed().as_millis() } + fn now_micros(&self) -> u128 { + self.origin.elapsed().as_micros() + } + fn sleep(&self, ms: u64) -> BoxFut<'_, ()> { Box::pin(tokio::time::sleep(std::time::Duration::from_millis(ms))) } @@ -100,6 +104,12 @@ impl MemberPermits { } } + /// The free slots of `destination`; `None` for a member with no limit. + #[must_use] + pub fn available(&self, destination: DestinationId) -> Option<usize> { + self.limits.get(&destination).map(|s| s.available_permits()) + } + fn slot(destination: DestinationId, held: Option<OwnedSemaphorePermit>) -> Permit { Permit::new(Box::new(Slot { destination, diff --git a/crates/busbar/src/root/linked.rs b/crates/busbar/src/root/linked.rs index 70f47d1e08..c248441ea2 100644 --- a/crates/busbar/src/root/linked.rs +++ b/crates/busbar/src/root/linked.rs @@ -611,8 +611,8 @@ pub(crate) fn door_probe_dispatcher() -> Arc<crate::root::loader::dispatch::Disp /// THE DOOR PLANES' REGISTRY ROWS (DECL-FOLD; ARCHITECT RULING 2026-10-03, Q-DEL-A2A-DECL; spec #49 /// and R2-C): every plane [`dropped_planes_of`] discovered through a door, linked or dropped, bound -/// once through the loader's one load on a dispatcher of its own (the process's is built after the -/// configuration is read, and the rows must be in before the config prepass), and its Statement +/// once through the loader's one load on the process's one dispatcher (booted as main's first act, +/// before any plane registers and before the config prepass), and its Statement /// folded into a registry row by the kernel (`busbar_kernel::plane::door::fold`). The row's every /// word is the door's; a door owns the plane axis for its key, a linked legacy row of the same key /// keeping only its other axes ([`doors_own_their_plane_keys`]), and two doors registering one key diff --git a/crates/busbar/src/root/mod.rs b/crates/busbar/src/root/mod.rs index 5f3156a5c5..7baaac6607 100644 --- a/crates/busbar/src/root/mod.rs +++ b/crates/busbar/src/root/mod.rs @@ -87,6 +87,7 @@ pub mod legacy; pub mod linked; pub(crate) mod loader; pub mod migration; +pub mod model_egress; pub mod observe; // The node a plane's units run through, compiled in every build: a door plane's units are driven // and posted on it whether the plane is compiled in or dropped in (ARCHITECT Q1). diff --git a/crates/busbar/src/root/model_egress.rs b/crates/busbar/src/root/model_egress.rs new file mode 100644 index 0000000000..513d41f81c --- /dev/null +++ b/crates/busbar/src/root/model_egress.rs @@ -0,0 +1,734 @@ +// SPDX-License-Identifier: Apache-2.0 +// Copyright (C) 2026 Busbar Inc and contributors + +//! THE EGRESS OF THE PLANE SERVING THE `pools` MAP, sealed with the previous release's pool +//! semantics (BUSBAR-1.6.0.md Part 3 §12 "The route pump": the route step is the kernel's existing +//! egress walk; Part 1: customer-visible behaviour is 1.5.5's). +//! +//! The generic door egress ([`crate::root::door_steps::compose_egress`]) seals every member at +//! weight 1 under the default breaker ladder on a breaker of its own. The `pools:` grammar is the +//! kernel's and says more, and 1.5.5 honoured all of it: each member's weight and attempt timeout, +//! each model's concurrency cap and context window, each pool's failover budget, hop cap and +//! exclusions, its exhaustion terminal and its breaker ladder. And the breaker cells a member trips +//! are the kernel's own lane cells (`AppHandle::store`), keyed by the model's lane index, so the +//! admission the kernel reads, the `busbar_lane_*` gauges a scrape renders and the admin health views +//! all read the trip a door-served attempt recorded, as they read the previous release's. Every +//! outcome is recorded through the lane store's own record calls, the ones the previous release's +//! attempt made (success, transient, rate limit, client fault, hard-down), so the lane's lifetime +//! counters `/stats` renders count it too. A route that names a model directly walks it alone under +//! the model's name (its metric label) on the lane-default cell (`""`), as 1.5.5 attributed a +//! model-routed fault. + +use std::collections::{BTreeMap, HashMap, HashSet}; +use std::sync::Arc; + +use busbar_contract::caps::{Pass, Route}; +use busbar_contract::dest::DestinationId; +use busbar_kernel_egress::ports::{ + Admit, Breaker, Classified, Outcome, Unavailable, UpstreamStatus, +}; +use busbar_kernel_egress::{Failover, Member, OnExhausted, Pool}; + +use crate::root::adapters::{BreakerAdapter, BreakerPolicy}; + +/// The kernel's App, as the live configuration answers it (a config apply rebuilds it, its lane +/// store and its telemetry bank with it). +pub type AppSource = Arc<dyn Fn() -> Arc<busbar_kernel::state::App> + Send + Sync>; + +/// One configured pool: its members (model, weight, attempt timeout), failover bounds and +/// exhaustion terminal. +#[derive(Debug, Clone)] +struct ModelPool { + members: Vec<(String, u32, Option<u64>)>, + failover: Failover, + on_exhausted: OnExhausted, + /// Its members are reached with the caller's own credential (the pool's + /// `upstream_credentials:`, else the all-pools default, is `passthrough`). + passthrough: bool, +} + +/// One model's own bounds: its concurrency cap, its attempt timeout and its context window. +#[derive(Debug, Clone, Default)] +struct ModelLane { + max_concurrent: Option<usize>, + attempt_timeout_ms: Option<u64>, + context_max: Option<usize>, +} + +/// THE MODEL-SERVING POOLS as the configuration resolved them, captured before the configuration +/// is consumed: every pool and model bound the walk honours, and the breaker ladder of each pool. +#[derive(Debug, Clone, Default)] +pub struct ModelPools { + pools: BTreeMap<String, ModelPool>, + lanes: BTreeMap<String, ModelLane>, + breaker: BreakerPolicy, + /// The all-pools `upstream_credentials:` default is `passthrough`: a route naming a model + /// directly reaches it with the caller's own credential. + passthrough: bool, + /// Each pool's ladder as the lane store's record calls take it (the previous release's + /// `resolve_breaker_cfg`: the pool's own, else the default); `""` = the lane-default cell's. + records: HashMap<String, busbar_kernel::store::BreakerCfg>, + /// Each model's active health probing (K7): its provider's `health:` over the process-wide + /// defaults, as 1.5.5's prober resolved it per lane. + probes: BTreeMap<String, busbar_kernel::probe::ProbeCfg>, + /// What each pool's members show the ranking hooks, by pool then model: the member's tier and + /// tags, and its model's cost per million tokens off the rate card (1.5.5's pool member meta). + meta: BTreeMap<String, Arc<HashMap<String, busbar_kernel::plane_driver::MemberFacts>>>, +} + +impl ModelPools { + /// The pools and models `cfg` resolved. + #[must_use] + pub fn of(cfg: &busbar_kernel::config::RootCfg) -> Self { + use busbar_kernel::config::pools::OnExhaustedCfg; + let mut lanes: BTreeMap<String, ModelLane> = cfg + .models + .iter() + .map(|(name, m)| { + ( + name.clone(), + ModelLane { + max_concurrent: m.max_concurrent, + attempt_timeout_ms: m.attempt_timeout_ms, + context_max: None, + }, + ) + }) + .collect(); + let mut pools = BTreeMap::new(); + for (name, pool) in &cfg.pools { + for member in &pool.members { + if let (Some(lane), Some(c)) = (lanes.get_mut(&member.model), member.context_max) { + lane.context_max.get_or_insert(c); + } + } + let failover = pool.failover.as_ref().map_or( + Failover { + timeout_secs: busbar_kernel::config::DEFAULT_FAILOVER_DEADLINE_SECS, + max_hops: busbar_kernel::config::DEFAULT_FAILOVER_CAP, + exclusions: Vec::new(), + }, + |f| Failover { + timeout_secs: f.timeout_secs, + max_hops: f.max_hops, + exclusions: f.exclusions.clone().unwrap_or_default(), + }, + ); + let on_exhausted = match &pool.on_exhausted { + None | Some(OnExhaustedCfg::Reject) => OnExhausted::Status503, + Some(OnExhaustedCfg::LeastBad) => OnExhausted::LeastBad, + Some(OnExhaustedCfg::FallbackPool(p)) => OnExhausted::FallbackPool(p.clone()), + Some(OnExhaustedCfg::Queue { max_ms }) => OnExhausted::Queue { max_ms: *max_ms }, + }; + pools.insert( + name.clone(), + ModelPool { + members: pool + .members + .iter() + .map(|m| (m.model.clone(), m.weight, m.attempt_timeout_ms)) + .collect(), + failover, + on_exhausted, + passthrough: pool + .upstream_credentials + .unwrap_or(cfg.upstream_credentials) + == busbar_contract::config::UpstreamCreds::Passthrough, + }, + ); + } + let mut records: HashMap<String, busbar_kernel::store::BreakerCfg> = cfg + .pools + .iter() + .map(|(name, pool)| { + ( + name.clone(), + pool.breaker + .as_ref() + .map(busbar_kernel::store::breaker_cfg_to_runtime) + .unwrap_or_default(), + ) + }) + .collect(); + records.insert(String::new(), busbar_kernel::store::BreakerCfg::default()); + let probes = cfg + .models + .iter() + .filter_map(|(name, m)| { + let health = cfg.providers.get(&m.provider)?.health.as_ref()?; + Some(( + name.clone(), + busbar_kernel::probe::ProbeCfg::resolve( + match health.mode { + busbar_kernel::config::HealthMode::None => { + busbar_kernel::plane_host::HealthModeInput::None + } + busbar_kernel::config::HealthMode::Dead => { + busbar_kernel::plane_host::HealthModeInput::Dead + } + busbar_kernel::config::HealthMode::Active => { + busbar_kernel::plane_host::HealthModeInput::Active + } + }, + health.interval_secs, + health.timeout_secs, + cfg.limits.default_probe_interval_secs, + cfg.limits.default_probe_timeout_secs, + ), + )) + }) + .collect(); + let meta = cfg + .pools + .iter() + .map(|(name, pool)| { + let members = pool + .members + .iter() + .map(|m| { + ( + m.model.clone(), + busbar_kernel::plane_driver::MemberFacts { + model: m.model.clone(), + tags: m.tags.clone(), + tier: m.tier.clone(), + cost_per_mtok: cfg + .rate_card + .as_ref() + .and_then(|card| card.get(&m.model)) + .map(busbar_kernel::config::rate_entry_per_mtok), + }, + ) + }) + .collect(); + (name.clone(), Arc::new(members)) + }) + .collect(); + ModelPools { + meta, + pools, + lanes, + breaker: BreakerPolicy::from_pools(&cfg.pools), + passthrough: cfg.upstream_credentials + == busbar_contract::config::UpstreamCreds::Passthrough, + records, + probes, + } + } +} + +impl ModelServing { + /// THE MEMBERS THE HEALTH-PROBE SERVICE SCHEDULES (K7), in lane order: each model with a + /// probing mode, at its lane's index and destination, under its resolved settings. + #[must_use] + pub fn probe_members( + &self, + ) -> ( + usize, + Vec<(DestinationId, busbar_kernel::probe::ProbeMember)>, + ) { + let mut members: Vec<(DestinationId, busbar_kernel::probe::ProbeMember)> = self + .lanes + .iter() + .filter_map(|(model, lane)| { + let cfg = *self.pools.probes.get(model)?; + Some(( + DestinationId::new(*lane as u64), + busbar_kernel::probe::ProbeMember { + index: *lane, + name: model.clone(), + cfg, + }, + )) + }) + .collect(); + members.sort_by_key(|(_, m)| m.index); + (self.lanes.values().max().map_or(0, |m| m + 1), members) + } +} + +/// WHAT THE MODEL-SERVING EGRESS IS SEALED OVER beside the plane's member routes: the pools, the +/// lane index of each model in the generation's lane table (the kernel's breaker cells are keyed by +/// it), and the breaker unit those cells live in, as the live configuration answers it. +pub struct ModelServing { + /// The pools and models. + pub pools: ModelPools, + /// Each model's lane index. + pub lanes: HashMap<String, usize>, + /// The kernel's App, read on every call: its lane store and its telemetry bank. + pub app: AppSource, +} + +impl std::fmt::Debug for ModelServing { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("ModelServing") + .field("pools", &self.pools) + .field("lanes", &self.lanes) + .finish_non_exhaustive() + } +} + +/// The number of latency samples a lane's p95 reservoir holds (1.5.5's ring). +const LATENCY_RESERVOIR_LEN: usize = 128; + +/// ONE LANE'S P95 LATENCY RESERVOIR (1.5.5's `LatencyReservoir`, moved verbatim): a ring of the +/// last [`LATENCY_RESERVOIR_LEN`] samples in microseconds, written lock-free. +#[derive(Debug)] +pub struct LatencyReservoir { + cursor: std::sync::atomic::AtomicUsize, + samples: [std::sync::atomic::AtomicU32; LATENCY_RESERVOIR_LEN], +} + +impl Default for LatencyReservoir { + fn default() -> Self { + Self { + cursor: std::sync::atomic::AtomicUsize::new(0), + samples: std::array::from_fn(|_| std::sync::atomic::AtomicU32::new(0)), + } + } +} + +impl LatencyReservoir { + /// Record one sample, in milliseconds. A non-finite or non-positive sample is ignored. + pub fn record(&self, latency_ms: f64) { + use std::sync::atomic::Ordering; + if !latency_ms.is_finite() || latency_ms <= 0.0 { + return; + } + // `as` saturates a float-to-int cast, so an absurdly long sample clamps to `u32::MAX` us. + #[allow(clippy::cast_possible_truncation, clippy::cast_sign_loss)] + let us = ((latency_ms * 1000.0).round() as u32).max(1); + let slot = self.cursor.fetch_add(1, Ordering::Relaxed) % LATENCY_RESERVOIR_LEN; + self.samples[slot].store(us, Ordering::Relaxed); + } + + /// The nearest-rank 95th percentile of the held samples, in whole milliseconds rounded UP; + /// `None` while the ring holds no sample. + #[must_use] + pub fn p95_ms(&self) -> Option<u64> { + let mut held: Vec<u32> = self + .samples + .iter() + .map(|s| s.load(std::sync::atomic::Ordering::Relaxed)) + .filter(|v| *v != 0) + .collect(); + if held.is_empty() { + return None; + } + held.sort_unstable(); + let rank = (held.len() * 95).div_ceil(100); + Some(u64::from(held[rank - 1]).div_ceil(1000)) + } +} + +/// EACH MEMBER'S STANDING AS THE RANKING HOOKS READ IT, off the generation's lane store, as 1.5.5's +/// candidate projection read it: the latency signal, the free concurrency (the member's own permits +/// where its `max_concurrent` is stated, the lane's otherwise), the remaining lifetime budget, the +/// breaker state and error rate in the routing pool, and the lane's p95 reservoir. +pub struct ModelSignals { + app: AppSource, + permits: Arc<crate::root::egress_ports::MemberPermits>, + reservoirs: Arc<HashMap<usize, LatencyReservoir>>, +} + +impl busbar_kernel::plane_driver::MemberSignals for ModelSignals { + fn standing( + &self, + pool: &str, + destination: DestinationId, + ) -> busbar_kernel::plane_driver::MemberStanding { + let (app, lane) = ((self.app)(), lane_of(destination)); + let store = &app.store; + let state = match store.breaker_state_snapshot_in(pool, lane) { + busbar_kernel::store::BreakerState::Closed => "closed", + busbar_kernel::store::BreakerState::Open { .. } => "open", + busbar_kernel::store::BreakerState::HalfOpen => "half_open", + }; + busbar_kernel::plane_driver::MemberStanding { + latency_ms: store.lane_latency_ms(lane), + available_concurrency: self + .permits + .available(destination) + .unwrap_or_else(|| store.available_permits(lane)), + budget_remaining: store.lane_budget_remaining(lane), + breaker_state: Some(state), + error_rate: store.error_rate_in(pool, lane, busbar_kernel::store::now()), + latency_p95_ms: self + .reservoirs + .get(&lane) + .and_then(LatencyReservoir::p95_ms), + } + } +} + +/// What the hooks are shown of the model-serving pools beside the walk: each pool's member meta +/// and the members' live standing. +pub struct ModelShown { + /// Each pool's member meta, by pool. + pub described: BTreeMap<String, Arc<HashMap<String, busbar_kernel::plane_driver::MemberFacts>>>, + /// The members' live standing. + pub signals: Arc<dyn busbar_kernel::plane_driver::MemberSignals>, +} + +/// The kernel's breaker port over its lane store: a model-named direct route is mapped onto the +/// lane-default cell, and every outcome is recorded through the store's own record calls. +struct DirectCells { + inner: BreakerAdapter, + app: AppSource, + records: HashMap<String, busbar_kernel::store::BreakerCfg>, + direct: HashSet<String>, +} + +/// The lane a destination of this egress names (its destination is the model's lane index). +fn lane_of(destination: DestinationId) -> usize { + usize::try_from(destination.get()).unwrap_or(usize::MAX) +} + +impl DirectCells { + fn cell<'p>(&self, pool: &'p str) -> &'p str { + if self.direct.contains(pool) { + "" + } else { + pool + } + } +} + +impl Breaker for DirectCells { + fn try_admit( + &self, + pool: &str, + destination: DestinationId, + now: u64, + ) -> Result<Admit, Unavailable> { + self.inner.try_admit(self.cell(pool), destination, now) + } + + fn ready(&self, pool: &str, destination: DestinationId, now: u64, token: &Pass<Route>) -> bool { + self.inner.ready(self.cell(pool), destination, now, token) + } + + fn admissible(&self, destination: DestinationId) -> bool { + self.inner.admissible(destination) + } + + fn cooldown_remaining( + &self, + pool: &str, + destination: DestinationId, + now: u64, + token: &Pass<Route>, + ) -> u64 { + self.inner + .cooldown_remaining(self.cell(pool), destination, now, token) + } + + fn classify(&self, destination: DestinationId, status: UpstreamStatus) -> Classified { + self.inner.classify(destination, status) + } + + fn observe( + &self, + pool: &str, + destination: DestinationId, + outcome: Outcome, + _now: u64, + _token: &Pass<Route>, + ) -> bool { + let cell = self.cell(pool); + let Some(cfg) = self.records.get(cell) else { + return false; + }; + let (store, lane) = (Arc::clone(&(self.app)().store), lane_of(destination)); + match outcome { + Outcome::Success => { + store.record_success_in(cell, lane); + false + } + Outcome::Transient { retry_after } => { + store.record_transient_in(cell, lane, "transient", cfg, retry_after) + } + Outcome::HardDown => store.record_hard_down_all_cells(lane, "hard_down"), + Outcome::RecordNothing => false, + } + } + + fn judge( + &self, + pool: &str, + destination: DestinationId, + status: UpstreamStatus, + now: u64, + token: &Pass<Route>, + ) -> (Classified, bool) { + let classified = self.classify(destination, status); + // The far end's number in the kernel's first reserved numbering (the one the previous + // release's ladder reads its 429 and 4xx in), read as the kernel's far end reads it. + let code = status.code.and_then(|c| { + c.in_namespace(busbar_contract::transport::registry::status_ns::RESERVED[0]) + }); + let cell = self.cell(pool); + let lane = lane_of(destination); + let tripped = match (classified.outcome, code) { + // The previous release's rate-limit record: its own ladder step, the far end's wait + // honoured. + (Outcome::Transient { retry_after }, Some(429)) => match self.records.get(cell) { + Some(cfg) => { + (self.app)() + .store + .record_rate_limit_in(cell, lane, now, cfg, retry_after) + } + None => false, + }, + // The caller's own fault: counted on the lane, its breaker untouched. + (Outcome::RecordNothing, Some(400..=499)) => { + (self.app)().store.record_client_fault(lane); + false + } + (outcome, _) => self.observe(pool, destination, outcome, now, token), + }; + (classified, tripped) + } + + fn suppressing(&self, destination: DestinationId, now: u64) -> bool { + (self.app)() + .store + .lane_needs_probe(lane_of(destination), now) + } + + /// A health probe's answer, recorded as 1.5.5's prober recorded it (`engine/health.rs`): a + /// success recovers the lane in every cell, then joins every cell's window; a transient failure + /// joins every cell's window under its own pool's ladder; a hard-down parks the lane in every + /// cell; a client fault records nothing. + fn probed(&self, destination: DestinationId, outcome: Outcome, now: u64, _token: &Pass<Route>) { + let (app, lane) = ((self.app)(), lane_of(destination)); + let store = &app.store; + match outcome { + Outcome::Success => { + if store.lane_needs_probe(lane, now) { + store.recover_lane(lane); + } + store.record_probe_success_all_cells(lane); + } + Outcome::Transient { retry_after } => { + let resolve = |pool: &str| self.records.get(pool).cloned().unwrap_or_default(); + store.record_probe_failure_all_cells(lane, "health-probe", &resolve, retry_after); + } + Outcome::HardDown => { + let _ = + store.record_hard_down_all_cells(lane, "health-probe hard-down (auth/billing)"); + } + Outcome::RecordNothing => {} + } + } + + fn release_probe(&self, pool: &str, destination: DestinationId, epoch: u64, now: u64) { + self.inner + .release_probe(self.cell(pool), destination, epoch, now); + } + + fn spend_budget(&self, destination: DestinationId) -> bool { + self.inner.spend_budget(destination) + } + + fn refund_budget(&self, destination: DestinationId) { + self.inner.refund_budget(destination); + } +} + +/// The walk's counters on the kernel's telemetry bank, as the previous release's attempt counted +/// them: each member by its lane index, under the pool label the walk names (a pool, or the model a +/// direct route names); the wait-terminal depth as the root's walk counters keep it. +struct ModelTelemetry { + app: AppSource, + queued: crate::root::egress_ports::WalkTelemetry, + /// Each lane's p95 reservoir, fed only while the generation declares the signal. + reservoirs: Arc<HashMap<usize, LatencyReservoir>>, +} + +impl busbar_kernel_egress::ports::Telemetry for ModelTelemetry { + fn upstream_attempt(&self, pool: &str, destination: DestinationId) { + busbar_kernel::telemetry::upstream_attempt(&(self.app)(), pool, lane_of(destination)); + } + + fn upstream_failure(&self, pool: &str, destination: DestinationId, disposition: &'static str) { + busbar_kernel::telemetry::upstream_failure( + &(self.app)(), + pool, + lane_of(destination), + disposition, + ); + } + + fn failover(&self, pool: &str, reason: &'static str) { + busbar_kernel::telemetry::failover(&(self.app)(), pool, reason); + } + + fn breaker_trip(&self, pool: &str, destination: DestinationId) { + busbar_kernel::telemetry::breaker_trip(&(self.app)(), pool, lane_of(destination)); + } + + fn upstream_latency(&self, destination: DestinationId, ms: f64) { + // The lane's latency signal (the admin pool listing's `latency_ms`, the `fastest` order's + // input), as 1.5.5's served attempt recorded it. + let app = (self.app)(); + let lane = lane_of(destination); + app.store.record_latency_in("", lane, ms); + // The same sample into the lane's p95 reservoir, ONLY while the generation declares the + // signal, as 1.5.5 collected it. + if app + .requested_signals + .wants(busbar_contract::signal::Signal::CandidateLatencyP95Ms) + { + if let Some(r) = self.reservoirs.get(&lane) { + r.record(ms); + } + } + } + + fn queued(&self, pool: &str, delta: i64) { + busbar_kernel_egress::ports::Telemetry::queued(&self.queued, pool, delta); + // The scrape's pool-queue gauge reads the depth the kernel's tables report. + busbar_kernel::plane_host::set_pool_queued_depth( + pool, + u64::try_from(self.queued.depth(pool)).unwrap_or(0), + ); + } +} + +/// THE EGRESS OF THE PLANE SERVING THE `pools` MAP (module doc): one member per sealed entry (its +/// destination the model's lane index, its name the model's), a pool per configured pool with its +/// own bounds and terminal, a pool per model a route may name directly, the kernel's breaker cells +/// under each pool's ladder, each model's concurrency cap, the node's clock and counters, and +/// `journal`. +/// +/// # Errors +/// +/// A configured pool naming a model with no sealed route or no lane: the load is refused, naming +/// both. +#[allow(clippy::too_many_arguments)] +pub fn compose( + facts: &crate::root::door_steps::DoorFacts, + serving: &ModelServing, + caller: busbar_contract::conn::InstanceId, + conns: Arc<dyn busbar_contract::conn::PollConns>, + routes: &BTreeMap<String, busbar_kernel::plane_driver::MemberRoute>, + journal: Arc<dyn busbar_kernel_egress::ports::Journal>, + stream_ceiling_secs: u64, +) -> Result<(busbar_kernel::plane_driver::Egress, ModelShown), String> { + let plan = &serving.pools; + let mut sealed = HashMap::new(); + let mut names = Vec::new(); + let mut limits = Vec::new(); + let mut member_of = |model: &str, + weight: u32, + attempt: Option<u64>, + passthrough: bool| + -> Result<Member, String> { + let lane = *serving + .lanes + .get(model) + .ok_or_else(|| format!("model '{model}' has no lane in this generation"))?; + let destination = DestinationId::new(lane as u64); + let route = routes + .get(model) + .ok_or_else(|| format!("model '{model}' has no sealed route"))?; + let own = plan.lanes.get(model).cloned().unwrap_or_default(); + if let std::collections::hash_map::Entry::Vacant(slot) = sealed.entry(destination) { + let mut route = route.clone(); + route.keep = facts + .keeps + .get(route.need.0 as usize) + .cloned() + .unwrap_or_default(); + slot.insert(route); + names.push((destination, model.to_string())); + if let Some(n) = own.max_concurrent { + limits.push((destination, n)); + } + } + let mut member = Member::new(destination, model, weight); + member.attempt_timeout_ms = attempt.or(own.attempt_timeout_ms); + member.context_max = own.context_max.map(|c| c as u64); + member.passthrough = passthrough; + Ok(member) + }; + let mut built: HashMap<String, Pool> = HashMap::new(); + for (name, pool) in &plan.pools { + let members = pool + .members + .iter() + .map(|(model, weight, attempt)| { + member_of(model, *weight, *attempt, pool.passthrough) + .map_err(|e| format!("pool '{name}': {e}")) + }) + .collect::<Result<Vec<_>, _>>()?; + let mut p = Pool::new(name.clone(), members); + p.failover = pool.failover.clone(); + p.on_exhausted = pool.on_exhausted.clone(); + built.insert(name.clone(), p); + } + let mut direct = HashSet::new(); + for model in routes.keys() { + if built.contains_key(model) || !serving.lanes.contains_key(model) { + continue; + } + let member = member_of(model, 1, None, plan.passthrough)?; + let mut p = Pool::new(model.clone(), vec![member]); + p.failover = Failover { + timeout_secs: busbar_kernel::config::DEFAULT_FAILOVER_DEADLINE_SECS, + max_hops: busbar_kernel::config::DEFAULT_FAILOVER_CAP, + exclusions: Vec::new(), + }; + built.insert(model.clone(), p); + direct.insert(model.clone()); + } + let permits = Arc::new(crate::root::egress_ports::MemberPermits::new(limits)); + let reservoirs: Arc<HashMap<usize, LatencyReservoir>> = Arc::new( + serving + .lanes + .values() + .map(|lane| (*lane, LatencyReservoir::default())) + .collect(), + ); + let shown = ModelShown { + described: plan.meta.clone(), + signals: Arc::new(ModelSignals { + app: Arc::clone(&serving.app), + permits: Arc::clone(&permits), + reservoirs: Arc::clone(&reservoirs), + }), + }; + let app = Arc::clone(&serving.app); + let breaker = DirectCells { + inner: BreakerAdapter::over( + Arc::new(move || app().store.breaker_unit()), + plan.breaker.clone(), + ), + app: Arc::clone(&serving.app), + records: plan.records.clone(), + direct, + }; + Ok(( + busbar_kernel::plane_driver::Egress { + caller, + conns, + breaker: Arc::new(breaker), + capacity: permits, + clock: Arc::new(crate::root::egress_ports::NodeClock::new()), + journal, + telemetry: Arc::new(ModelTelemetry { + app: Arc::clone(&serving.app), + queued: crate::root::egress_ports::WalkTelemetry::new(names), + reservoirs, + }), + floor: busbar_kernel_egress::WeightedFloor::new(), + pools: built, + routes: sealed, + stream_ceiling_secs, + stated_ceiling_secs: facts.stream_ceiling_secs, + error_body_max: busbar_kernel::plane_driver::DEFAULT_ERROR_BODY_MAX, + }, + shown, + )) +} diff --git a/crates/busbar/src/root/plane_node.rs b/crates/busbar/src/root/plane_node.rs index cf89ab9263..6796e73bbc 100644 --- a/crates/busbar/src/root/plane_node.rs +++ b/crates/busbar/src/root/plane_node.rs @@ -180,7 +180,7 @@ impl LaneNames { /// One per process. The per-request half is the plane's unit, which a plane's arrival hands this /// node ([`Handed`]) and which is thrown away with the unit. pub struct Node { - kernel: busbar_kernel::teller::Kernel, + kernel: Arc<busbar_kernel::teller::Kernel>, inflight: busbar_kernel::inflight::InFlight, gauge: busbar_kernel::slice::ConcurrencyGauge, canary: busbar_contract::caps::Canary, @@ -193,7 +193,7 @@ pub struct Node { /// The names this node has already put through that interner, so the request path does not put /// them through it again. Shared with the resolver every unit is lent ([`Node::resolver`]). lane_names: Arc<Mutex<LaneNames>>, - next_key: busbar_kernel::door::UnitKeyMint, + next_key: Arc<busbar_kernel::door::UnitKeyMint>, /// How many slots the drop guard has MARKED since the sweep last walked the table. /// /// A counter rather than a walk on every arrival: a unit whose task went away leaves its slot @@ -289,7 +289,7 @@ impl Node { usage_token: kernel.usage_token(), book: std::sync::OnceLock::new(), lane: std::sync::OnceLock::new(), - kernel, + kernel: Arc::new(kernel), // The data listener already carries the operator-configured inbound-concurrency layer, // which is where this deployment's admission-to-the-node decision is made and has always // been made. A second cap here would be a second answer to one question, and the one @@ -308,7 +308,7 @@ impl Node { resolved: HashMap::new(), consulted: 0, })), - next_key: busbar_kernel::door::UnitKeyMint::default(), + next_key: Arc::default(), marked: AtomicU64::new(0), lost: Mutex::new(HashMap::new()), mono: AtomicU64::new(0), @@ -319,6 +319,18 @@ impl Node { } } + /// The node's kernel and its one unit-key mint, shared with a unit the node does not drive but + /// whose key must be unique beside its own (a door plane's health probe, K7). + #[must_use] + pub fn kernel_and_keys( + &self, + ) -> ( + Arc<busbar_kernel::teller::Kernel>, + Arc<busbar_kernel::door::UnitKeyMint>, + ) { + (Arc::clone(&self.kernel), Arc::clone(&self.next_key)) + } + /// How many of this node's units have driven the engine through the Route seam. /// /// The instrument the switch-over is measured with, and it measures what a status cannot: a diff --git a/crates/busbar/src/root/serve.rs b/crates/busbar/src/root/serve.rs index 83e4f32059..518ac06605 100644 --- a/crates/busbar/src/root/serve.rs +++ b/crates/busbar/src/root/serve.rs @@ -507,6 +507,8 @@ pub struct ServedPlane { pub live: Arc<DoorApply>, /// The kernel's host services it was admitted to: its units' records are written there. pub kernel: Arc<KernelServices>, + /// Its tail's dialects, in order: what a unit's dialect index names. + pub dialects: Vec<&'static str>, } /// WHAT ONE GENERATION OF A DOOR PLANE IS SERVED OVER: its section as written, its refresh @@ -522,6 +524,15 @@ pub struct DoorLive { pub pools: DoorPools, /// Its egress. pub egress: Option<Arc<Egress>>, + /// The tail's facts as this generation states them (with the generation's provider dialects, + /// for the translation counter). + pub facts: DoorFacts, + /// The health-probe service's target for this generation (K7), when the plane answers probes + /// and a member probes: held here, so a replaced generation's probers exit at their next tick. + pub probes: Option<Arc<busbar_kernel::plane_driver::PlaneProbes>>, + /// What the ranking hooks are shown of the model-serving pools' members beside the walk: their + /// meta and their live standing (`None` for a plane whose egress is the generic walk). + pub shown: Option<crate::root::model_egress::ModelShown>, } /// What a door plane's egress is re-sealed over on a config apply: the providers the deployment @@ -547,6 +558,12 @@ pub struct DoorApply { facts: DoorFacts, reach: Option<ApplyReach>, live: std::sync::RwLock<Arc<DoorLive>>, + /// The plane's driver, which a generation's health probes run their units on (K7). + driver: Arc<PlaneDriver>, + /// The health-probe schedule its generations share, phase-stable across a config apply. + probe_schedule: Arc<busbar_kernel::probe::ProbeSchedule>, + /// The scope kinds its Statement declares. + scope_kinds: Vec<&'static str>, } impl DoorApply { @@ -565,6 +582,12 @@ impl DoorApply { /// section the plane serves now), a new refresh generation, and pools and egress re-derived. /// A plane that will not refresh keeps serving its current generation, logged. pub fn apply(&self, app: &busbar_kernel::state::App) { + // The plane serving the kernel-owned `pools` map reads the deployment's top-level sections, + // not a slot of its own: it is refreshed from the generation's configuration projection + // ([`DoorAppliers::refresh_models`]). + if self.served_facts.section == busbar_contract::section::RESERVED_POOLS_KEY { + return; + } let now = self.current(); // THE SECTION AS BOOT BUILDS IT (audit root-R1 leftover C1): the slot's section as written, // with the generation's unified pools at its reserved `pools` key ([`with_pools`]); the plane @@ -617,6 +640,7 @@ impl DoorApply { conns: Arc::clone(&r.conns), stream_ceiling_secs: r.stream_ceiling_secs, upgrades: r.upgrades.clone(), + models: None, }; let routes = crate::root::door_steps::member_routes( section, @@ -641,6 +665,9 @@ impl DoorApply { generation, pools, egress, + facts: self.facts.clone(), + probes: None, + shown: None, }) } } @@ -680,6 +707,73 @@ impl DoorAppliers { plane.apply(app); } } + + /// A CONFIG APPLY, as the plane serving the kernel-owned `pools` map takes it (spec Part 1 + /// line 569: one validated object per `refresh`): refreshed onto the next generation with the + /// new configuration's sections, and the generation its units are served by sealed anew over the + /// new providers and pool bounds and swapped in; a unit already running keeps the generation it + /// started on, as 1.5.5's request kept the snapshot it arrived on. The generation before the + /// previous one is retired. A plane that will not refresh or seal keeps serving its previous + /// generation, and says why. + pub fn refresh_models( + &self, + sections: &BTreeMap<&'static str, serde_yaml::Value>, + egress: &DoorEgress<'_>, + ) { + for p in &self.0 { + let facts = &p.served_facts; + if facts.section != busbar_contract::section::RESERVED_POOLS_KEY { + continue; + } + let Some(section) = sections.get(facts.section) else { + continue; + }; + let name = p.plugin.name(); + let settings = match serde_json::to_vec(&settings_of(facts, section, sections)) { + Ok(settings) => settings, + Err(e) => { + tracing::error!(plane = name, error = %e, "door plane's new configuration is not JSON; it keeps serving the previous one"); + continue; + } + }; + let walked = walked_section(facts, section, sections); + let now = p.current(); + let next = now.generation + 1; + if let Err(e) = crate::root::loader::dispatch::kinds::plane::refresh_door( + &p.plugin, &settings, next, + ) { + tracing::error!(plane = name, error = %e, "door plane did not refresh onto the new configuration; it keeps serving the previous one"); + continue; + } + match seal_live( + name, + &p.plugin, + facts, + &p.scope_kinds, + &walked, + Some(egress), + next, + p.facts.bench_below_trip_threshold, + ) { + Ok(mut live) => { + arm_probes(&p.driver, facts, &mut live, Some(egress), &p.probe_schedule); + *p.live + .write() + .unwrap_or_else(std::sync::PoisonError::into_inner) = Arc::new(live); + } + Err(e) => { + tracing::error!(plane = name, error = %e, "door plane's new configuration did not seal; it keeps serving the previous one"); + continue; + } + } + if now.generation > 1 { + crate::root::loader::dispatch::kinds::plane::retire_door( + &p.plugin, + now.generation - 1, + ); + } + } + } } impl std::fmt::Debug for DoorLive { @@ -1275,6 +1369,11 @@ pub(crate) fn compose_planes_over( continue; } }; + // A MODEL-SERVING PLANE in the previous release's layout is handed the `pools` map and each + // section it reads beside it, keyed; its walk is the uniform model-serving section over its + // `pools:` and the top-level `models:` (#49). + let settings = settings_of(&served_facts, section, sections); + let walked = walked_section(&served_facts, section, sections); // The plane's other owned sections this document writes, as written, by section name: it // reads them beside its settings (ARCHITECT Q-L3B-AUD). Names come from its Statement. let owned: serde_json::Map<String, serde_json::Value> = served_facts @@ -1285,13 +1384,14 @@ pub(crate) fn compose_planes_over( Some(((*name).to_string(), value)) }) .collect(); - // The dialect facts of the providers its section references (THE DESIGN §4): it resolves - // model to dialect itself. Read off the providers its egress is sealed over. + // The dialect facts of the providers its walked section references (THE DESIGN §4): it + // resolves model to dialect itself. Read off the providers its egress is sealed over. let dialects = egress - .map(|e| crate::root::door_steps::dialect_facts(section, e.reach.providers)) + .map(|e| crate::root::door_steps::dialect_facts(&walked, e.reach.providers)) .unwrap_or_default(); - let snapshot = open(plugin, section, public_url, &owned, &dialects) + let snapshot = open(plugin, &settings, public_url, &owned, &dialects) .map_err(|e| format!("{instance}: {e}"))?; + let section = &walked; let calls = Arc::new(PlaneInstance::new( plugin.clone(), Arc::clone(dispatcher), @@ -1325,9 +1425,11 @@ pub(crate) fn compose_planes_over( ) .map_err(|e| format!("{instance}: {e}"))? .with_records(Arc::clone(&kernel), caller.clone()); - // THE HOOK STAGE IN THE PLANE'S OWN ORDER (spec Part 3 section 12 "Hooks"): a plane whose - // tail states the gate-first order has its entries' gates and rewrites bound, filed under - // its registry key. + // THE HOOK STAGE IN THE PLANE'S OWN ORDER (spec Part 3 section 12 "Hooks": the hook stages + // run at the head of the route leg, in the hook order 1.5.5 used for that plane): a plane + // whose tail states the gate-first order has its entries' gates and rewrites bound, filed + // under its registry key; every other (the plane serving the `pools` map, the one 1.5.5 ran + // its request hooks on, among them) the routed order (ARCHITECT K5, U22). let driver = match hooks { Some(stage) if served_facts.tail_flags & busbar_contract::abi::plane::TAIL_HOOKS_GATED != 0 => @@ -1380,73 +1482,52 @@ pub(crate) fn compose_planes_over( )); } } - let mut facts = door_facts( - plugin.name(), - &declared.scope_kinds, - &served_facts.billable_classes, - &served_facts.fee_units, - served_facts.audit_kind, - served_facts - .keeps - .iter() - .map(|k| busbar_kernel::plane_driver::ResponseKeep { - mode: k.mode, - kept: k.kept.iter().map(|n| (*n).to_string()).collect(), - denied: k.denied.iter().map(|n| (*n).to_string()).collect(), - }) - .collect(), - ); // THE PLANE'S BREAKER FACT, as it declares it (ARCHITECT Q4): read off its `declares` // section, whichever plane it is; absent, its members' cells keep the default. - facts.bench_below_trip_threshold = crate::root::linked::door_breaker( + let bench = crate::root::linked::door_breaker( linked, crate::root::linked::dropped(), plugin.name(), ) .map_err(|e| format!("{instance}: {e}"))? .map(|b| b.bench_below_trip_threshold); - // THE PLANE'S STATED STREAM CEILING (ARCHITECT ruling 2026-10-07, STREAM-CEILING), off its - // tail: the deadline of its streamed units once their route is known; `0` = none. - facts.stream_ceiling_secs = served_facts.stream_ceiling_secs; - let pools = DoorPools::of(section); - // THE EGRESS, SEALED (THE DESIGN §6 steps 2-3): each member's route resolved and its - // credential bound by the auth plugin serving its style, over the connector its needs were - // declared on. - let egress_sealed = match egress { - Some(egress) => { - let routes = crate::root::door_steps::member_routes( - section, - &pools, - &served_facts, - egress.reach, - ) - .map_err(|e| format!("{instance}: {e}"))?; - // The door's own requests to its members carry the members' bindings (ARCHITECT - // round 5 Q-L3B-DOOR-EXCHANGE), held on the connection table its need is declared on. - if let Some(table) = plugin.conn_table() { - crate::root::door_steps::bind_member_fetches( - &served_facts, - &routes, - plugin.instance(), - &*table, - kernel.units(), - ); - } - Some(Arc::new( - crate::root::door_steps::compose_egress( - &facts, - &pools, - plugin.instance(), - Arc::clone(&egress.reach.conns), - &routes, - Arc::clone(&egress.journal), - egress.reach.stream_ceiling_secs, - ) - .map_err(|e| format!("{instance}: {e}"))?, - )) - } - None => None, - }; + let mut live = seal_live( + instance, + plugin, + &served_facts, + &declared.scope_kinds, + section, + egress, + 1, + bench, + )?; + // The door's own requests to its members carry the members' bindings (ARCHITECT round 5 + // Q-L3B-DOOR-EXCHANGE), held on the connection table its need is declared on. + if let (Some(egress), Some(table)) = (egress, plugin.conn_table()) { + let pools = DoorPools::of(section); + let routes = crate::root::door_steps::member_routes( + section, + &pools, + &served_facts, + egress.reach, + ) + .map_err(|e| format!("{instance}: {e}"))?; + crate::root::door_steps::bind_member_fetches( + &served_facts, + &routes, + plugin.instance(), + &*table, + kernel.units(), + ); + } + let driver = Arc::new(driver); + let probe_schedule = Arc::new(busbar_kernel::probe::ProbeSchedule::new( + egress + .and_then(|e| e.reach.models) + .map_or(0, |m| m.probe_members().0), + )); + arm_probes(&driver, &served_facts, &mut live, egress, &probe_schedule); + let facts = live.facts.clone(); let reach = egress.map(|e| ApplyReach { providers: e.reach.providers.clone(), auths: Arc::clone(&e.reach.auths), @@ -1460,27 +1541,235 @@ pub(crate) fn compose_planes_over( served_facts: served_facts.clone(), facts: facts.clone(), reach, - live: std::sync::RwLock::new(Arc::new(DoorLive { - section: section.clone(), - generation: 1, - pools, - egress: egress_sealed, - })), + live: std::sync::RwLock::new(Arc::new(live)), + driver: Arc::clone(&driver), + probe_schedule, + scope_kinds: declared.scope_kinds.clone(), }); served.planes.push(ServedPlane { instance: instance.clone(), - driver: Arc::new(driver), + driver, snapshot, audit_kind: served_facts.audit_kind, facts, money: plane_money, live, kernel: Arc::clone(&kernel), + dialects: served_facts.dialects.clone(), }); } Ok(served) } +/// THE GENERATION A DOOR PLANE'S UNITS ARE SERVED BY ([`DoorLive`], refresh generation `generation`), sealed over its walked +/// `section`: the tail's facts (the flat card's empty plane key for the plane serving the `pools` +/// map, which owns 1.5.5's `rate_card:`; each provider's dialect, for the translation counter), the +/// section's pools and, with `egress`, each member's route resolved and its credential bound by the +/// auth plugin serving its style over the connector its needs were declared on (THE DESIGN §6 steps +/// 2-3), the plane serving the `pools` map walking its pools with the previous release's semantics +/// on the kernel's own breaker cells ([`crate::root::model_egress`]). +/// +/// # Errors +/// +/// A member whose route or credential binding cannot be resolved, named. +#[allow(clippy::too_many_arguments)] +fn seal_live( + instance: &str, + plugin: &DoorPlane, + served_facts: &crate::root::loader::dispatch::kinds::plane::ServedFacts, + scope_kinds: &[&str], + section: &serde_yaml::Value, + egress: Option<&DoorEgress<'_>>, + generation: u64, + bench_below_trip_threshold: Option<bool>, +) -> Result<DoorLive, String> { + let models_plane = served_facts.section == busbar_contract::section::RESERVED_POOLS_KEY; + let card_plane = if models_plane { "" } else { plugin.name() }; + let mut facts = door_facts( + card_plane, + scope_kinds, + &served_facts.billable_classes, + &served_facts.fee_units, + served_facts.audit_kind, + served_facts + .keeps + .iter() + .map(|k| busbar_kernel::plane_driver::ResponseKeep { + mode: k.mode, + kept: k.kept.iter().map(|n| (*n).to_string()).collect(), + denied: k.denied.iter().map(|n| (*n).to_string()).collect(), + }) + .collect(), + ); + facts.bench_below_trip_threshold = bench_below_trip_threshold; + // THE PLANE'S STATED STREAM CEILING (ARCHITECT ruling 2026-10-07, STREAM-CEILING), off its + // tail: the deadline of its streamed units once their route is known; `0` = none. + facts.stream_ceiling_secs = served_facts.stream_ceiling_secs; + if let Some(egress) = egress { + facts.translations = ( + served_facts + .dialects + .iter() + .map(|d| (*d).to_string()) + .collect(), + Arc::new( + egress + .reach + .providers + .iter() + .map(|(name, p)| (name.clone(), p.protocol.clone())) + .collect(), + ), + ); + } + let pools = DoorPools::of(section); + let egress = match egress { + Some(egress) => { + let routes = + crate::root::door_steps::member_routes(section, &pools, served_facts, egress.reach) + .map_err(|e| format!("{instance}: {e}"))?; + let sealed = match egress.reach.models { + Some(models) if models_plane => crate::root::model_egress::compose( + &facts, + models, + plugin.instance(), + Arc::clone(&egress.reach.conns), + &routes, + Arc::clone(&egress.journal), + egress.reach.stream_ceiling_secs, + ) + .map(|(e, shown)| (e, Some(shown))), + _ => crate::root::door_steps::compose_egress( + &facts, + &pools, + plugin.instance(), + Arc::clone(&egress.reach.conns), + &routes, + Arc::clone(&egress.journal), + egress.reach.stream_ceiling_secs, + ) + .map(|e| (e, None)), + }; + let (sealed, shown) = sealed.map_err(|e| format!("{instance}: {e}"))?; + Some((Arc::new(sealed), shown)) + } + None => None, + }; + let (egress, shown) = match egress { + Some((e, shown)) => (Some(e), shown), + None => (None, None), + }; + Ok(DoorLive { + section: section.clone(), + generation, + pools, + egress, + facts, + probes: None, + shown, + }) +} + +/// THE HEALTH PROBES OF ONE GENERATION (K7; 1.5.5 `health:` per provider, `none | dead | active`): +/// for the plane serving the `pools` map whose tail states `TAIL_PROBES`, a probe target over +/// `live`'s sealed egress and `driver`, its probers spawned as a new generation of `schedule` (the +/// previous generation's exit at their next tick). A build without the node has no kernel to run +/// a probe unit on and probes nothing. +fn arm_probes( + driver: &Arc<PlaneDriver>, + served_facts: &crate::root::loader::dispatch::kinds::plane::ServedFacts, + live: &mut DoorLive, + egress: Option<&DoorEgress<'_>>, + schedule: &Arc<busbar_kernel::probe::ProbeSchedule>, +) { + #[cfg(linked_axis_node)] + { + let (Some(sealed), Some(models)) = (&live.egress, egress.and_then(|e| e.reach.models)) + else { + return; + }; + if served_facts.section != busbar_contract::section::RESERVED_POOLS_KEY { + return; + } + let (lanes, members) = models.probe_members(); + if members.is_empty() { + return; + } + let (kernel, keys) = crate::root::plane_node::node().kernel_and_keys(); + let Some(target) = busbar_kernel::plane_driver::PlaneProbes::new( + if served_facts.probes { + busbar_contract::abi::plane::TAIL_PROBES + } else { + 0 + }, + Arc::clone(driver), + Arc::clone(sealed), + kernel, + keys, + (0..lanes) + .map(|i| busbar_contract::DestinationId::new(i as u64)) + .collect(), + ) else { + return; + }; + let target = Arc::new(target); + let members: Vec<busbar_kernel::probe::ProbeMember> = + members.into_iter().map(|(_, m)| m).collect(); + busbar_kernel::probe::spawn_probers(&target, schedule, &members); + live.probes = Some(target); + } + #[cfg(not(linked_axis_node))] + let _ = (driver, served_facts, live, egress, schedule); +} + +/// THE SETTINGS A DOOR PLANE OPENS WITH (spec Part 1 §4: one validated JSON object `{section: +/// value}`): a plane that declares the kernel-owned `pools` map (the previous release's top-level +/// model-serving layout) is handed it and each section it reads beside it (`SECTION_CONSUMED`) +/// that this deployment writes, keyed; a plane that declares a section of its own is handed that +/// section, as its door reads it. +fn settings_of( + facts: &crate::root::loader::dispatch::kinds::plane::ServedFacts, + section: &serde_yaml::Value, + sections: &BTreeMap<&'static str, serde_yaml::Value>, +) -> serde_yaml::Value { + if facts.section != busbar_contract::section::RESERVED_POOLS_KEY { + return section.clone(); + } + let mut keyed = serde_yaml::Mapping::new(); + keyed.insert(facts.section.into(), section.clone()); + for name in &facts.consumed { + if let Some(value) = sections.get(name) { + keyed.insert((*name).into(), value.clone()); + } + } + serde_yaml::Value::Mapping(keyed) +} + +/// THE SECTION A DOOR PLANE'S WALK IS SEALED OVER: its declaring section, or, for a plane that +/// declares a `pools` map and reads the `models` map beside it (the previous release's top-level +/// layout), the uniform model-serving section over both (`{models, pools}`, #49). +fn walked_section( + facts: &crate::root::loader::dispatch::kinds::plane::ServedFacts, + section: &serde_yaml::Value, + sections: &BTreeMap<&'static str, serde_yaml::Value>, +) -> serde_yaml::Value { + use busbar_contract::section::{RESERVED_MODELS_KEY, RESERVED_POOLS_KEY}; + let models = facts + .consumed + .contains(&RESERVED_MODELS_KEY) + .then(|| sections.get(RESERVED_MODELS_KEY)) + .flatten(); + match models { + Some(models) if facts.section == RESERVED_POOLS_KEY => { + let mut uniform = serde_yaml::Mapping::new(); + uniform.insert(RESERVED_MODELS_KEY.into(), models.clone()); + uniform.insert(RESERVED_POOLS_KEY.into(), section.clone()); + serde_yaml::Value::Mapping(uniform) + } + _ => section.clone(), + } +} + /// `open` the plane, generation 1, its settings `section` as JSON, the deployment's `public_url` /// (absent = none stated), its `owned` sections and the `dialects` of the providers its section /// references; the snapshot it published. @@ -2388,11 +2677,18 @@ impl DataRoutes { Some(key) => PrincipalId::new(key.id.as_str()), None => PrincipalId::new(AuthPrincipal(None).actor_id()), }; - let open = self.served.planes[plane] - .snapshot - .claims - .get(claim as usize) - .is_some_and(|c| c.flags & CLAIM_OPEN != 0); + // An anonymous caller is admitted on an open claim; and, on the plane serving the `pools` + // map, on every claim when the deployment's data front door is open (`auth.chain: []` + // without `keys`: 1.5.5 admitted every model request anonymously, its open relay). Any + // other plane's credential claim fails closed (ARCHITECT P3 (a)). + let pools_plane = self.served.planes[plane].live.served_facts.section + == busbar_contract::section::RESERVED_POOLS_KEY; + let open = (pools_plane && app.auth.is_open()) + || self.served.planes[plane] + .snapshot + .claims + .get(claim as usize) + .is_some_and(|c| c.flags & CLAIM_OPEN != 0); let key = gov.key.clone(); let (from_caller, from_rx) = mpsc::channel(SESSION_QUEUE); let (to_tx, to_caller) = mpsc::channel(SESSION_QUEUE); @@ -2462,6 +2758,7 @@ impl DataRoutes { ); let far = DoorFar { egress: live.egress.as_deref(), + shown: live.shown.as_ref(), steps: &steps, unit, credential: door.credential, @@ -2533,11 +2830,18 @@ impl DataRoutes { Some(key) => PrincipalId::new(key.id.as_str()), None => PrincipalId::new(AuthPrincipal(None).actor_id()), }; - let open = self.served.planes[plane] - .snapshot - .claims - .get(claim as usize) - .is_some_and(|c| c.flags & CLAIM_OPEN != 0); + // An anonymous caller is admitted on an open claim; and, on the plane serving the `pools` + // map, on every claim when the deployment's data front door is open (`auth.chain: []` + // without `keys`: 1.5.5 admitted every model request anonymously, its open relay). Any + // other plane's credential claim fails closed (ARCHITECT P3 (a)). + let pools_plane = self.served.planes[plane].live.served_facts.section + == busbar_contract::section::RESERVED_POOLS_KEY; + let open = (pools_plane && app.auth.is_open()) + || self.served.planes[plane] + .snapshot + .claims + .get(claim as usize) + .is_some_and(|c| c.flags & CLAIM_OPEN != 0); let key = gov.key.clone(); // A STREAM ANOTHER FRAMER FRAMES (ARCHITECT 4l): the claim's carrier is answered by a // framer whose claim rows state it rides a stream (an upgrade or a session, Q128 U7); that @@ -2747,7 +3051,11 @@ impl DataRoutes { nesting: Option<Nesting>, ) -> Option<Rendered> { let served = &self.served.planes[plane]; + // The generation this unit is served by, held to its end. let live = served.live.current(); + let started = std::time::Instant::now(); + let claim = arrival.claim; + let app_for_metrics = Arc::clone(&app); let node = self.post.node(); let unit = node.mint(); let arrived = node.arrived(); @@ -2758,7 +3066,7 @@ impl DataRoutes { unit: unit.get(), }; let mut steps = DoorSteps::new( - &served.facts, + &live.facts, &live.pools, node.resolver(), app, @@ -2778,6 +3086,7 @@ impl DataRoutes { } let far = DoorFar { egress: live.egress.as_deref(), + shown: live.shown.as_ref(), steps: &steps, unit, credential, @@ -2785,11 +3094,23 @@ impl DataRoutes { }; let units = served.driver.unit(&steps, &far, caller, arrival, 0); let money = Arc::clone(&served.money); - let facts = served.facts.clone(); + let facts = live.facts.clone(); let late: crate::root::linked::node::Late = Box::new(move || report_of(&money, unit, &facts)); - let ended = node - .drive_borrowed( + // THE UNIT'S REQUEST SPAN, as the previous release's forward opened one around the whole + // walk (at the hot-path level, so the OTLP export carries it): the pool and the dialect the + // request was read in are recorded once the plane has read it. + let span = tracing::span!( + busbar_kernel::observability::HOTPATH_LEVEL, + "forward", + plane = %served.instance, + pool = tracing::field::Empty, + ingress = tracing::field::Empty, + request_id = tracing::field::Empty + ); + let ended = { + use tracing::Instrument as _; + node.drive_borrowed( unit, arrived, &principal, @@ -2799,7 +3120,18 @@ impl DataRoutes { (self.pin)(), nesting.as_ref().map(|n| &n.parent), ) - .await; + .instrument(span.clone()) + .await + }; + if let Some(decoded) = units.decoded() { + if let Some(pool) = decoded.pool.as_ref() { + span.record("pool", String::from_utf8_lossy(pool).as_ref()); + } + if let Some(dialect) = served.dialects.get(decoded.dialect as usize) { + span.record("ingress", *dialect); + } + } + drop(span); // The caller hears how the unit ended (a framed stream closes a cut or failed unit with a // refusal, never as whole). caller.ended(ended); @@ -2810,6 +3142,45 @@ impl DataRoutes { .or_else(|| caller.stated()) .unwrap_or_else(|| refusal_status(ReasonCode::PlanePanic)); served.money.settle_end(unit, status); + // THE REQUEST FAMILIES of the plane serving the `pools` map (the flat card's plane), as the + // previous release's `ingress::finish_inner` emitted them: for every request its dialect + // read (decoded, or refused for its body; a path it does not serve, or a verb it does not + // take, never reached a dialect), under the dialect it arrived in and the pool or model it + // named ("unresolved" when it named none the configuration holds). + if live.facts.plane.is_empty() { + let decoded = units.decoded(); + let counted = decoded.is_some() + || units + .declined_status() + .is_some_and(|s| s != 404 && s != 405); + if counted { + let dialect = decoded.as_ref().map_or_else( + || { + served + .snapshot + .claims + .get(claim as usize) + .map_or(0, |c| u32::from(c.refusal_dialect)) + }, + |d| d.dialect, + ); + let pool = decoded + .and_then(|d| d.pool) + .map(|p| String::from_utf8_lossy(&p).into_owned()) + .filter(|name| { + live.pools.pools().contains_key(name) + || live.pools.entries().iter().any(|e| e == name) + }) + .unwrap_or_else(|| "unresolved".to_string()); + busbar_kernel::telemetry::model_request_finished( + &app_for_metrics, + served.dialects.get(dialect as usize).copied().unwrap_or(""), + &pool, + u16::try_from(status).unwrap_or(u16::MAX), + started.elapsed().as_secs_f64(), + ); + } + } rendered } } @@ -2855,6 +3226,8 @@ fn report_of( /// composed, or no route, the walk is exhausted at once and nothing is dialled. struct DoorFar<'d, 's> { egress: Option<&'d Egress>, + /// What the hooks are shown of the members beside the walk. + shown: Option<&'d crate::root::model_egress::ModelShown>, steps: &'d DoorSteps<'s>, unit: busbar_contract::UnitKey, /// The caller's verified credential, lent to the walk for a passthrough member alone. @@ -2868,16 +3241,26 @@ impl<'d> DoorFar<'d, '_> { .get_or_init(|| { let egress = self.egress?; let routed = self.steps.routed()?; - Some(egress.unit(UnitRoute { + let pool = egress_pool(self.steps.plane(), &routed); + let described = self.shown.and_then(|s| s.described.get(&pool).cloned()); + let mut far = egress.unit(UnitRoute { unit: self.unit, - pool: egress_pool(self.steps.plane(), &routed), + pool, caller_credential: self.credential.clone(), once: self.steps.once(), // The plane's `ROUTE_STREAM`: the stream ceiling bounds the send, and the // plane's stated one, if any, is the unit's deadline. wants_stream: self.steps.streamed(), + affinity: self.steps.affinity(), ..UnitRoute::default() - })) + }); + if let Some(described) = described { + far = far.described(described); + } + if let Some(shown) = self.shown { + far = far.signals(Arc::clone(&shown.signals)); + } + Some(far) }) .as_ref() } @@ -2888,6 +3271,7 @@ fn spent() -> Pick { Pick::Exhausted { status: refusal_status(ReasonCode::NoDestination), retry_after: None, + detail: busbar_kernel_egress::wire::DETAIL_OVERLOADED, } } @@ -2938,6 +3322,28 @@ impl FarEnd for DoorFar<'_, '_> { None => false, } } + + // THE WALK'S FACTS FOR THE HOOK STAGE, as the unit's egress walk states them: its candidates, + // what it has left and why it last failed, and the hooks' constraint handed to it. Without + // these the kernel's hooks were shown no candidate and their constraint never reached the + // walk on the door path. + fn remaining(&self, token: &Pass<Route>) -> Option<usize> { + self.far().and_then(|far| far.remaining(token)) + } + + fn failure(&self, token: &Pass<Route>) -> Option<&'static str> { + self.far().and_then(|far| far.failure(token)) + } + + fn candidates(&self, token: &Pass<Route>) -> Option<busbar_kernel::plane_driver::Candidates> { + self.far().and_then(|far| far.candidates(token)) + } + + fn constrain(&self, token: &Pass<Route>, constraint: busbar_kernel::plane_driver::Constraint) { + if let Some(far) = self.far() { + far.constrain(token, constraint); + } + } } // ── a session's caller side over today's upgrade (TRANSITIONAL) ────────────────────────────────── @@ -3406,7 +3812,7 @@ pub(crate) mod planes_tests; // `root-mcp` leg's loop cells, under the linked plane-door axis), each gated item by item inside, so // either plane's switch alone still compiles its own. Its MCP cells reach the door_steps helpers // that need the default build's linked auth rows (gated with the node-axis plane that carries that -// build). +// build), and the capability cells of the door serving the `pools` map. #[cfg(all(test, linked_axis_node))] #[path = "tests/serve_tests.rs"] mod door_tests; @@ -3415,6 +3821,10 @@ mod door_tests; #[path = "tests/serve_money.rs"] mod money_tests; +#[cfg(all(test, linked_fold_on_driver, linked_axis_node))] +#[path = "tests/serve_hook_seats.rs"] +mod hook_seat_tests; + #[cfg(all(test, linked_axis_node))] #[path = "tests/serve_framed.rs"] mod framed_tests; diff --git a/crates/busbar/src/root/tests/door_steps.rs b/crates/busbar/src/root/tests/door_steps.rs index fc976e974d..675d3401ad 100644 --- a/crates/busbar/src/root/tests/door_steps.rs +++ b/crates/busbar/src/root/tests/door_steps.rs @@ -364,7 +364,7 @@ fn styled() -> crate::root::loader::dispatch::kinds::plane::ServedFacts { crate::root::loader::dispatch::kinds::plane::ServedFacts { need_auths: vec![(DIRECTION_OUTBOUND, "bearer")], dialects: vec!["d"], - dialect_auth: vec![(0, "bearer")], + dialect_auth: vec![(0, "bearer", b"")], ..Default::default() } } @@ -431,6 +431,7 @@ fn resolve_upgrading( auths: std::sync::Arc::new(auths), conns, stream_ceiling_secs: 1, + models: None, upgrades, }; super::member_routes(§ion, &DoorPools::of(§ion), served, &reach) @@ -585,6 +586,65 @@ fn a_member_that_cannot_be_reached_refuses_the_load_naming_it() { ); } +/// A MEMBER'S BINDING IS OPENED WITH ITS DIALECT'S PARAMETERS (ARCHITECT RULING 2026-10-03, +/// Q-L6-AUTHPARAMS): the params object, a host directive resolved from the provider's base URL +/// (the label after a named one, when it reads as a dashed name ending in a number), the +/// provider's own settings over it; no params hands the provider's settings as they are. +#[test] +fn a_members_binding_is_opened_with_its_dialects_parameters_under_the_providers_own() { + use super::sealed_settings; + let params = br#"{"service":"svc","content_type":"application/json","region":{"host_label_after":["svc-runtime","svc-runtime-fips","svc"],"default":"us-east-1","unread":"w"}}"#; + let region = |base_url: &str| { + sealed_settings(params, base_url, &serde_json::json!({})).expect("the parameters seal") + ["region"] + .clone() + }; + for (base_url, want) in [ + ("https://svc-runtime.eu-west-1.amazonaws.com", "eu-west-1"), + ( + "https://svc-runtime-fips.us-gov-west-1.amazonaws.com/", + "us-gov-west-1", + ), + ( + "https://vpce-0abc-1xyz.svc-runtime.ap-southeast-2.vpce.amazonaws.com:443/x", + "ap-southeast-2", + ), + ("https://svc.ca-central-1.amazonaws.com", "ca-central-1"), + // No label reads: the default. + ("https://svc-runtime.amazonaws.com", "us-east-1"), + ("http://127.0.0.1:9000", "us-east-1"), + ("https://svc-runtime.eu-west.amazonaws.com", "us-east-1"), + ] { + assert_eq!(region(base_url), serde_json::json!(want), "{base_url}"); + } + let sealed = sealed_settings( + params, + "https://svc-runtime.eu-west-1.amazonaws.com", + &serde_json::json!({ "service": "own", "scope": "s" }), + ) + .expect("the parameters seal"); + assert_eq!( + sealed["service"], "own", + "the provider's own setting stands" + ); + assert_eq!(sealed["scope"], "s"); + assert_eq!(sealed["content_type"], "application/json"); + assert!( + sealed.get("region_unread").is_none(), + "a host whose region reads hands on no warning" + ); + let defaulted = sealed_settings(params, "http://127.0.0.1:9000", &serde_json::json!({})) + .expect("the parameters seal"); + assert_eq!(defaulted["region"], "us-east-1"); + assert_eq!( + defaulted["region_unread"], "w", + "the warning travels with the default" + ); + let own = serde_json::json!({ "token_url": "https://t" }); + assert_eq!(sealed_settings(b"", "https://x", &own), Ok(own)); + assert!(sealed_settings(b"[1]", "https://x", &serde_json::json!({})).is_err()); +} + // ── the minted credential of a member bound under an OAuth grant (TODO row 22) ───────────────── /// A connection table standing in for the token endpoint: framed needs, each declaration and each @@ -825,6 +885,7 @@ async fn a_member_under_an_oauth_grant_presents_its_minted_then_refreshed_bearer auths: std::sync::Arc::new(linked_mint_plugin(&table)), conns: std::sync::Arc::new(busbar_core_connector::Connector::new()), stream_ceiling_secs: 1, + models: None, upgrades: Vec::new(), }; let section: serde_yaml::Value = @@ -961,6 +1022,7 @@ fn jwt_bearer_routes( auths: std::sync::Arc::new(linked_mint_plugin(table)), conns: std::sync::Arc::new(busbar_core_connector::Connector::new()), stream_ceiling_secs: 1, + models: None, upgrades: Vec::new(), }; let section: serde_yaml::Value = @@ -1182,6 +1244,12 @@ pub(crate) mod tool_door { .unwrap_or_else(|| panic!("fixtures/tool_door_wire.txt has no `{key}` row")) } + /// A fixture sentence with its `{PLANE}` filled: the plane's own key (`plane_key`), upper-cased, + /// as the plane's sentences spell it. + pub(crate) fn sentence(key: &str) -> String { + surface(key).replace("{PLANE}", &surface("plane_key").to_ascii_uppercase()) + } + /// The section the tool door owns beside its settings: its endpoint block, as the fixture /// states it (the root spells no plane's section). pub(crate) fn endpoint_section() -> &'static str { @@ -1674,6 +1742,7 @@ pub(crate) mod tool_door { secrets: &secrets, auths: Arc::new(auths), conns: Arc::clone(&connector) as Arc<dyn PollConns>, + models: None, stream_ceiling_secs: 600, upgrades: Vec::new(), }; @@ -3672,8 +3741,12 @@ pub(crate) mod door_boundary { door_checked > 0, "the door's half of the boundary was asserted" ); + // The kernel's own guarded routes (`/stats`, `/v1/models`, `/v1beta/models`): the convenience + // body-ingress routes are a plane's and mount only where a plane installs body ingress (the + // kernel owns no route that exists for one plugin, BUSBAR-1.6.0.md l.4093), which this rig + // does not; a door claims its own paths and is walked in the door arm above. assert!( - checked >= 4, + checked >= 3, "the walk covered only {checked} guarded core routes — it is not seeing the router" ); for path in &boot_plugin_paths { diff --git a/crates/busbar/src/root/tests/fixtures/tool_door_wire.txt b/crates/busbar/src/root/tests/fixtures/tool_door_wire.txt index d410d5fb07..8529939751 100644 --- a/crates/busbar/src/root/tests/fixtures/tool_door_wire.txt +++ b/crates/busbar/src/root/tests/fixtures/tool_door_wire.txt @@ -39,3 +39,9 @@ plane_key = mcp # resource noun AuditRow::tool names a tool by: `<noun>:<published name>`). audit_action_tool_call = mcp_tool.call audit_resource_tool = mcp_tool +# The plane's refusal words for an exhausted walk, byte for byte as predev served them (tool_door.rs +# RefusalSlot; the `*_in_predevs_bytes` cells in serve_tests.rs): a tripped server's sentence +# ({server} and {wait} filled by the cell), and a task continuation the walk refused. {PLANE} is the +# plane's own key above (`plane_key`), upper-cased, as its sentences spell it. +tripped_server_message = {PLANE} server `{server}` is unavailable: its circuit breaker is open after repeated failures; busbar did not dispatch this call. Retry after {wait}s. +continuation_refused_error = {"code":-32603,"message":"the {PLANE} upstream call failed: breaker_open"} diff --git a/crates/busbar/src/root/tests/secret_source.rs b/crates/busbar/src/root/tests/secret_source.rs index edffce24e7..35c1087706 100644 --- a/crates/busbar/src/root/tests/secret_source.rs +++ b/crates/busbar/src/root/tests/secret_source.rs @@ -220,6 +220,8 @@ fn driven() -> Vec<String> { )]; #[cfg(linked_section_decisions)] names.push(super::door_tests::decisions_name()); + #[cfg(linked_fold_on_driver)] + names.push(statement_name(super::hook_seat_tests::pools_door())); names } @@ -429,6 +431,90 @@ async fn decisions_a_provider_key_gone_by_an_apply_reaches_the_caller_without_it assert_clean(instance, status, &headers, &body, &secret); } +/// THE POOLS DOOR, COMPOSE TIME: a provider whose `api_key` does not resolve refuses the boot with the +/// operator's text, which names the source; nothing is served to a caller. +#[cfg(linked_fold_on_driver)] +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn pools_door_an_unresolved_provider_key_refuses_the_boot_naming_its_source() { + use super::hook_seat_tests::{far_end_answering, try_rig, RigOpts, ANSWER}; + let _one = PUBLISHING.lock().await; + let instance = "pools-key-compose"; + let _published = Published(instance); + let path = missing(instance); + let secret = SecretRef::file(path.clone()); + refused_naming_its_source(&secret); + let far = far_end_answering(200, ANSWER).await; + let api_key = format!("{{ file: '{path}' }}"); + let Err(refusal) = try_rig( + instance, + RigOpts { + members: &[(far.port, 1)], + api_key: Some(&api_key), + ..RigOpts::default() + }, + ) + .await + else { + panic!("a provider key that does not resolve refuses the composition"); + }; + println!("SECRET-SOURCE {instance}: compose refused (operator): {refusal}"); + assert!( + !source_in(&refusal, &secret).is_empty(), + "the operator's refusal names the source: {refusal}" + ); + assert_eq!( + far.hits.load(std::sync::atomic::Ordering::SeqCst), + 0, + "nothing reached the far end" + ); +} + +/// THE POOLS DOOR, ACROSS AN APPLY: a provider served while its `api_key` file was there, the file then +/// gone and the configuration applied: the door keeps the generation it serves, and the caller is +/// served by it, its render clean of the source. +#[cfg(linked_fold_on_driver)] +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn pools_door_a_provider_key_gone_by_an_apply_reaches_the_caller_without_its_source() { + use super::hook_seat_tests::{far_end_answering, rig, RigOpts, ANSWER}; + let _one = PUBLISHING.lock().await; + let instance = "pools-key-apply"; + let _published = Published(instance); + let path = missing(instance); + std::fs::write(&path, "sk-secret-source-test").expect("the credential file"); + let secret = SecretRef::file(path.clone()); + let far = far_end_answering(200, ANSWER).await; + let api_key = format!("{{ file: '{path}' }}"); + let rig = rig( + instance, + RigOpts { + members: &[(far.port, 1)], + api_key: Some(&api_key), + ..RigOpts::default() + }, + ) + .await; + let (status, _, body) = rig.chat().await; + assert_eq!( + status, + 200, + "served while it resolves: {}", + String::from_utf8_lossy(&body) + ); + std::fs::remove_file(&path).expect("the credential file goes"); + refused_naming_its_source(&secret); + let before = rig.generation(); + rig.apply(); + assert_eq!( + rig.generation(), + before, + "the door keeps the generation it serves" + ); + let (status, headers, body) = rig.chat().await; + let status = StatusCode::from_u16(status).expect("a status"); + assert_clean(instance, status, &headers, &body, &secret); + assert_eq!(status, StatusCode::OK, "the kept generation serves"); +} + // The scan's fixed-class half and its kept RED arm (status row 127). #[path = "secret_source_class.rs"] mod class; diff --git a/crates/busbar/src/root/tests/serve_hook_seats.rs b/crates/busbar/src/root/tests/serve_hook_seats.rs new file mode 100644 index 0000000000..b6b67ad79b --- /dev/null +++ b/crates/busbar/src/root/tests/serve_hook_seats.rs @@ -0,0 +1,1126 @@ +// SPDX-License-Identifier: Apache-2.0 +// Copyright (C) 2026 Busbar Inc and contributors + +//! THE SEAT ORDER OF THE REQUEST-STAGE HOOKS, ON THE DOOR (ported from the predev-only engine test +//! `hook_seat_order_tests`, BUSBAR-1.6.0.md Part 5 F25: before any engine deletion every test names +//! its new home). The plane serving the `pools` map, served through its door end to end: a keyed +//! caller behind the data listener's auth gate, admitted and charged by the door's money steps, its +//! unit's hooks bound by the kernel's hook stage (the global rewrite, the request-stage tap, a gate +//! and the candidate tap), its far end a real loopback upstream through the process connector. The +//! admission door charges the request first, then the global rewrite runs, then the request-stage +//! tap observes the REWRITTEN request, then the gate decides, and only after the gate does the +//! candidate tap fire. A gate reject refunds the billable request (the fee base) but keeps the +//! admission count. +//! +//! The root names no plane and no dialect: the dialects these cells speak are the plane's own door +//! words (`DRIVER_WORDS`, its `tests/on_driver/words.txt`), read as data through [`word`]. +//! +//! Every seat is proven structurally rather than by wall-clock order where the seat is a detached +//! task (taps are fire-and-forget): the rewrite hook reads the ledger AT the moment it runs, the +//! request tap's payload carries the rewritten prompt, and a rejecting gate stops everything seated +//! after it — so a request tap that still arrives was seated before the gate and a candidate tap +//! that never arrives was seated after it. + +use std::collections::{BTreeMap, HashMap}; +use std::sync::{Arc, Mutex}; + +use busbar_contract::conn::{DeclaredConns, PollConns}; +use busbar_contract::hooks::{ + Candidate, PolicyResult, RewriteReply, RoutingContext, RoutingDecision, RoutingPolicy, + RoutingRequest, TransformOutcome, +}; +use busbar_kernel::cost::CostModel; +use busbar_kernel::governance::signing::{TokenSigner, DEFAULT_KID}; +use busbar_kernel::governance::{GovState, MemoryStore, NewKeySpec}; +use busbar_kernel::hooks::ResolvedPolicy; +use busbar_kernel::plane_driver::{EndPost, PlaneMoney}; +use tokio::io::{AsyncReadExt, AsyncWriteExt}; + +use super::planes_tests::{composed_services, Published, PUBLISHING}; +use super::{compose_planes, door_routes, DoorEgress, HookStage}; +use crate::root::door_steps::{kernel_sections, provider_routes, DoorReach, OutboundAuths}; +use crate::root::loader::dispatch::kinds::plane::Plane; +use crate::root::loader::dispatch::{ + load_linked, Bind, DispatchConfig, Dispatcher, LinkedRow, NoSink, +}; +use crate::root::plane_node::{Node, NodeEndPost}; + +// The plane the node is handed, as the manifest's linked table names it (the `node` axis): the +// legacy row's test seams the registry rows that own the `pools:`/`models:` sections come from. +include!(concat!(env!("OUT_DIR"), "/node_plane.rs")); + +// The words of the plane under proof that these cells read as data (`DRIVER_WORDS`: the plane's own +// `tests/on_driver/words.txt`, from the manifest's driver-proofs row). +include!(concat!(env!("OUT_DIR"), "/driver_words.rs")); + +/// One of the plane's door words (`key = value` per line, `#` a comment): what the root's cells +/// would otherwise have to spell, a dialect of the plane under proof. +pub(super) fn word(key: &str) -> &'static str { + DRIVER_WORDS + .lines() + .map(str::trim) + .filter(|l| !l.is_empty() && !l.starts_with('#')) + .find_map(|l| { + let (k, v) = l.split_once('=')?; + (k.trim() == key).then(|| v.trim()) + }) + .unwrap_or_else(|| panic!("the plane's door words have no `{key}` row")) +} + +/// THE LINKED DOOR THAT SERVES THE `pools` MAP, found by what its Statement declares (the composition +/// root names no plane): every linked plane door bound on a probe dispatcher of its own, the one +/// whose declaring section is `pools` kept. +pub(super) fn pools_door() -> busbar_contract::abi::mechanism::door::DoorFn { + crate::LINKED + .plane_doors + .iter() + .copied() + .find(|door| { + let probe = Dispatcher::new(DispatchConfig::default()); + let row = LinkedRow::of(*door).expect("a linked door states itself"); + load_linked::<Plane>( + &row, + Bind { + instance: Arc::from("pools-door-probe"), + max_inflight_cap: 64, + sink: Arc::new(NoSink), + dispatcher: probe.adopter(), + conns: crate::root::loader::dispatch::ConnTable::Probe, + }, + ) + .expect("a linked door binds") + .served() + .section + == busbar_contract::section::RESERVED_POOLS_KEY + }) + .expect("the fold switch links the door serving the `pools` map") +} + +/// The card history the served unit is pinned to at its door: one entry, no price. +static CARD: std::sync::LazyLock<crate::root::kernel::RootHistory> = + std::sync::LazyLock::new(|| { + let holder = crate::root::kernel::RootHistory::default(); + holder.apply( + crate::root::kernel::card_from_config( + std::iter::empty::<(&str, busbar_contract::billing::RawTierRates)>(), + 0, + true, + ), + 1_000, + ); + holder + }); + +/// The marker the rewrite hook plants in the prompt; a tap payload carrying it saw the rewrite. +pub(super) const REWRITTEN: &str = "rewritten-by-the-global-rewrite-seat"; + +/// The far end's answer: a chat completion in the door words' `dialect`. +pub(super) const ANSWER: &str = r#"{"id":"chatcmpl-1","object":"chat.completion","created":0,"model":"m0","choices":[{"index":0,"message":{"role":"assistant","content":"hi"},"finish_reason":"stop"}],"usage":{"prompt_tokens":1,"completion_tokens":1,"total_tokens":2}}"#; + +/// One probe policy playing one seat: it appends its seat name to the shared log when it runs and +/// (for the rewrite seat) samples the key's admission count at that instant. +struct SeatProbe { + seat: &'static str, + log: Arc<Mutex<Vec<String>>>, + /// Rewrite seat only: reads the key's `requests` count when the rewrite runs. + requests_now: Option<Box<dyn Fn() -> u64 + Send + Sync>>, + /// Gate seat only: the reject the gate answers with. + reject: Option<(u16, &'static str)>, + /// Tap seats: the last delivered projection. + last_payload: Mutex<Option<Vec<u8>>>, + /// Gate seat: each candidate the last decide was shown, as one line. + candidates: Mutex<Vec<String>>, +} + +impl SeatProbe { + fn new(seat: &'static str, log: &Arc<Mutex<Vec<String>>>) -> Self { + Self { + seat, + log: log.clone(), + requests_now: None, + reject: None, + last_payload: Mutex::new(None), + candidates: Mutex::new(Vec::new()), + } + } +} + +#[async_trait::async_trait] +impl RoutingPolicy for SeatProbe { + async fn decide( + &self, + _req: &RoutingRequest<'_>, + candidates: &[Candidate<'_>], + _ctx: &RoutingContext<'_>, + _budget: std::time::Duration, + ) -> PolicyResult { + self.log.lock().unwrap().push(self.seat.to_string()); + *self.candidates.lock().unwrap() = candidates + .iter() + .map(|c| { + format!( + "{} tier={:?} tags={:?} latency={} avail={} budget={:?} signals={:?}", + c.model, + c.tier, + c.tags, + c.latency_ms.is_some(), + c.available_concurrency > 0, + c.budget_remaining, + c.signals + .iter() + .map(|(s, v)| format!( + "{s:?}={}", + match v { + busbar_contract::SignalValue::Str(t) => t.to_string(), + other => format!( + "{}", + matches!( + other, + busbar_contract::SignalValue::F64(_) + | busbar_contract::SignalValue::U64(_) + ) + ), + } + )) + .collect::<Vec<_>>() + ) + }) + .collect(); + Ok(match self.reject { + Some((status, message)) => RoutingDecision::Reject { + status, + message: message.to_string(), + }, + None => RoutingDecision::Abstain, + }) + } + fn name(&self) -> &'static str { + self.seat + } + async fn transform( + &self, + _req: &RoutingRequest<'_>, + _budget: std::time::Duration, + ) -> TransformOutcome { + let requests = self.requests_now.as_ref().map(|f| f()); + self.log + .lock() + .unwrap() + .push(format!("{}:requests={:?}", self.seat, requests)); + TransformOutcome::Rewrite(RewriteReply { + messages: vec![serde_json::json!({"role": "user", "content": REWRITTEN})], + tools: vec![], + }) + } + async fn notify( + &self, + tap: std::sync::Arc<busbar_contract::abi::host::hook::NotifyFrame>, + _budget: std::time::Duration, + ) { + let projection = serde_json::to_vec(&tap.projection_json()).expect("the tap's JSON"); + self.log.lock().unwrap().push(self.seat.to_string()); + *self.last_payload.lock().unwrap() = Some(projection); + } +} + +fn gate(policy: Arc<dyn RoutingPolicy>) -> ResolvedPolicy { + ResolvedPolicy::Policy { + policy, + on_error: busbar_kernel::config::PolicyOnError::default(), + on_error_chain: Vec::new(), + timeout: std::time::Duration::from_millis(500), + send_prompt: false, + send_user: false, + on_empty: busbar_kernel::config::PolicyOnError::Reject, + } +} + +/// Poll a tap probe until its detached delivery lands (bounded), or return `None`. +async fn delivered(tap: &SeatProbe, budget_ms: u64) -> Option<Vec<u8>> { + for _ in 0..(budget_ms / 10) { + if let Some(p) = tap.last_payload.lock().unwrap().clone() { + return Some(p); + } + tokio::time::sleep(std::time::Duration::from_millis(10)).await; + } + None +} + +/// A FAR END ON LOOPBACK: every request it reads is answered with its status and body; it counts +/// the requests it served and keeps each one's `authorization` field. +pub(super) struct FarEnd { + /// Its port. + pub port: u16, + /// The requests it served. + pub hits: Arc<std::sync::atomic::AtomicUsize>, + /// Each served request's `authorization` field, as received. + pub authorizations: Arc<Mutex<Vec<String>>>, +} + +impl FarEnd { + /// The requests it served so far. + pub fn served(&self) -> usize { + self.hits.load(std::sync::atomic::Ordering::SeqCst) + } +} + +/// What a scripted far end writes for each request it reads: its raw head, then each piece after +/// its delay, then (when `finish` is set) the closing bytes; it then closes the connection, so a +/// script whose head promises more than it writes is a far end that CUT its answer. +#[derive(Clone)] +pub(super) struct Script { + /// The status line and head fields, through the blank line. + pub head: String, + /// Each piece of the body, after its delay in milliseconds. + pub pieces: Vec<(u64, Vec<u8>)>, + /// The bytes that complete the body (a chunked body's last chunk); `None` cuts it. + pub finish: Option<Vec<u8>>, +} + +/// One chunk of a chunked body. +pub(super) fn chunk(bytes: &[u8]) -> Vec<u8> { + let mut out = format!("{:x}\r\n", bytes.len()).into_bytes(); + out.extend_from_slice(bytes); + out.extend_from_slice(b"\r\n"); + out +} + +/// A far end on loopback answering every request by `script`. +pub(super) async fn far_end_scripted(script: Script) -> FarEnd { + let listener = tokio::net::TcpListener::bind("127.0.0.1:0") + .await + .expect("a loopback port"); + let port = listener.local_addr().expect("its address").port(); + let hits = Arc::new(std::sync::atomic::AtomicUsize::new(0)); + let authorizations = Arc::new(Mutex::new(Vec::new())); + let (served, seen) = (Arc::clone(&hits), Arc::clone(&authorizations)); + tokio::spawn(async move { + while let Ok((mut socket, _)) = listener.accept().await { + let (served, seen, script) = (Arc::clone(&served), Arc::clone(&seen), script.clone()); + tokio::spawn(async move { + let Some(head) = read_request(&mut socket).await else { + return; + }; + served.fetch_add(1, std::sync::atomic::Ordering::SeqCst); + seen.lock().unwrap().push(head); + if socket.write_all(script.head.as_bytes()).await.is_err() { + return; + } + for (delay, piece) in &script.pieces { + tokio::time::sleep(std::time::Duration::from_millis(*delay)).await; + if socket.write_all(piece).await.is_err() { + return; + } + let _ = socket.flush().await; + } + if let Some(finish) = &script.finish { + let _ = socket.write_all(finish).await; + } + let _ = socket.shutdown().await; + }); + } + }); + FarEnd { + port, + hits, + authorizations, + } +} + +/// Read one request off `socket` through its body; its `authorization` field (empty when none), +/// or `None` when the connection closed first. +async fn read_request(socket: &mut tokio::net::TcpStream) -> Option<String> { + let mut buf = vec![0u8; 16 * 1024]; + let mut got = Vec::new(); + loop { + let n = socket.read(&mut buf).await.ok()?; + if n == 0 { + return None; + } + got.extend_from_slice(&buf[..n]); + let text = String::from_utf8_lossy(&got).to_string(); + if let Some(at) = text.find("\r\n\r\n") { + let field = |name: &str| { + text[..at].lines().find_map(|l| { + let (n, v) = l.split_once(':')?; + n.eq_ignore_ascii_case(name).then(|| v.trim().to_string()) + }) + }; + let length = field("content-length") + .and_then(|v| v.parse::<usize>().ok()) + .unwrap_or(0); + if got.len() >= at + 4 + length { + return Some(field("authorization").unwrap_or_default()); + } + } + } +} + +/// A far end on loopback answering every request with `status` and `body` (JSON). +pub(super) async fn far_end_answering(status: u16, body: &'static str) -> FarEnd { + let listener = tokio::net::TcpListener::bind("127.0.0.1:0") + .await + .expect("a loopback port"); + let port = listener.local_addr().expect("its address").port(); + let hits = Arc::new(std::sync::atomic::AtomicUsize::new(0)); + let authorizations = Arc::new(Mutex::new(Vec::new())); + let (served, seen) = (Arc::clone(&hits), Arc::clone(&authorizations)); + tokio::spawn(async move { + while let Ok((mut socket, _)) = listener.accept().await { + let (served, seen) = (Arc::clone(&served), Arc::clone(&seen)); + tokio::spawn(async move { + let mut buf = vec![0u8; 16 * 1024]; + let mut got = Vec::new(); + let head = loop { + let Ok(n) = socket.read(&mut buf).await else { + return; + }; + if n == 0 { + return; + } + got.extend_from_slice(&buf[..n]); + let text = String::from_utf8_lossy(&got).to_string(); + if let Some(at) = text.find("\r\n\r\n") { + let field = |name: &str| { + text[..at].lines().find_map(|l| { + let (n, v) = l.split_once(':')?; + n.eq_ignore_ascii_case(name).then(|| v.trim().to_string()) + }) + }; + let length = field("content-length") + .and_then(|v| v.parse::<usize>().ok()) + .unwrap_or(0); + if got.len() >= at + 4 + length { + break field("authorization").unwrap_or_default(); + } + } + }; + served.fetch_add(1, std::sync::atomic::Ordering::SeqCst); + seen.lock().unwrap().push(head); + let reason = if status == 200 { "OK" } else { "Error" }; + let reply = format!( + "HTTP/1.1 {status} {reason}\r\ncontent-type: application/json\r\n\ + content-length: {}\r\nconnection: close\r\n\r\n{body}", + body.len() + ); + let _ = socket.write_all(reply.as_bytes()).await; + let _ = socket.shutdown().await; + }); + } + }); + FarEnd { + port, + hits, + authorizations, + } +} + +/// A far end on loopback answering every request with [`ANSWER`]. +async fn far_end() -> u16 { + far_end_answering(200, ANSWER).await.port +} + +/// The key's ledger as the test reads it: (admission count, derived spend in cents, the durable +/// row's admission count, the durable row's billable count). +pub(super) type Ledger = (u64, i64, u64, u64); + +/// What a run left: the caller's status, the seat log, the two tap probes and the ledger reader. +struct Ran { + status: u16, + log: Arc<Mutex<Vec<String>>>, + request_tap: Arc<SeatProbe>, + candidate_tap: Arc<SeatProbe>, + read: Arc<dyn Fn() -> Ledger + Send + Sync>, +} + +/// HOW A RIG IS BUILT: the far ends of its pool `p`'s members (each a model `m<i>` on a provider +/// of its own, at `weight`), the gate's verdict, and the key's all-time budget. +#[derive(Clone, Copy, Default)] +pub(super) struct RigOpts<'a> { + /// Each member's far-end port and weight, in order. + pub members: &'a [(u16, u32)], + /// The global gate rejects with 451. + pub reject_at_gate: bool, + /// The key's group holds this many cents a day (a flat 1-cent fee per request). + pub budget_cents: Option<u64>, + /// How many of the members (the first ones) pool `p` holds; the rest are models no pool + /// names. `None`: all of them. + pub pooled: Option<usize>, + /// The pools the key may reach (`None`: every pool). + pub allowed_pools: Option<&'a [&'a str]>, + /// The members' dialect (`None`: the door words' `dialect`). + pub dialect: Option<&'static str>, + /// Each pool member states a tier `t<i>` and a tag `g<i>`, and the generation declares the + /// three candidate catalog signals (breaker state, error rate, p95 latency). + pub described: bool, + /// Pool `p`'s `failover.timeout_secs` (`None`: the default). + pub failover_timeout_secs: Option<u64>, + /// The node's stream ceiling, in seconds (`None`: 600). + pub stream_ceiling_secs: Option<u64>, + /// Pool `p`'s `affinity.header_name` (`None`: the pool states no affinity block). + pub affinity_header: Option<&'a str>, + /// Each member's lifetime request budget (`None`: unlimited). + pub lane_budget: Option<i64>, + /// Every provider's `api_key`, a secret reference as written (YAML) (`None`: a credential file + /// of the rig's own, gone once the door composed). + pub api_key: Option<&'a str>, +} + +/// THE POOLS DOOR, SERVED END TO END, as production composes it: the data router built with the +/// door's claims, a keyed caller's token, the generation's App and the node's book. +pub(super) struct DoorRig { + /// The data router. + pub router: axum::Router, + /// The caller's bearer token. + pub token: String, + /// The generation. + pub app: Arc<busbar_kernel::state::App>, + /// The node's book: every unit's line and audit record. + pub book: Arc<std::sync::Mutex<crate::root::durability::Durability>>, + /// The key's id. + pub key_id: String, + /// The served door's generations, as a config apply refreshes them. + appliers: super::DoorAppliers, + /// A config apply of the same deployment onto the door ([`DoorAppliers::refresh_models`], as + /// the root's apply hook runs it), its secrets resolved anew. + reapply: Arc<dyn Fn() + Send + Sync>, + log: Arc<Mutex<Vec<String>>>, + request_tap: Arc<SeatProbe>, + candidate_tap: Arc<SeatProbe>, + gate_seat: Arc<SeatProbe>, + read: Arc<dyn Fn() -> Ledger + Send + Sync>, + tokens: Arc<dyn Fn() -> u64 + Send + Sync>, +} + +impl DoorRig { + /// Apply the deployment again, as an operator's config apply does: the door refreshes onto it, + /// or keeps the generation it serves. + pub fn apply(&self) { + (self.reapply)(); + } + + /// The generation the door serves. + pub fn generation(&self) -> u64 { + self.appliers + .0 + .first() + .expect("one door") + .current() + .generation + } + + /// Each candidate the pool's ranking policy was last shown, one line each. + pub fn gate_saw(&self) -> Vec<String> { + self.gate_seat.candidates.lock().unwrap().clone() + } + + /// One request through the door as the keyed caller: its status, head and body. + pub async fn send( + &self, + method: &str, + path: &str, + body: Option<serde_json::Value>, + ) -> (u16, axum::http::HeaderMap, Vec<u8>) { + let response = self.open(method, path, body, &[]).await; + let status = response.status().as_u16(); + let head = response.headers().clone(); + let bytes = axum::body::to_bytes(response.into_body(), 1 << 20) + .await + .expect("the body") + .to_vec(); + (status, head, bytes) + } + + /// One request through the door as the keyed caller, with `fields` beside its own: the + /// response, its body unread. + pub async fn open( + &self, + method: &str, + path: &str, + body: Option<serde_json::Value>, + fields: &[(&str, &str)], + ) -> axum::response::Response { + use tower::ServiceExt as _; + let mut req = axum::http::Request::builder() + .method(method) + .uri(path) + .header("authorization", format!("Bearer {}", self.token)); + for (name, value) in fields { + req = req.header(*name, *value); + } + if body.is_some() { + req = req.header("content-type", "application/json"); + } + let req = req + .body(match body { + Some(b) => axum::body::Body::from(b.to_string()), + None => axum::body::Body::empty(), + }) + .expect("a request"); + self.router + .clone() + .oneshot(req) + .await + .expect("the router answers") + } + + /// The tokens on the key's ledger once they are nonzero or the wait is spent (the accrual is + /// write-behind). + pub async fn tokens_after(&self) -> u64 { + let mut tokens = (self.tokens)(); + for _ in 0..100 { + if tokens != 0 { + break; + } + tokio::time::sleep(std::time::Duration::from_millis(10)).await; + tokens = (self.tokens)(); + } + tokens + } + + /// One chat completion on pool `p`. + pub async fn chat(&self) -> (u16, axum::http::HeaderMap, Vec<u8>) { + self.send( + "POST", + "/v1/chat/completions", + Some(serde_json::json!({"model": "p", "max_tokens": 16, + "messages": [{"role": "user", "content": "the original prompt"}]})), + ) + .await + } + + /// The key's ledger, once its last unit's money settled (bounded). + pub async fn ledger_after(&self, billable: u64) -> Ledger { + let mut ledger = (self.read)(); + for _ in 0..200 { + if ledger.3 >= billable { + break; + } + tokio::time::sleep(std::time::Duration::from_millis(10)).await; + ledger = (self.read)(); + } + ledger + } + + /// The audit records the node's book sealed so far. + pub fn audit_records(&self) -> usize { + self.book + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) + .audit_records + .len() + } +} + +/// BUILD THE RIG: the door serving the `pools` map, linked and bound through the loader's one load +/// on the process connector, over `opts`' far ends, its egress the model-serving walk over the +/// kernel's lane cells, its hooks the kernel's stage with the four probed seats installed. +pub(super) async fn rig(instance: &'static str, opts: RigOpts<'_>) -> DoorRig { + try_rig(instance, opts) + .await + .unwrap_or_else(|e| panic!("the door serving the pools map composes: {e}")) +} + +/// [`rig`], the composition's refusal answered rather than panicked on: what the root's boot dies +/// on. +pub(super) async fn try_rig(instance: &'static str, opts: RigOpts<'_>) -> Result<DoorRig, String> { + // The registry rows that own the `pools:`/`models:` sections the deployment writes. + node_plane::testkit::install_test_seams(); + busbar_kernel::snapshot::init(); + let judge = crate::root::connector::guard_for(&busbar_kernel::config::Destinations { + block_private_addresses: false, + ..Default::default() + }) + .expect("the guard"); + let connector = busbar_core_connector::process::build( + || { + crate::root::connector::entries( + crate::LINKED_TRANSPORT_DOORS, + &busbar_contract::transport::TransportSettings::default(), + ) + }, + judge, + &[], + Arc::new(|_| {}), + busbar_core_connector::pool::PoolPosture::NONE, + ) + .expect("the connector builds"); + + // THE POOLS DOOR, linked, bound through the loader's one load, its needs declared on the + // connector. + let dispatcher = Arc::new(Dispatcher::new(DispatchConfig::default())); + let row = LinkedRow::of(pools_door()).expect("the door states itself"); + let plane = load_linked::<Plane>( + &row, + Bind { + instance: Arc::from(instance), + max_inflight_cap: 64, + sink: Arc::new(NoSink), + dispatcher: dispatcher.adopter(), + conns: crate::root::loader::dispatch::ConnTable::Host( + Arc::clone(&connector) as Arc<dyn DeclaredConns> + ), + }, + ) + .expect("the linked door binds"); + + // THE DEPLOYMENT: one provider per member (in the members' dialect), one model per provider, + // pool `p` over them. + let key_file = std::env::temp_dir().join(format!( + "busbar-hook-seats-{}-{instance}", + std::process::id() + )); + let own_key = format!("{{ file: '{}' }}", key_file.display()); + if opts.api_key.is_none() { + std::fs::write(&key_file, "sk-seats").expect("the credential file"); + } + let mut yaml = String::from("providers:\n"); + for (i, _) in opts.members.iter().enumerate() { + yaml.push_str(&format!( + " oai{i}:\n api_key: {}\n", + opts.api_key.unwrap_or(&own_key) + )); + } + yaml.push_str("models:\n"); + for (i, _) in opts.members.iter().enumerate() { + yaml.push_str(&format!(" m{i}:\n provider: oai{i}\n")); + } + yaml.push_str("pools:\n p:\n members:\n"); + let pooled = opts.pooled.unwrap_or(opts.members.len()); + for (i, (_, weight)) in opts.members.iter().enumerate().take(pooled) { + yaml.push_str(&format!(" - model: m{i}\n weight: {weight}\n")); + if opts.described { + yaml.push_str(&format!(" tier: t{i}\n tags: [g{i}]\n")); + } + } + if let Some(secs) = opts.failover_timeout_secs { + yaml.push_str(&format!(" failover:\n timeout_secs: {secs}\n")); + } + if let Some(header) = opts.affinity_header { + yaml.push_str(&format!( + " affinity:\n mode: session\n header_name: {header}\n" + )); + } + let deploy = busbar_kernel::config::deploy_from_yaml_str(&yaml).map_err(|e| e.to_string())?; + let mut defs_yaml = String::new(); + for (i, (port, _)) in opts.members.iter().enumerate() { + defs_yaml.push_str(&format!( + "oai{i}:\n protocol: {}\n base_url: 'http://127.0.0.1:{port}'\n", + opts.dialect.unwrap_or(word("dialect")) + )); + } + let defs: HashMap<String, busbar_kernel::config::ProviderDef> = + serde_yaml::from_str(&defs_yaml).expect("the providers"); + let cfg = busbar_kernel::config::resolve(&deploy, &defs).map_err(|e| e.join("; "))?; + let providers = provider_routes( + &cfg.providers, + &busbar_kernel::config_validate::MetadataPosture::default(), + ); + let sections = kernel_sections(&cfg); + let model_pools = crate::root::model_egress::ModelPools::of(&cfg); + + // THE MONEY: a signing governance book with one minted key (in a budgeted group when asked), + // the node's book bound. + let signer = TokenSigner::from_secret_bytes(&[7u8; 32], DEFAULT_KID); + let gov = Arc::new( + GovState::new_with_signer(Arc::new(MemoryStore::new()), None, Some(signer)) + .expect("governance"), + ); + let groups: BTreeMap<String, busbar_kernel::config::GroupCfg> = opts + .budget_cents + .map(|amount| { + let limit = busbar_kernel::config::groups::LimitCfg { + metric: busbar_kernel::config::groups::LimitMetric::Budget, + amount, + per: Some(busbar_kernel::config::groups::LimitWindow::Day), + scope: None, + on_exhaust: None, + downgrade_to: None, + admission: None, + on_exhaustion: None, + }; + ( + format!("{instance}-group"), + busbar_kernel::config::GroupCfg { + parent: None, + enabled: true, + limits: vec![limit], + ..Default::default() + }, + ) + }) + .into_iter() + .collect(); + let cost = if groups.is_empty() { + CostModel::flat(1) + } else { + CostModel::resolve_parts(None, 1, &groups) + }; + let (key, token) = gov + .mint_signed( + NewKeySpec { + name: "seats".to_string(), + group: groups.keys().next().cloned(), + allowed_pools: opts + .allowed_pools + .map(|pools| pools.iter().map(|p| (*p).to_string()).collect()), + ..Default::default() + }, + 4_000_000_000, + 1_700_000_000, + ) + .expect("mint"); + gov.hydrate_budgets(&cost, 0).expect("hydrate"); + let node = Arc::new(Node::new()); + let book = crate::root::durability::node_book_over(Box::new(|| CARD.pin())); + node.bind_book(Arc::clone(&book.durability)); + let post = Arc::new(NodeEndPost::new(Arc::clone(&node))); + let site = Arc::clone(&post); + let book_money = Arc::clone(&gov); + let plane_money = move || { + Arc::new(PlaneMoney::new( + Arc::clone(&book_money), + Arc::clone(&site) as Arc<dyn EndPost>, + )) + }; + + // THE GENERATION: the lanes, the pool, the fee, and the four seats. + let mut builder = busbar_kernel::test_support::TestApp::new() + .keys_chain() + .governance(Arc::clone(&gov)) + .cost(cost); + for (i, (port, _)) in opts.members.iter().enumerate() { + let lane = busbar_kernel::test_support::LaneSpec::new( + &format!("m{i}"), + opts.dialect.unwrap_or(word("dialect")), + &format!("http://127.0.0.1:{port}"), + ); + builder = builder.lane(match opts.lane_budget { + Some(n) => lane.budget(n), + None => lane, + }); + } + let weights: Vec<(usize, u32)> = opts + .members + .iter() + .enumerate() + .take(pooled) + .map(|(i, (_, w))| (i, *w)) + .collect(); + let mut app = builder.pool("p", &weights).build(); + let log = Arc::new(Mutex::new(Vec::new())); + let read: Arc<dyn Fn() -> Ledger + Send + Sync> = { + let (gov, cost, key_id) = (Arc::clone(&gov), app.cost.clone(), key.id.clone()); + Arc::new(move || { + let u = gov + .usage_for(cost.as_ref(), &key_id, busbar_kernel::store::now()) + .expect("usage read") + .expect("the key exists"); + gov.flush_budgets(); + // Window 0 is the key's all-time bucket. + let row = gov.store().get_usage(&key_id, 0).expect("ledger row"); + ( + u.requests, + u.spend_cents, + row.requests, + row.billable_requests, + ) + }) + }; + let tokens: Arc<dyn Fn() -> u64 + Send + Sync> = { + let (gov, cost, key_id) = (Arc::clone(&gov), app.cost.clone(), key.id.clone()); + Arc::new(move || { + gov.usage_for(cost.as_ref(), &key_id, busbar_kernel::store::now()) + .expect("usage read") + .map_or(0, |u| u.tokens) + }) + }; + let mut rewrite = SeatProbe::new("rewrite", &log); + let read_for_rewrite = Arc::clone(&read); + rewrite.requests_now = Some(Box::new(move || read_for_rewrite().0)); + let request_tap = Arc::new(SeatProbe::new("request-tap", &log)); + let candidate_tap = Arc::new(SeatProbe::new("candidate-tap", &log)); + let mut gate_probe = SeatProbe::new("gate", &log); + // The pool's ranking policy, which is shown the candidates (an abstaining ranker). + let ranker = Arc::new(SeatProbe::new("ranker", &log)); + if opts.reject_at_gate { + gate_probe.reject = Some((451, "the gate says no")); + } + { + let a = Arc::get_mut(&mut app).expect("sole owner"); + a.rewrite_hooks = vec![(std::time::Duration::from_millis(500), Arc::new(rewrite))]; + // The request tap holds the prompt grant so its payload carries the (rewritten) messages. + let rt: Arc<dyn RoutingPolicy> = request_tap.clone(); + a.tap_hooks = vec![(std::time::Duration::from_millis(500), true, rt, Vec::new())]; + let ct: Arc<dyn RoutingPolicy> = candidate_tap.clone(); + a.tap_hooks_candidate = + vec![(std::time::Duration::from_millis(500), false, ct, Vec::new())]; + a.global_gates = vec![(0u16, gate(Arc::new(gate_probe)))]; + if opts.described { + let r: Arc<dyn RoutingPolicy> = ranker.clone(); + a.pool_orderings.insert("p".to_string(), gate(r)); + use busbar_contract::Signal; + for s in [ + Signal::CandidateBreakerState, + Signal::CandidateErrorRate, + Signal::CandidateLatencyP95Ms, + ] { + a.requested_signals.insert(s); + } + } + } + + // THE COMPOSITION, as production seals it: the door opened with the deployment's sections, + // its egress the model-serving walk over the kernel's lane cells, its hooks the kernel's stage. + let secrets = busbar_kernel::config::secret::SecretResolver::builtins_only(); + let auths = Arc::new(OutboundAuths::new( + Arc::clone(&dispatcher), + crate::LINKED.auths, + None, + crate::root::loader::dispatch::ConnTable::NoNeeds, + )); + let models = crate::root::model_egress::ModelServing { + pools: model_pools.clone(), + lanes: (0..opts.members.len()) + .map(|i| (format!("m{i}"), i)) + .collect(), + app: { + let app = Arc::clone(&app); + Arc::new(move || Arc::clone(&app)) + }, + }; + let reach = DoorReach { + providers: &providers, + secrets: &secrets, + auths: Arc::clone(&auths), + conns: Arc::clone(&connector) as Arc<dyn PollConns>, + stream_ceiling_secs: opts.stream_ceiling_secs.unwrap_or(600), + models: Some(&models), + upgrades: Vec::new(), + }; + let stage = HookStage { + host: { + let app = Arc::clone(&app); + Arc::new(move || busbar_kernel::plane_host::engine_host(&app)) + }, + gov: Arc::clone(&gov), + }; + let sections: BTreeMap<&'static str, serde_yaml::Value> = sections; + let served = compose_planes( + &[(instance.to_string(), plane)], + &dispatcher, + &composed_services(), + §ions, + None, + &plane_money, + Some(&DoorEgress { + reach: &reach, + journal: Arc::clone(&post) as Arc<dyn busbar_kernel_egress::ports::Journal>, + }), + Some(&stage), + ); + let _ = std::fs::remove_file(&key_file); + let mut served = served?; + served.post = Some(Arc::clone(&post)); + let appliers = served.appliers(); + // THE APPLY, as the root's apply hook runs it: the same deployment's sections, the pools and + // lanes, the door's egress resealed over the providers with their secrets resolved anew. + let reapply: Arc<dyn Fn() + Send + Sync> = { + let (appliers, app, post) = (appliers.clone(), Arc::clone(&app), Arc::clone(&post)); + let (connector, auths) = (Arc::clone(&connector), Arc::clone(&auths)); + let lanes = opts.members.len(); + let stream_ceiling_secs = opts.stream_ceiling_secs.unwrap_or(600); + let (sections, providers) = (sections.clone(), providers.clone()); + Arc::new(move || { + let secrets = busbar_kernel::config::secret::SecretResolver::builtins_only(); + let models = crate::root::model_egress::ModelServing { + pools: model_pools.clone(), + lanes: (0..lanes).map(|i| (format!("m{i}"), i)).collect(), + app: { + let app = Arc::clone(&app); + Arc::new(move || Arc::clone(&app)) + }, + }; + let reach = DoorReach { + providers: &providers, + secrets: &secrets, + auths: Arc::clone(&auths), + conns: Arc::clone(&connector) as Arc<dyn PollConns>, + stream_ceiling_secs, + models: Some(&models), + upgrades: Vec::new(), + }; + appliers.refresh_models( + §ions, + &DoorEgress { + reach: &reach, + journal: Arc::clone(&post) as Arc<dyn busbar_kernel_egress::ports::Journal>, + }, + ); + }) + }; + let doors = door_routes( + served, + || CARD.pin(), + &[], + &busbar_kernel::base_data_core_lines(&app), + ) + .expect("its claims mount"); + let (router, _admin, _handle) = + busbar_kernel::build_split_routers_serving(Arc::clone(&app), doors, 1 << 20, 0, false); + Ok(DoorRig { + router, + token: token.expose_secret().to_string(), + app, + book: Arc::clone(&book.durability), + key_id: key.id.to_string(), + appliers, + reapply, + log, + request_tap, + candidate_tap, + gate_seat: ranker, + read, + tokens, + }) +} + +/// ONE KEYED REQUEST through the pools door with the four probed seats installed, a rejecting gate +/// when `reject_at_gate`. +async fn run(instance: &'static str, reject_at_gate: bool) -> Ran { + let port = far_end().await; + let rig = rig( + instance, + RigOpts { + members: &[(port, 1)], + reject_at_gate, + ..RigOpts::default() + }, + ) + .await; + let (status, _, _) = rig.chat().await; + Ran { + status, + log: Arc::clone(&rig.log), + request_tap: Arc::clone(&rig.request_tap), + candidate_tap: Arc::clone(&rig.candidate_tap), + read: Arc::clone(&rig.read), + } +} + +impl DoorRig { + /// The request-stage tap's last delivered projection (bounded wait). + pub async fn request_tap_payload(&self, budget_ms: u64) -> Option<Vec<u8>> { + delivered(&self.request_tap, budget_ms).await + } + + /// The seat log. + pub fn seats(&self) -> Vec<String> { + self.log.lock().unwrap().clone() + } +} + +/// A rejecting gate: the admission charge, the rewrite and the request tap all happened +/// before it; the candidate tap never fires; the billable request is refunded and the admission +/// count is kept. +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn migrated_request_hook_seats_after_admit_before_candidate() { + let _one = PUBLISHING.lock().await; + let instance = "serve-hook-seats-reject"; + let _published = Published(instance); + let ran = run(instance, true).await; + assert_eq!( + ran.status, 451, + "the gate's reject is what the caller receives" + ); + + // Admission door before rewrite: the rewrite seat saw the request already counted. + let log = ran.log.lock().unwrap().clone(); + assert!( + log.iter().any(|e| e == "rewrite:requests=Some(1)"), + "the rewrite seat must run AFTER the admission door charged the request; log: {log:?}" + ); + // Rewrite before the request tap: the tap's payload carries the rewritten prompt, and the tap + // was delivered at all even though the gate rejected (so it was seated before the gate). + let payload = delivered(&ran.request_tap, 2_000) + .await + .expect("the request-stage tap fires before the gate and is delivered despite the reject"); + let text = String::from_utf8_lossy(&payload); + assert!( + text.contains(REWRITTEN), + "the request tap must observe the REWRITTEN request: {text}" + ); + assert!( + !text.contains("the original prompt"), + "the request tap must not see the pre-rewrite prompt: {text}" + ); + // The gate ran (it produced the 451) and the candidate tap, seated after the gates, never + // fires on a rejected request. + assert!( + log.iter().any(|e| e == "gate"), + "the gate seat ran; log: {log:?}" + ); + assert!( + delivered(&ran.candidate_tap, 300).await.is_none(), + "the candidate tap is seated after the gates, so a gate reject must never reach it" + ); + // The reject consumed the requests slot but refunded the fee base (a flat 1-cent fee per + // billable request, so derived spend reads the billable count directly). + let (requests, spend, row_requests, row_billable) = (ran.read)(); + assert_eq!(requests, 1, "the admission count is never refunded"); + assert_eq!( + spend, 0, + "the billable request (the fee base) is refunded on a gate reject" + ); + assert_eq!( + (row_requests, row_billable), + (1, 0), + "the durable ledger row keeps the admission and drops the billable request" + ); +} + +/// The positive twin: an abstaining gate lets the request through, the candidate tap fires AFTER +/// the gate decided, and a served request keeps its billable request (one fee). +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn candidate_tap_fires_after_an_abstaining_gate_and_a_served_request_bills_once() { + let _one = PUBLISHING.lock().await; + let instance = "serve-hook-seats-serve"; + let _published = Published(instance); + let ran = run(instance, false).await; + assert_eq!(ran.status, 200, "the live lane serves the request"); + assert!( + delivered(&ran.candidate_tap, 2_000).await.is_some(), + "the candidate tap fires on a request the gates let through" + ); + let log = ran.log.lock().unwrap().clone(); + let gate_at = log.iter().position(|e| e == "gate").expect("the gate ran"); + let cand_at = log + .iter() + .position(|e| e == "candidate-tap") + .expect("the candidate tap was delivered"); + let rewrite_at = log + .iter() + .position(|e| e.starts_with("rewrite:")) + .expect("the rewrite ran"); + assert!( + rewrite_at < gate_at && gate_at < cand_at, + "seat order must be rewrite, gate, candidate tap; log: {log:?}" + ); + // The unit's money settles at its end, which the caller's last byte may precede: read the + // ledger once it has (bounded). + let mut ledger = (ran.read)(); + for _ in 0..200 { + if ledger.3 == 1 { + break; + } + tokio::time::sleep(std::time::Duration::from_millis(10)).await; + ledger = (ran.read)(); + } + let (requests, spend, row_requests, row_billable) = ledger; + assert_eq!((requests, row_requests), (1, 1)); + assert_eq!( + (spend, row_billable), + (1, 1), + "a served request keeps its one billable request (1-cent flat fee)" + ); +} diff --git a/crates/busbar/src/root/tests/serve_money.rs b/crates/busbar/src/root/tests/serve_money.rs index b30b5ed7b1..d429107341 100644 --- a/crates/busbar/src/root/tests/serve_money.rs +++ b/crates/busbar/src/root/tests/serve_money.rs @@ -355,8 +355,11 @@ async fn a_keyed_unit_is_admitted_and_its_money_settles_at_its_end() { let (status, body) = g.post("/call/direct:m", true).await; assert_eq!( (status, body.as_str()), - (503, "refused:503:breaker_open"), - "admitted, then the walk is exhausted" + ( + 503, + "refused:503:The service is temporarily overloaded. Please retry shortly." + ), + "admitted, then the walk is exhausted (the shed in 1.5.5's words)" ); assert_eq!(g.requests(), 1, "its request was charged at admission"); assert_eq!(g.money.open_units(), 0, "its money facts closed at its end"); @@ -399,7 +402,10 @@ async fn an_anonymous_unit_on_an_open_claim_routes_and_opens_no_money() { let (status, body) = g.post("/open", false).await; assert_eq!( (status, body.as_str()), - (503, "refused:503:breaker_open"), + ( + 503, + "refused:503:The service is temporarily overloaded. Please retry shortly." + ), "admitted, then the walk is exhausted" ); assert_eq!(g.requests(), 0, "nothing charged"); @@ -755,8 +761,12 @@ async fn node_boot_hooks_arm() { let (status, body) = g.post("/call/direct:m", true).await; assert_eq!( (status, body.as_str()), - (503, "refused:503:breaker_open"), - "admitted, then the walk is exhausted" + ( + 503, + "refused:503:The service is temporarily overloaded. Please retry shortly." + ), + "admitted, then the walk is exhausted (the shed in 1.5.5's words, as every plane is handed \ + them: RefusalIn.text is the kernel's own message for the refusal)" ); assert_eq!(g.requests(), 1, "its request was charged at admission"); assert_eq!(g.post.open_units(), 0, "its node facts closed at its end"); @@ -821,6 +831,7 @@ async fn the_boot_composition_serves_a_dropped_in_door_plane_in_every_build() { crate::root::loader::dispatch::ConnTable::NoNeeds, )), conns: Arc::new(busbar_core_connector::Connector::new()), + models: None, stream_ceiling_secs: 1, upgrades: Vec::new(), }; diff --git a/crates/busbar/src/root/tests/serve_tests.rs b/crates/busbar/src/root/tests/serve_tests.rs index 802960b45d..6e5b7fb3e4 100644 --- a/crates/busbar/src/root/tests/serve_tests.rs +++ b/crates/busbar/src/root/tests/serve_tests.rs @@ -328,6 +328,7 @@ async fn a_claimed_request_is_served_through_the_door_and_its_money_posted() { auths: Arc::new(auths), conns: Arc::clone(&connector) as Arc<dyn PollConns>, stream_ceiling_secs: 600, + models: None, upgrades: Vec::new(), }; @@ -605,9 +606,9 @@ mod tools_door { use crate::root::door_steps::tests::hook_parity; use crate::root::door_steps::tests::tool_door::{ - protocol_version, rig_tools, send_as, send_headed, surface, three_tools, tool_digest, - tool_listing, tool_server, tool_server_listing, tool_server_replying, Footing, Rig, STALL, - TOOL_DESCRIPTION, + protocol_version, rig_tools, send_as, send_headed, sentence, surface, three_tools, + tool_digest, tool_listing, tool_server, tool_server_listing, tool_server_replying, Footing, + Rig, STALL, TOOL_DESCRIPTION, }; use crate::root::serve::planes_tests::{Published, PUBLISHING}; @@ -1979,6 +1980,308 @@ mod tools_door { let (status, answer) = call(&rig, &reader, "cat_read_file", serde_json::json!({})).await; assert_eq!(status, StatusCode::OK, "its own tool is served: {answer}"); } + + // ── THE WALK'S EXHAUSTION ON THIS DOOR: PREDEV'S BYTES ────────────────────────────────────── + // + // The kernel hands every plane the walk's own sentence for an exhausted walk + // (`RefusalIn.text`); this door's baseline is predev's (spec l.4452), which said the reason's + // own word there. Each cell below drives a `tools/call` to an exhausted walk on one path the + // door's refusal slot takes, and pins the bytes predev serves on it. + + /// The `tools/call` of `tool` with the tasks extension declared (a call that creates a task + /// where its tool supports one). + fn task_call_of(tool: &str) -> String { + let mut call: serde_json::Value = + serde_json::from_str(&call_of(tool, serde_json::json!({}))).expect("the call"); + call["params"]["_meta"]["io.modelcontextprotocol/clientCapabilities"] = + serde_json::json!({ "extensions": { "io.modelcontextprotocol/tasks": {} } }); + call.to_string() + } + + /// `tasks/get` of `task_id` on `rig`, the extension declared: the status and the JSON-RPC body. + async fn task_get(rig: &Rig, task_id: &str) -> (u16, serde_json::Value) { + let verb = serde_json::json!({ + "jsonrpc": "2.0", "id": 43, "method": "tasks/get", + "params": { + "taskId": task_id, + "_meta": { + "io.modelcontextprotocol/protocolVersion": protocol_version(), + "io.modelcontextprotocol/clientCapabilities": + { "extensions": { "io.modelcontextprotocol/tasks": {} } }, + }, + }, + }) + .to_string(); + let (status, body) = send_as( + &rig.router, + Some(&rig.token), + &verb, + "tasks/get", + Some(task_id), + ) + .await; + ( + status.as_u16(), + serde_json::from_slice(&body).expect("JSON-RPC"), + ) + } + + /// `tasks/get` of `task_id` until the task is terminal (or the test gives up): its body. + async fn task_settled(rig: &Rig, task_id: &str) -> serde_json::Value { + for _ in 0..600 { + let (_, body) = task_get(rig, task_id).await; + if ["completed", "failed", "cancelled"] + .iter() + .any(|s| body["result"]["status"] == *s) + { + return body; + } + tokio::time::sleep(Duration::from_millis(25)).await; + } + panic!("task {task_id} never settled"); + } + + /// A tool server answering its tool list as approved and holding every `tools/call` unanswered. + async fn stalling_calls() -> (u16, Heard) { + tool_server_replying(Arc::new(|r: &str| { + if r.contains("\"tools/list\"") { + let list = serde_json::json!({"jsonrpc": "2.0", "id": 1, "result": {"tools": tool_listing()}}); + (200, list.to_string()) + } else { + (STALL, String::new()) + } + })) + .await + } + + /// Wait for the server to hear a `tools/call`. + async fn heard_a_call(heard: &mut Heard) { + tokio::time::timeout(Duration::from_secs(10), async { + loop { + let r = heard.recv().await.expect("the server hears"); + if r.contains("\"tools/call\"") { + return; + } + } + }) + .await + .expect("the call reached the server"); + } + + /// Publish generation 2 of `rig`'s door over the `tools:` section `section` (the operator's + /// reload): the plane's catalogue moves; the kernel's sealed egress does not. + fn reload(rig: &Rig, section: &serde_yaml::Value) { + use crate::root::loader::dispatch::{in_head, out_head, Frame}; + use busbar_contract::abi::mechanism::call::{Blob, Outcome as AbiOutcome, BLOB_JSON}; + use busbar_contract::abi::mechanism::lifecycle::RefreshIn; + use busbar_contract::abi::plane::PlaneRefreshOut; + let settings = serde_json::to_vec(section).expect("json"); + let mut frame = Frame::new( + RefreshIn { + head: in_head(), + generation: 2, + settings: Blob { + ptr: settings.as_ptr(), + len: settings.len(), + fmt: BLOB_JSON, + flags: 0, + }, + secrets: std::ptr::null(), + secrets_len: 0, + }, + PlaneRefreshOut { + head: out_head(), + snapshot: std::ptr::null(), + }, + ); + let (called, snapshot) = rig.plane.refresh(&mut frame); + assert_eq!(called.outcome, AbiOutcome::Ready, "the refresh is taken"); + assert!(snapshot.is_some(), "generation 2 is published"); + } + + /// EXHAUSTED, THE SERVER RESOLVED (the control the two cells after it stand beside): a tripped + /// server's call is refused as the served engine refused it, `503` / `-32030` / + /// `upstream_unavailable` with its sentence and the wait, byte for byte as predev serves it; the + /// walk's own sentence never reaches the caller. + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] + async fn a_tripped_servers_exhausted_walk_is_refused_in_predevs_bytes() { + let _one = PUBLISHING.lock().await; + let instance = "serve-door-tools-exhausted-bytes"; + let _published = Published(instance); + let (port, mut heard) = answering_calls_with(503).await; + let rig = rig_tools(instance, port, registration("flaky", port, ""), &|app| app); + fail_to_the_trip(&rig, "flaky_read_file", &mut heard, 0).await; + + let (status, headers, body) = send_headed( + &rig.router, + Some(&rig.token), + &call_of("flaky_read_file", serde_json::json!({})), + "tools/call", + Some("flaky_read_file"), + ) + .await; + let body = String::from_utf8_lossy(&body).to_string(); + let wait: u64 = headers + .get("retry-after") + .and_then(|v| v.to_str().ok()) + .and_then(|v| v.parse().ok()) + .unwrap_or_else(|| panic!("a Retry-After: {headers:?} {body}")); + assert!(wait > 0, "the open cell's wait: {headers:?}"); + assert_eq!(status.as_u16(), 503, "{headers:?} {body}"); + assert_eq!( + body, + format!( + "{{\"error\":{{\"code\":-32030,\"data\":{{\"reason\":\"upstream_unavailable\",\ + \"retry_after_ms\":{},\"server\":\"flaky\"}},\"message\":\"{}\"}},\"id\":41,\ + \"jsonrpc\":\"2.0\"}}", + wait * 1000, + sentence("tripped_server_message") + .replace("{server}", "flaky") + .replace("{wait}", &wait.to_string()) + ), + "{headers:?}" + ); + assert!(drain(&mut heard).is_empty(), "nothing reached the server"); + assert!(rig.all_ended(), "every unit ended"); + } + + /// EXHAUSTED, THE SERVER NO LONGER RESOLVED (the generic arm of the door's refusal slot): a call + /// dispatched to a server that never answers, whose tool leaves the catalogue (an operator's + /// reload) while the call waits out the server's `timeout:`, is refused when the walk is + /// exhausted as predev refused it: the walk's status, `-32000`, and the reason's own word, no + /// data and no wait. + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] + async fn an_exhausted_walk_whose_tool_left_the_catalogue_is_refused_in_predevs_bytes() { + let _one = PUBLISHING.lock().await; + let instance = "serve-door-tools-exhausted-unlisted"; + let _published = Published(instance); + let (port, mut heard) = stalling_calls().await; + let rig = rig_tools( + instance, + port, + registration("stall", port, " timeout: 2s\n"), + &|app| app, + ); + let (router, token) = (rig.router.clone(), rig.token.clone()); + let pending = tokio::spawn(async move { + send_headed( + &router, + Some(&token), + &call_of("stall_read_file", serde_json::json!({})), + "tools/call", + Some("stall_read_file"), + ) + .await + }); + heard_a_call(&mut heard).await; + // The reload: the server is registered under another name, so the stalled call's tool is no + // longer one the catalogue publishes. + reload(&rig, ®istration("other", port, "")); + + let (status, headers, body) = tokio::time::timeout(Duration::from_secs(20), pending) + .await + .expect("the stalled call was answered") + .expect("the call ran"); + let body = String::from_utf8_lossy(&body).to_string(); + assert_eq!(status.as_u16(), 503, "{headers:?} {body}"); + assert_eq!( + body, r#"{"error":{"code":-32000,"message":"breaker_open"},"id":41,"jsonrpc":"2.0"}"#, + "{headers:?}" + ); + assert!( + headers.get("retry-after").is_none(), + "no wait on this arm: {headers:?}" + ); + assert!(rig.all_ended(), "every unit ended"); + } + + /// EXHAUSTED UNDER A TASK (the continuation's refusal): a task whose call is dispatched to a + /// server that never answers fails when its continuation's walk is exhausted, and `tasks/get` + /// states the failure as predev stated it: `-32603`, the upstream call failed, and the reason's + /// own word. + #[cfg(linked_axis_node)] + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] + async fn a_tasks_exhausted_walk_fails_the_task_in_predevs_bytes() { + let _one = PUBLISHING.lock().await; + let instance = "serve-door-tools-exhausted-task"; + let _published = Published(instance); + let (port, mut heard) = stalling_calls().await; + let tools: serde_yaml::Value = serde_yaml::from_str(&format!( + "stall:\n url: \"http://127.0.0.1:{port}/rpc\"\n {PIN}\n timeout: 2s\n \ + tools_allow:\n read_file: {{ schema_hash: \"{}\", task_support: optional }}\n", + tool_digest() + )) + .expect("a section"); + let rig = rig_tools(instance, port, tools, &|app| app); + + let (status, body) = send_as( + &rig.router, + Some(&rig.token), + &task_call_of("stall_read_file"), + "tools/call", + Some("stall_read_file"), + ) + .await; + let created: serde_json::Value = serde_json::from_slice(&body).expect("JSON-RPC"); + assert_eq!(status.as_u16(), 200, "{created}"); + let task_id = created["result"]["taskId"] + .as_str() + .unwrap_or_else(|| panic!("a task: {created}")) + .to_string(); + heard_a_call(&mut heard).await; + + let settled = task_settled(&rig, &task_id).await; + assert_eq!(settled["result"]["status"], "failed", "{settled}"); + assert_eq!( + settled["result"]["error"].to_string(), + sentence("continuation_refused_error"), + "{settled}" + ); + } + + /// EXHAUSTED UNDER A TASK, THE SERVER TRIPPED: a task created on a tool whose server's cell is + /// open fails as predev failed it, its `tasks/get` stating `-32603`, the upstream call failed, + /// and the reason's own word. + #[cfg(linked_axis_node)] + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] + async fn a_task_on_a_tripped_server_fails_in_predevs_bytes() { + let _one = PUBLISHING.lock().await; + let instance = "serve-door-tools-exhausted-task-tripped"; + let _published = Published(instance); + let (port, mut heard) = answering_calls_with(503).await; + let tools: serde_yaml::Value = serde_yaml::from_str(&format!( + "flaky:\n url: \"http://127.0.0.1:{port}/rpc\"\n {PIN}\n \ + tools_allow:\n read_file: {{ schema_hash: \"{}\", task_support: optional }}\n", + tool_digest() + )) + .expect("a section"); + let rig = rig_tools(instance, port, tools, &|app| app); + fail_to_the_trip(&rig, "flaky_read_file", &mut heard, 0).await; + + let (status, headers, body) = send_headed( + &rig.router, + Some(&rig.token), + &task_call_of("flaky_read_file"), + "tools/call", + Some("flaky_read_file"), + ) + .await; + let created: serde_json::Value = serde_json::from_slice(&body).expect("JSON-RPC"); + assert_eq!(status.as_u16(), 200, "{headers:?} {created}"); + let task_id = created["result"]["taskId"] + .as_str() + .unwrap_or_else(|| panic!("a task: {created}")) + .to_string(); + + let settled = task_settled(&rig, &task_id).await; + assert_eq!(settled["result"]["status"], "failed", "{settled}"); + assert_eq!( + settled["result"]["error"].to_string(), + sentence("continuation_refused_error"), + "{settled}" + ); + assert!(drain(&mut heard).is_empty(), "nothing reached the server"); + } } #[cfg(feature = "plane-decisions")] @@ -2264,6 +2567,7 @@ async fn serve_configured( auths: Arc::new(auths), conns: Arc::clone(&connector) as Arc<dyn PollConns>, stream_ceiling_secs: 600, + models: None, upgrades: Vec::new(), }; let mut sections = BTreeMap::new(); @@ -2461,6 +2765,907 @@ async fn a_plane_stating_no_breaker_fact_keeps_the_default_bench() { ); } +// ── THE DOOR SERVING THE `pools` MAP: ITS CAPABILITY CELLS (Q128 U14) ─────────────────────────── +// +// The capability-equality matrix's root column for this plane, witnessed on its door path: each +// cell below drives a keyed caller through the data router built with the door's claims, the +// kernel's hook stage, the model-serving walk over the kernel's lane cells and the node's book +// (`super::hook_seat_tests::rig`), and reads the capability where the kernel keeps it. +#[cfg(linked_fold_on_driver)] +use super::hook_seat_tests::{ + chunk, far_end_answering as seat_far_end, far_end_scripted, rig, word as door_word, RigOpts, + Script as SeatScript, REWRITTEN, +}; +#[cfg(linked_fold_on_driver)] +use super::planes_tests::{Published as Withdrawn, PUBLISHING as ONE_PUBLISHER}; + +/// A served chat completion, marked by the member that answered it. +#[cfg(linked_fold_on_driver)] +const SERVED_BY_TWIN: &str = r#"{"id":"chatcmpl-2","object":"chat.completion","created":0,"model":"m1","choices":[{"index":0,"message":{"role":"assistant","content":"served-by-the-twin"},"finish_reason":"stop"}],"usage":{"prompt_tokens":1,"completion_tokens":1,"total_tokens":2}}"#; + +/// A far end that cannot serve now. +#[cfg(linked_fold_on_driver)] +const OVERLOADED: &str = r#"{"error":{"message":"overloaded","type":"server_error"}}"#; + +/// A far end refusing the request itself. +#[cfg(linked_fold_on_driver)] +const MALFORMED: &str = r#"{"error":{"message":"bad request","type":"invalid_request_error"}}"#; + +/// BREAKER-TRIP: a member whose far end fails records into the kernel's ONE breaker cell for its +/// (pool, lane), and the cell opens: the walk's failure benched it. +#[cfg(linked_fold_on_driver)] +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn the_pools_door_trips_a_failing_members_breaker_cell() { + let _one = ONE_PUBLISHER.lock().await; + let instance = "serve-door-breaker-trip"; + let _published = Withdrawn(instance); + let (down, twin) = ( + seat_far_end(503, OVERLOADED).await, + seat_far_end(200, SERVED_BY_TWIN).await, + ); + let rig = rig( + instance, + RigOpts { + members: &[(down.port, 100), (twin.port, 1)], + ..RigOpts::default() + }, + ) + .await; + let ready = || rig.app.store.ready_in("p", 0, busbar_kernel::store::now()); + assert!(ready(), "the member's cell admits before any failure"); + let (status, _, _) = rig.chat().await; + assert_eq!(status, 200, "the twin serves"); + assert_eq!(down.served(), 1, "the failing member was dialled once"); + assert!( + !ready(), + "the failure recorded into the member's (pool, lane) cell and opened it" + ); +} + +/// BREAKER-FASTFAIL: a tripped member is refused before dispatch, at once: the next unit never +/// dials it, and its twin serves. +#[cfg(linked_fold_on_driver)] +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn the_pools_door_refuses_a_tripped_member_before_dispatch() { + let _one = ONE_PUBLISHER.lock().await; + let instance = "serve-door-breaker-fastfail"; + let _published = Withdrawn(instance); + let (down, twin) = ( + seat_far_end(503, OVERLOADED).await, + seat_far_end(200, SERVED_BY_TWIN).await, + ); + let rig = rig( + instance, + RigOpts { + members: &[(down.port, 100), (twin.port, 1)], + ..RigOpts::default() + }, + ) + .await; + assert_eq!(rig.chat().await.0, 200); + assert_eq!(down.served(), 1); + let started = std::time::Instant::now(); + assert_eq!(rig.chat().await.0, 200, "the twin serves the next unit"); + assert_eq!( + down.served(), + 1, + "the tripped member is refused before dispatch: never dialled again" + ); + assert_eq!(twin.served(), 2); + assert!( + started.elapsed() < std::time::Duration::from_secs(5), + "refused at once, not after a timeout" + ); +} + +/// FAILOVER-REROUTE: a second interchangeable candidate is tried before the first byte, through +/// the one walk: the first member fails, the twin's answer is the caller's. +#[cfg(linked_fold_on_driver)] +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn the_pools_door_reroutes_to_the_twin_before_the_first_byte() { + let _one = ONE_PUBLISHER.lock().await; + let instance = "serve-door-failover"; + let _published = Withdrawn(instance); + let (down, twin) = ( + seat_far_end(503, OVERLOADED).await, + seat_far_end(200, SERVED_BY_TWIN).await, + ); + let rig = rig( + instance, + RigOpts { + members: &[(down.port, 100), (twin.port, 1)], + ..RigOpts::default() + }, + ) + .await; + let (status, _, body) = rig.chat().await; + assert_eq!(status, 200); + assert!( + String::from_utf8_lossy(&body).contains("served-by-the-twin"), + "the twin's answer reached the caller: {}", + String::from_utf8_lossy(&body) + ); + assert_eq!((down.served(), twin.served()), (1, 1)); +} + +/// DISPOSITION: the far end's answer is classified before it is relayed: a refusal of the request +/// itself is the caller's own fault, relayed with its status, never failed over and never charged +/// to the member's breaker cell. +#[cfg(linked_fold_on_driver)] +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn the_pools_door_classifies_a_refused_request_as_the_callers_fault() { + let _one = ONE_PUBLISHER.lock().await; + let instance = "serve-door-disposition"; + let _published = Withdrawn(instance); + let (refusing, twin) = ( + seat_far_end(400, MALFORMED).await, + seat_far_end(200, SERVED_BY_TWIN).await, + ); + let rig = rig( + instance, + RigOpts { + members: &[(refusing.port, 100), (twin.port, 1)], + ..RigOpts::default() + }, + ) + .await; + let (status, _, _) = rig.chat().await; + assert_eq!(status, 400, "the far end's refusal is relayed"); + assert_eq!( + (refusing.served(), twin.served()), + (1, 0), + "a caller's fault is not failed over" + ); + assert!( + rig.app.store.ready_in("p", 0, busbar_kernel::store::now()), + "nor charged to the member's cell" + ); +} + +/// EGRESS-AUTH: the member's credential is the one the egress mechanism binds and injects; the +/// caller's own token never crosses. +#[cfg(linked_fold_on_driver)] +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn the_pools_door_injects_the_members_credential_not_the_callers() { + let _one = ONE_PUBLISHER.lock().await; + let instance = "serve-door-egress-auth"; + let _published = Withdrawn(instance); + let far = seat_far_end(200, SERVED_BY_TWIN).await; + let rig = rig( + instance, + RigOpts { + members: &[(far.port, 1)], + ..RigOpts::default() + }, + ) + .await; + assert_eq!(rig.chat().await.0, 200); + let seen = far.authorizations.lock().unwrap().clone(); + assert_eq!(seen, vec!["Bearer sk-seats".to_string()]); + assert!( + !seen[0].contains(&rig.token), + "the caller's token never crosses" + ); +} + +/// METRICS: the door's traffic appears on the scrape, under the pool and lane it was served on. +#[cfg(linked_fold_on_driver)] +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn the_pools_doors_traffic_appears_on_the_metrics_scrape() { + let _one = ONE_PUBLISHER.lock().await; + let instance = "serve-door-metrics"; + let _published = Withdrawn(instance); + let far = seat_far_end(200, SERVED_BY_TWIN).await; + let rig = rig( + instance, + RigOpts { + members: &[(far.port, 1)], + ..RigOpts::default() + }, + ) + .await; + assert_eq!(rig.chat().await.0, 200); + let scrape = busbar_kernel::snapshot::render(); + assert!( + scrape + .lines() + .any(|l| l.starts_with("busbar_upstream_attempts_total{") + && l.contains("pool=\"p\"") + && l.contains("lane=\"m0\"")), + "the attempt is on the scrape under its pool and lane:\n{scrape}" + ); +} + +/// GOVERNANCE-BUDGET: the unit's spend is attributed to the presenting key, and a key whose budget +/// is spent is refused before any far end is dialled. +#[cfg(linked_fold_on_driver)] +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn the_pools_door_attributes_spend_and_a_spent_budget_refuses() { + let _one = ONE_PUBLISHER.lock().await; + let instance = "serve-door-budget"; + let _published = Withdrawn(instance); + let far = seat_far_end(200, SERVED_BY_TWIN).await; + let rig = rig( + instance, + RigOpts { + members: &[(far.port, 1)], + budget_cents: Some(1), + ..RigOpts::default() + }, + ) + .await; + assert_eq!(rig.chat().await.0, 200); + let (requests, spend, _, billable) = rig.ledger_after(1).await; + assert_eq!( + (requests, spend, billable), + (1, 1, 1), + "the one fee on the key" + ); + let (status, head, _) = rig.chat().await; + assert_eq!(status, 429, "the spent budget refuses"); + assert!( + head.contains_key("retry-after"), + "with a Retry-After: {head:?}" + ); + assert_eq!(far.served(), 1, "the refused unit dialled nothing"); +} + +/// AUDIT-CHAIN: every unit the door serves seals exactly one record on the node's chain. +#[cfg(linked_fold_on_driver)] +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn the_pools_door_seals_one_audit_record_per_unit() { + let _one = ONE_PUBLISHER.lock().await; + let instance = "serve-door-audit"; + let _published = Withdrawn(instance); + let far = seat_far_end(200, SERVED_BY_TWIN).await; + let rig = rig( + instance, + RigOpts { + members: &[(far.port, 1)], + ..RigOpts::default() + }, + ) + .await; + assert_eq!(rig.chat().await.0, 200); + let _ = rig.ledger_after(1).await; + assert_eq!(rig.audit_records(), 1, "one record for the one unit"); +} + +/// HOOKS-GATE: a rejecting gate refuses the door's traffic before dispatch. +#[cfg(linked_fold_on_driver)] +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn the_pools_doors_gate_hook_refuses_before_dispatch() { + let _one = ONE_PUBLISHER.lock().await; + let instance = "serve-door-gate"; + let _published = Withdrawn(instance); + let far = seat_far_end(200, SERVED_BY_TWIN).await; + let rig = rig( + instance, + RigOpts { + members: &[(far.port, 1)], + reject_at_gate: true, + ..RigOpts::default() + }, + ) + .await; + assert_eq!(rig.chat().await.0, 451, "the gate's own status"); + assert_eq!(far.served(), 0, "nothing was dispatched"); +} + +/// HOOKS-TAP: the rewrite and observe hooks run over the door's payloads: the request-stage tap +/// sees the request as the global rewrite left it. +#[cfg(linked_fold_on_driver)] +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn the_pools_doors_tap_hooks_observe_the_rewritten_request() { + let _one = ONE_PUBLISHER.lock().await; + let instance = "serve-door-tap"; + let _published = Withdrawn(instance); + let far = seat_far_end(200, SERVED_BY_TWIN).await; + let rig = rig( + instance, + RigOpts { + members: &[(far.port, 1)], + ..RigOpts::default() + }, + ) + .await; + assert_eq!(rig.chat().await.0, 200); + let payload = rig + .request_tap_payload(2_000) + .await + .expect("the request-stage tap is delivered"); + let text = String::from_utf8_lossy(&payload); + assert!(text.contains(REWRITTEN), "{text}"); + assert!(!text.contains("the original prompt"), "{text}"); +} + +/// CATALOGUE: what a restricted key may SEE is the one catalogue walk's answer: the pool it may +/// reach and that pool's members, never a model it cannot reach. +#[cfg(linked_fold_on_driver)] +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn the_pools_doors_catalogue_shows_a_restricted_key_only_what_it_reaches() { + let _one = ONE_PUBLISHER.lock().await; + let instance = "serve-door-catalogue"; + let _published = Withdrawn(instance); + let (a, b) = ( + seat_far_end(200, SERVED_BY_TWIN).await, + seat_far_end(200, SERVED_BY_TWIN).await, + ); + let rig = rig( + instance, + RigOpts { + members: &[(a.port, 1), (b.port, 1)], + pooled: Some(1), + allowed_pools: Some(&["p"]), + ..RigOpts::default() + }, + ) + .await; + let (status, _, body) = rig.send("GET", "/v1/models", None).await; + assert_eq!(status, 200); + let listed: serde_json::Value = serde_json::from_slice(&body).expect("a JSON list"); + let ids: Vec<&str> = listed["data"] + .as_array() + .expect("the list") + .iter() + .filter_map(|m| m["id"].as_str()) + .collect(); + assert_eq!( + ids, + ["p", "m0"], + "the reachable pool and its member, nothing else" + ); +} + +/// THE DROPPED-CONTROLS AUDIT ROW ON THE DOOR (ARCHITECT, Q128 gap): a TRANSLATE attempt that +/// cannot carry a control the caller set writes 1.5.5's `egress.control_unrepresentable` row, +/// outcome `degraded`, `<control> on <dialect>`; an answer member the caller's dialect has no form +/// for writes `<path> from <dialect>`. Both reach the kernel's audit chain through the plane's +/// `RECORD_AUDIT` write. RED arm: a request that sets nothing the far dialect drops writes no row. +#[cfg(linked_fold_on_driver)] +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn the_pools_door_audits_a_control_the_far_dialect_cannot_carry() { + let _one = ONE_PUBLISHER.lock().await; + let instance = "serve-door-dropped-controls"; + let _published = Withdrawn(instance); + // The far dialect is the plane's own door word (`untranslatable_dialect`), and so is its answer: + // a message whose usage carries a member the caller's dialect has no form for. + let far_dialect = door_word("untranslatable_dialect"); + let far = seat_far_end(200, door_word("untranslatable_answer")).await; + let rig = rig( + instance, + RigOpts { + members: &[(far.port, 1)], + dialect: Some(far_dialect), + ..RigOpts::default() + }, + ) + .await; + let rows = |resource: &str| { + busbar_kernel::audit::auditlog::AUDIT_LOG + .list_filtered( + 0, + 1000, + Some("egress.control_unrepresentable"), + Some(resource), + ) + .into_iter() + .filter(|e| e.principal == rig.key_id && e.outcome == "degraded") + .count() + }; + let (status, _, _) = rig + .send( + "POST", + "/v1/chat/completions", + Some(serde_json::json!({"model": "p", "max_tokens": 16, + "messages": [{"role": "user", "content": "hi"}]})), + ) + .await; + assert_eq!(status, 200); + assert_eq!( + rows(format!("logit_bias on {far_dialect}").as_str()), + 0, + "nothing set, nothing dropped" + ); + let (status, _, _) = rig + .send( + "POST", + "/v1/chat/completions", + Some(serde_json::json!({"model": "p", "max_tokens": 16, + "logit_bias": {"50256": -100}, + "messages": [{"role": "user", "content": "hi"}]})), + ) + .await; + assert_eq!(status, 200); + assert_eq!( + rows(format!("logit_bias on {far_dialect}").as_str()), + 1, + "the dropped control, one row" + ); + assert!( + rows(format!("usage.output_tokens_details from {far_dialect}").as_str()) >= 1, + "the answer member the caller's dialect cannot carry" + ); +} + +/// THE RANKING SIGNALS ON THE DOOR (ARCHITECT, Q128 gap): the hooks see each candidate as 1.5.5's +/// projection fed it: the pool member's tier and tags, the lane's latency signal once a sample is +/// recorded, its free concurrency, its remaining budget, and, where the generation declares them, +/// its breaker state, error rate and p95 latency in the routing pool. RED before: every one of them +/// was empty on the door path. +#[cfg(linked_fold_on_driver)] +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn the_pools_doors_hooks_see_each_candidates_standing() { + let _one = ONE_PUBLISHER.lock().await; + let instance = "serve-door-signals"; + let _published = Withdrawn(instance); + let (a, b) = ( + seat_far_end(200, SERVED_BY_TWIN).await, + seat_far_end(200, SERVED_BY_TWIN).await, + ); + let rig = rig( + instance, + RigOpts { + members: &[(a.port, 1), (b.port, 1)], + described: true, + ..RigOpts::default() + }, + ) + .await; + assert_eq!(rig.chat().await.0, 200); + let first = rig.gate_saw(); + assert_eq!(first.len(), 2, "{first:?}"); + assert!( + first[0].starts_with("m0 tier=Some(\"t0\") tags=[\"g0\"] latency=false avail=true"), + "{first:?}" + ); + assert!( + first[0].contains("CandidateBreakerState=closed"), + "the declared breaker state: {first:?}" + ); + assert!( + !first[0].contains("CandidateLatencyP95Ms"), + "no p95 before a sample: {first:?}" + ); + assert_eq!(rig.chat().await.0, 200); + let second = rig.gate_saw(); + let second: Vec<String> = second + .into_iter() + .filter(|c| c.contains("latency=true")) + .collect(); + assert!( + !second.is_empty(), + "the served attempt's latency sample: {second:?}" + ); + assert!( + second[0].contains("CandidateLatencyP95Ms=true") + && second[0].contains("CandidateErrorRate=true"), + "the p95 reservoir and the error rate, once fed: {second:?}" + ); +} + +// ── what only the plane reads, the kernel routes on (ARCHITECT Q1 ArriveOut, 2026-10-05) ──────── + +/// A streamed far end's head: chunked, so a far end that closes before its last chunk CUT it. +#[cfg(linked_fold_on_driver)] +const STREAM_HEAD: &str = "HTTP/1.1 200 OK\r\ncontent-type: text/event-stream\r\n\ + transfer-encoding: chunked\r\nconnection: close\r\n\r\n"; + +/// The streamed answer's frames, in order: a delta, the stop, the usage, the end. +#[cfg(linked_fold_on_driver)] +const FRAMES: [&str; 4] = [ + r#"{"id":"c1","object":"chat.completion.chunk","created":1,"model":"m0","choices":[{"index":0,"delta":{"role":"assistant","content":"hi"},"finish_reason":null}]}"#, + r#"{"id":"c1","object":"chat.completion.chunk","created":1,"model":"m0","choices":[{"index":0,"delta":{},"finish_reason":"stop"}]}"#, + r#"{"id":"c1","object":"chat.completion.chunk","created":1,"model":"m0","choices":[],"usage":{"prompt_tokens":11,"completion_tokens":7,"total_tokens":18}}"#, + "[DONE]", +]; + +/// One event-stream frame, as a chunk. +#[cfg(linked_fold_on_driver)] +fn frame(data: &str) -> Vec<u8> { + chunk(format!("data: {data}\n\n").as_bytes()) +} + +/// A stream that pauses `pause_ms` after its first frame, then ends cleanly. +#[cfg(linked_fold_on_driver)] +fn stream_pausing(pause_ms: u64) -> SeatScript { + SeatScript { + head: STREAM_HEAD.to_string(), + pieces: vec![ + (0, frame(FRAMES[0])), + (pause_ms, frame(FRAMES[1])), + (0, frame(FRAMES[2])), + (0, frame(FRAMES[3])), + ], + finish: Some(b"0\r\n\r\n".to_vec()), + } +} + +/// A caller's chat on pool `p` asking for its answer streamed. +#[cfg(linked_fold_on_driver)] +fn streamed_chat() -> serde_json::Value { + serde_json::json!({"model": "p", "stream": true, "max_tokens": 16, + "messages": [{"role": "user", "content": "hi"}]}) +} + +/// THE STREAM CEILING, NOT THE POOL'S TIMEOUT, bounds a streamed answer (the plane states +/// `ROUTE_STREAM`; v1.5.5 bounded a stream's whole send by the stream ceiling): a stream that +/// outlives its pool's one-second failover timeout is delivered whole. Before the plane stated the +/// stream, the door cut it at the pool's timeout. +#[cfg(linked_fold_on_driver)] +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn the_pools_door_bounds_a_stream_by_the_stream_ceiling_not_the_pools_timeout() { + let _one = ONE_PUBLISHER.lock().await; + let instance = "serve-door-stream-outlives-timeout"; + let _published = Withdrawn(instance); + let far = far_end_scripted(stream_pausing(2_000)).await; + let rig = rig( + instance, + RigOpts { + members: &[(far.port, 1)], + failover_timeout_secs: Some(1), + ..RigOpts::default() + }, + ) + .await; + let (status, _, body) = rig + .send("POST", "/v1/chat/completions", Some(streamed_chat())) + .await; + let body = String::from_utf8_lossy(&body); + assert_eq!(status, 200, "{body}"); + assert!( + body.contains("[DONE]") && body.contains("\"hi\""), + "the stream outlived the pool's timeout whole: {body}" + ); +} + +/// THE STREAM CEILING CUTS a streamed answer that outlives it, however long its pool's timeout +/// (RED before the plane stated `ROUTE_STREAM`: the door read the stream as buffered and waited the +/// pool's timeout out). +#[cfg(linked_fold_on_driver)] +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn the_pools_door_cuts_a_stream_at_the_stream_ceiling() { + let _one = ONE_PUBLISHER.lock().await; + let instance = "serve-door-stream-ceiling"; + let _published = Withdrawn(instance); + let far = far_end_scripted(stream_pausing(4_000)).await; + let rig = rig( + instance, + RigOpts { + members: &[(far.port, 1)], + failover_timeout_secs: Some(60), + stream_ceiling_secs: Some(1), + ..RigOpts::default() + }, + ) + .await; + let started = std::time::Instant::now(); + let (status, _, body) = rig + .send("POST", "/v1/chat/completions", Some(streamed_chat())) + .await; + let body = String::from_utf8_lossy(&body); + assert_eq!(status, 200, "the stream had begun: {body}"); + assert!( + body.contains("\"hi\""), + "the first frame reached the caller: {body}" + ); + assert!(!body.contains("[DONE]"), "the ceiling cut it: {body}"); + assert!( + started.elapsed() < std::time::Duration::from_millis(3_500), + "cut at the one-second ceiling, not when the far end ended: {:?}", + started.elapsed() + ); +} + +/// A MID-STREAM CUT IS NOT A REFUND (Part 2 #62): a streamed answer the far end cuts after its +/// first byte reached the caller keeps the member's lifetime budget unit it spent (RED before the +/// plane stated `ROUTE_STREAM`: the door refunded it as a buffered answer's). +#[cfg(linked_fold_on_driver)] +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn the_pools_door_keeps_the_budget_unit_of_a_stream_cut_after_its_first_byte() { + let _one = ONE_PUBLISHER.lock().await; + let instance = "serve-door-stream-cut"; + let _published = Withdrawn(instance); + let far = far_end_scripted(SeatScript { + head: STREAM_HEAD.to_string(), + pieces: vec![(0, frame(FRAMES[0])), (50, frame(FRAMES[1]))], + finish: None, + }) + .await; + let rig = rig( + instance, + RigOpts { + members: &[(far.port, 1)], + lane_budget: Some(5), + ..RigOpts::default() + }, + ) + .await; + let (status, _, body) = rig + .send("POST", "/v1/chat/completions", Some(streamed_chat())) + .await; + assert_eq!(status, 200, "{}", String::from_utf8_lossy(&body)); + assert_eq!( + rig.app.store.lane_budget_remaining(0), + Some(4), + "the streamed bytes were delivered: the unit is spent, not refunded" + ); +} + +/// THE CONTROL: a buffered answer the far end cuts reaches the caller whole or not at all, so its +/// budget unit is given back (v1.5.5 `engine/mod.rs:329-353`). +#[cfg(linked_fold_on_driver)] +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn the_pools_door_refunds_the_budget_unit_of_a_buffered_answer_cut() { + let _one = ONE_PUBLISHER.lock().await; + let instance = "serve-door-buffered-cut"; + let _published = Withdrawn(instance); + let far = far_end_scripted(SeatScript { + head: format!( + "HTTP/1.1 200 OK\r\ncontent-type: application/json\r\ncontent-length: {}\r\n\ + connection: close\r\n\r\n", + WHOLE.len() + ), + pieces: vec![(0, WHOLE.as_bytes()[..40].to_vec())], + finish: None, + }) + .await; + let rig = rig( + instance, + RigOpts { + members: &[(far.port, 1)], + lane_budget: Some(5), + ..RigOpts::default() + }, + ) + .await; + let _ = rig.chat().await; + assert_eq!(rig.app.store.lane_budget_remaining(0), Some(5)); +} + +/// A whole same-dialect answer that reports 1500 + 90 tokens. +#[cfg(linked_fold_on_driver)] +const WHOLE: &str = r#"{"id":"chatcmpl-9","object":"chat.completion","created":0,"model":"m0","choices":[{"index":0,"message":{"role":"assistant","content":"hello there"},"finish_reason":"stop"}],"usage":{"prompt_tokens":1500,"completion_tokens":90,"total_tokens":1590}}"#; + +/// A far end that answers [`WHOLE`] under its full length but writes only `prefix` of it, then +/// holds the connection `hold_ms` and closes (a cut). +#[cfg(linked_fold_on_driver)] +fn whole_cut_after(prefix: usize, hold_ms: u64) -> SeatScript { + SeatScript { + head: format!( + "HTTP/1.1 200 OK\r\ncontent-type: application/json\r\ncontent-length: {}\r\n\ + connection: close\r\n\r\n", + WHOLE.len() + ), + pieces: vec![ + (0, WHOLE.as_bytes()[..prefix].to_vec()), + (hold_ms, Vec::new()), + ], + finish: None, + } +} + +/// OWNER RULING Q31, oracle cell `route.failover|fo|primary-cut-body` (legacy +/// `nonstream_drop_billing_tests`, rows 338/342): a same-dialect non-stream answer the far end cuts +/// AFTER its `usage` bills exactly what the far end reported; one cut BEFORE its `usage` reported +/// nothing and bills nothing, never a floor over the relayed bytes. +#[cfg(linked_fold_on_driver)] +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn the_pools_door_bills_a_cut_non_stream_answer_only_what_its_far_end_reported() { + let _one = ONE_PUBLISHER.lock().await; + let instance = "serve-door-nonstream-cut"; + let _published = Withdrawn(instance); + let far = far_end_scripted(whole_cut_after(WHOLE.len() - 1, 0)).await; + let late = rig( + instance, + RigOpts { + members: &[(far.port, 1)], + ..RigOpts::default() + }, + ) + .await; + let _ = late.chat().await; + assert_eq!(late.tokens_after().await, 1590, "the far end's own report"); + drop(late); + drop(_published); + + let instance = "serve-door-nonstream-cut-early"; + let _published = Withdrawn(instance); + let usage_at = WHOLE.find(r#","usage""#).expect("the usage member"); + let far = far_end_scripted(whole_cut_after(usage_at, 0)).await; + let rig = rig( + instance, + RigOpts { + members: &[(far.port, 1)], + ..RigOpts::default() + }, + ) + .await; + let _ = rig.chat().await; + assert_eq!(rig.tokens_after().await, 0, "no report, no charge"); +} + +/// ITEM 367 (Q33 (a), told), legacy row 338: a same-dialect non-stream answer the CALLER drops +/// mid-relay was generated whole before its first byte, so it bills what it relayed: never 0. +#[cfg(linked_fold_on_driver)] +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn the_pools_door_bills_a_non_stream_answer_the_caller_dropped_mid_relay() { + use http_body_util::BodyExt as _; + let _one = ONE_PUBLISHER.lock().await; + let instance = "serve-door-nonstream-drop"; + let _published = Withdrawn(instance); + let usage_at = WHOLE.find(r#","usage""#).expect("the usage member"); + let far = far_end_scripted(whole_cut_after(usage_at, 5_000)).await; + let rig = rig( + instance, + RigOpts { + members: &[(far.port, 1)], + ..RigOpts::default() + }, + ) + .await; + let response = rig + .open( + "POST", + "/v1/chat/completions", + Some(serde_json::json!({"model": "p", "max_tokens": 16, + "messages": [{"role": "user", "content": "hi"}]})), + &[], + ) + .await; + assert_eq!(response.status().as_u16(), 200); + let mut body = response.into_body(); + let first = tokio::time::timeout(std::time::Duration::from_secs(5), body.frame()) + .await + .expect("the first bytes arrive before the far end's hold") + .expect("a frame") + .expect("relayed"); + assert!(first.data_ref().is_some_and(|b| !b.is_empty())); + // THE DISCONNECT: the caller goes before the answer's end. + drop(body); + // The driver finishes a buried unit at the start of its next one. + let rig = std::sync::Arc::new(rig); + let next = { + let rig = std::sync::Arc::clone(&rig); + tokio::spawn(async move { rig.chat().await }) + }; + assert_ne!( + rig.tokens_after().await, + 0, + "a non-stream answer dropped mid-relay never bills 0" + ); + next.abort(); +} + +/// SESSION AFFINITY (legacy rows 2-4; v1.5.5 `affinity_header_for` and the sticky position): the +/// plane states the session key it reads (the pool's header, `x-session-id` by default, else chat's +/// body `system`) and the kernel pins every unit with that key to one member; a unit without one +/// is spread by the weighted floor. +#[cfg(linked_fold_on_driver)] +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn the_pools_door_pins_a_session_to_one_member() { + let _one = ONE_PUBLISHER.lock().await; + let instance = "serve-door-affinity"; + let _published = Withdrawn(instance); + let fars = [ + seat_far_end(200, SERVED_BY_TWIN).await, + seat_far_end(200, SERVED_BY_TWIN).await, + seat_far_end(200, SERVED_BY_TWIN).await, + ]; + let rig = rig( + instance, + RigOpts { + members: &[(fars[0].port, 1), (fars[1].port, 1), (fars[2].port, 1)], + ..RigOpts::default() + }, + ) + .await; + let served = || fars.iter().map(|f| f.served()).collect::<Vec<_>>(); + let chat = serde_json::json!({"model": "p", "max_tokens": 16, + "messages": [{"role": "user", "content": "hi"}]}); + for _ in 0..6 { + let r = rig + .open( + "POST", + "/v1/chat/completions", + Some(chat.clone()), + &[("x-session-id", "session-42")], + ) + .await; + assert_eq!(r.status().as_u16(), 200); + let _ = axum::body::to_bytes(r.into_body(), 1 << 20).await; + } + let pinned = served(); + assert_eq!( + pinned.iter().filter(|n| **n == 6).count(), + 1, + "one member served the whole session: {pinned:?}" + ); + // The body's `system` is the key when no header names one. + let system = serde_json::json!({"model": "p", "max_tokens": 16, "system": "be brief", + "messages": [{"role": "user", "content": "hi"}]}); + let before = served(); + for _ in 0..3 { + assert_eq!( + rig.send("POST", "/v1/chat/completions", Some(system.clone())) + .await + .0, + 200 + ); + } + let after = served(); + let moved: Vec<usize> = after.iter().zip(&before).map(|(a, b)| a - b).collect(); + assert_eq!( + moved.iter().filter(|n| **n == 3).count(), + 1, + "one member served the system-keyed session: {moved:?}" + ); + // No key: the weighted floor spreads the units. + let before = served(); + for _ in 0..3 { + assert_eq!(rig.chat().await.0, 200); + } + let after = served(); + let moved = after.iter().zip(&before).filter(|(a, b)| a > b).count(); + assert!(moved >= 2, "unkeyed units spread: {before:?} -> {after:?}"); +} + +/// A POOL'S OWN AFFINITY HEADER (`affinity.header_name`) is the one its sessions are read from; the +/// default header is then no key. +#[cfg(linked_fold_on_driver)] +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn the_pools_door_reads_a_session_from_the_pools_own_affinity_header() { + let _one = ONE_PUBLISHER.lock().await; + let instance = "serve-door-affinity-header"; + let _published = Withdrawn(instance); + let fars = [ + seat_far_end(200, SERVED_BY_TWIN).await, + seat_far_end(200, SERVED_BY_TWIN).await, + seat_far_end(200, SERVED_BY_TWIN).await, + ]; + let rig = rig( + instance, + RigOpts { + members: &[(fars[0].port, 1), (fars[1].port, 1), (fars[2].port, 1)], + affinity_header: Some("x-user-id"), + ..RigOpts::default() + }, + ) + .await; + let served = || fars.iter().map(|f| f.served()).collect::<Vec<_>>(); + let chat = serde_json::json!({"model": "p", "max_tokens": 16, + "messages": [{"role": "user", "content": "hi"}]}); + let send = |name: &'static str, value: &'static str| { + let (rig, chat) = (&rig, chat.clone()); + async move { + let r = rig + .open("POST", "/v1/chat/completions", Some(chat), &[(name, value)]) + .await; + assert_eq!(r.status().as_u16(), 200); + let _ = axum::body::to_bytes(r.into_body(), 1 << 20).await; + } + }; + for _ in 0..4 { + send("x-user-id", "user-7").await; + } + assert_eq!( + served().iter().filter(|n| **n == 4).count(), + 1, + "the pool's header pins: {:?}", + served() + ); + let before = served(); + for _ in 0..3 { + send("x-session-id", "session-42").await; + } + let after = served(); + let moved = after.iter().zip(&before).filter(|(a, b)| a > b).count(); + assert!( + moved >= 2, + "the default header is not this pool's key: {before:?} -> {after:?}" + ); +} + // ── the core capabilities on the decisions plane (qa/capability-equality.json) ───────────────── // // The decisions plane is served only through its door on the kernel's plane driver, so each cell diff --git a/crates/busbar/tests/fixtures/computed_refusal_sites.txt b/crates/busbar/tests/fixtures/computed_refusal_sites.txt index 0c06063f93..93ba0478d8 100644 --- a/crates/busbar/tests/fixtures/computed_refusal_sites.txt +++ b/crates/busbar/tests/fixtures/computed_refusal_sites.txt @@ -8,5 +8,10 @@ busbar-llm/src/unit/verify.rs refusal.reason() busbar/src/root/units_admin/mod.rs verbs_reason(refusal.reason) busbar/src/root/units_admin/mod.rs verbs_reason(refusal.reason) busbar-kernel/src/plane_driver/mod.rs reason -busbar-kernel/src/plane_driver/mod.rs reason busbar-kernel/src/plane_driver/mod.rs cause +# Reviewed (lane-dg): the hook stage's refusal at the head of the route leg (HookVeto or +# DecodeFailed, the unit's money finished first; K5 de3ea6acc4) and the door's destination guard at +# verify, before admission (PoolNotPermitted or NoRate; the door flip b26be1ca6d). Neither is an +# overdraft ceiling or a stale slice. +busbar-kernel/src/plane_driver/mod.rs reason +busbar/src/root/door_steps.rs refusal.reason() diff --git a/crates/busbar/tests/fixtures/plane_driver_test_plane.rs b/crates/busbar/tests/fixtures/plane_driver_test_plane.rs index e05de7b633..df2b51015a 100644 --- a/crates/busbar/tests/fixtures/plane_driver_test_plane.rs +++ b/crates/busbar/tests/fixtures/plane_driver_test_plane.rs @@ -1452,9 +1452,10 @@ extern "C" fn refusal(_: *mut c_void, input: *const c_void, out: *mut c_void) -> unsafe { let i = &*input.cast::<RefusalIn>(); let o = &mut *out.cast::<RefusalOut>(); - // The reason crosses beside its text: a refusal whose code names another reason is FAULT. - let named = busbar_contract::abi::plane::reason_of(i.reason).map(|r| r.as_str().as_bytes()); - if named != Some(text(i.text)) { + // The reason crosses beside its text (`RefusalIn::text`, the kernel's own message: the + // reason's spelling, or the walk terminal's own words): a refusal whose code names no reason, + // or that carries no text, is FAULT. + if busbar_contract::abi::plane::reason_of(i.reason).is_none() || text(i.text).is_empty() { return RawOutcome::of(Outcome::Fault); } let mut body = [ diff --git a/crates/busbar/tests/hook_parity_driver/main.rs b/crates/busbar/tests/hook_parity_driver/main.rs new file mode 100644 index 0000000000..fdfe92947d --- /dev/null +++ b/crates/busbar/tests/hook_parity_driver/main.rs @@ -0,0 +1,37 @@ +// SPDX-License-Identifier: Apache-2.0 +// Copyright (C) 2026 Busbar Inc and contributors + +//! THE 1.5.5 HOOK PARITY SUITE, ON THE DRIVER (`BUSBAR-1.6.0.md` Part 3, section 12, "The switch": +//! the flip's gate is every hook test green on the driver; Appendix A: the hook parity suite is the +//! 184 v1.5.5 hook tests). +//! +//! The plane-origin tests of `v1.5.5`, each under its 1.5.5 name, with the values 1.5.5 asserted, +//! run with the plane under proof served on the plane driver: its REAL door, linked and loaded +//! through the one loader, driven by the kernel's plane driver with its hook stage bound +//! (`plane_driver::hooks`), every hook a 1.5.5 test's own in-process policy. What a hook sees of the +//! request is what the plane's `project` wrote; what the caller is answered is what the plane +//! rendered. The tests of kernel, loader and ranking origin are unchanged by the plane's path and +//! run in their own crates. +//! +//! THE COMPOSITION ROOT NAMES NO PLANE (BUSBAR-1.6.0.md THE DESIGN, the root is `Family::Neutral`), +//! so this suite is only the driver half: the rig (`rig.rs`) and the kernel's shared test steps. The +//! cases, written in the plane's own dialects, live in the plane's crate and are compiled in here +//! as `proofs` from the manifest's `[package.metadata.busbar.driver-proofs]` row (`build.rs` writes +//! the one `#[path]`); the plane under proof is the one whose Statement declares the `pools` map, +//! found by what it states, never by name. +//! +//! Development-only, like the switch it proves: built with a plane's development-only fold switch +//! (`linked_fold_on_driver`). + +#![cfg(linked_fold_on_driver)] + +// The linked plane doors (`LINKED_PLANE_DOORS`), generated from the manifest. +include!(concat!(env!("OUT_DIR"), "/linked_plane_doors.rs")); + +// The plane-owned cases (`mod proofs`), generated from the manifest. +include!(concat!(env!("OUT_DIR"), "/driver_proofs.rs")); + +#[path = "../../../busbar-kernel/tests/common/mod.rs"] +mod common; + +mod rig; diff --git a/crates/busbar/tests/hook_parity_driver/rig.rs b/crates/busbar/tests/hook_parity_driver/rig.rs new file mode 100644 index 0000000000..d4e5bc3c25 --- /dev/null +++ b/crates/busbar/tests/hook_parity_driver/rig.rs @@ -0,0 +1,700 @@ +// SPDX-License-Identifier: Apache-2.0 +// Copyright (C) 2026 Busbar Inc and contributors + +//! THE RIG the hook parity suite runs on: the REAL plane door serving the `pools` map (linked, loaded +//! through the one loader and driven through the one dispatcher), the kernel's plane driver with the +//! hook stage bound ([`BoundHooks`]), the kernel's passing steps, a far end that walks the configured +//! pools the way the egress walk does (its candidates, the hooks' constraint, a fallback pool holding +//! the restricts), and a caller that records what it was answered. No fixture plugin: the plane is +//! the real one, the hooks are the 1.5.5 tests' own in-process policies. +//! +//! The rig names no plane and no dialect. What it speaks to the plane under proof (the settings its +//! pools become, each far end's answer, the request a plain unit posts) is that plane's own, in the +//! plane's crate beside its cases (`crate::proofs::wire`). + +use std::collections::{HashMap, VecDeque}; +use std::future::Future; +use std::sync::{Arc, Mutex}; +use std::time::Duration; + +use busbar_contract::abi::mechanism::call::{Blob, Outcome as AbiOutcome, BLOB_OCTETS}; +use busbar_contract::abi::mechanism::lifecycle::{OpenIn, OpenOut}; +use busbar_contract::abi::plane::{PlaneOpenIn, PlaneOpenOut, UnitCount}; +use busbar_contract::abi::transport::STATUS_SUCCESS; +use busbar_contract::caps::{Canary, Outcome, Pass, ReasonCode, Route, StepName}; +use busbar_contract::hooks::{BudgetBucketState, RoutingPolicy}; +use busbar_kernel::hooks::{RequestedSignals, ResolvedPolicy}; +use busbar_kernel::plane_driver::{ + refusal_status, Arrival, BoundHooks, BufferCaps, CallerEnd, CallerFacts, CallerKey, CancelBill, + CandidateFacts, Candidates, Checkpoint, Constraint, DriverConfig, FarEnd, FarPiece, MoneySeam, + OutboundRequest, Pick, PlaneDriver, StageTaps, +}; +use busbar_kernel::slice::{ConcurrencyGauge, LeaseCell}; +use busbar_kernel::teller::{run_unit_async, AccrualMeter, Ended, Kernel, Run, UnitCtx}; +use busbar_plugin_loader::dispatch::{ + in_head, kinds::plane::Plane, load_linked, out_head, plane_calls::PlaneInstance, Bind, + DispatchConfig, Dispatcher, Frame, LinkedRow, NoSink, Plugin, +}; +use serde_json::Value; + +use super::common::{cell, ctx, TestUnits}; + +// ── the deployment ─────────────────────────────────────────────────────────────────────────────── + +/// One member of a pool: the model it serves (a key of `models:`), the model its far end names in +/// its answer (what tells a test WHICH member served), its tags, and whether it is down. +#[derive(Clone, Debug)] +pub struct Member { + pub model: &'static str, + pub label: &'static str, + pub provider: &'static str, + pub tags: Vec<String>, + pub dead: bool, +} + +/// A member served by the plane's default far end (`wire::DEFAULT_PROVIDER`). +pub fn member(model: &'static str, label: &'static str) -> Member { + Member { + model, + label, + provider: crate::proofs::wire::DEFAULT_PROVIDER, + tags: Vec::new(), + dead: false, + } +} + +impl Member { + pub fn tags(mut self, tags: &[&str]) -> Self { + self.tags = tags.iter().map(|t| t.to_string()).collect(); + self + } + pub fn dead(mut self) -> Self { + self.dead = true; + self + } + pub fn provider(mut self, p: &'static str) -> Self { + self.provider = p; + self + } +} + +/// A pool and its members. +#[derive(Clone, Debug)] +pub struct Pool { + pub name: &'static str, + pub members: Vec<Member>, +} + +/// The plane's settings for these pools, in the plane's own words. +fn settings(pools: &[&Pool]) -> Vec<u8> { + crate::proofs::wire::settings(pools) +} + +// ── the far end ───────────────────────────────────────────────────────────────────────────────── + +/// What a far end answers one attempt: the plane's own answer for that member's far end, whole. +fn answer_of(m: &Member) -> Vec<FarPiece> { + let body = crate::proofs::wire::answer_of(m); + vec![FarPiece { + bytes: serde_json::to_vec(&body).expect("answer serializes"), + status: Some((200, u32::from(STATUS_SUCCESS))), + last: true, + head: vec![(b"content-type".to_vec(), b"application/json".to_vec())], + ..FarPiece::default() + }] +} + +/// One step of a walk: a pool's member, or the walk's refusal (its status and words). +type Step = Result<(String, Member), (u32, String)>; +/// What a walk has left. +type Plan = VecDeque<Step>; +/// A reply head: the status and the fields. +type Head = (u32, Vec<(Vec<u8>, Vec<u8>)>); + +/// A WALK over the configured pools: the primary pool's live members in pool order (the hooks' +/// order first, when they set one, restricted to what they kept), then the fallback pool's, its +/// members held to the hooks' restricts (a required restrict nothing satisfies refuses, as 1.5.5 +/// failed closed at the fallback boundary). A dead member fails over before the plane sees it. +pub struct Far { + primary: Pool, + fallback: Option<Pool>, + constraint: Mutex<Option<Constraint>>, + plan: Mutex<Option<Plan>>, + tried: Mutex<usize>, + sent: Mutex<Vec<OutboundRequest>>, + current: Mutex<VecDeque<FarPiece>>, +} + +impl Far { + pub fn new(primary: Pool, fallback: Option<Pool>) -> Self { + Far { + primary, + fallback, + constraint: Mutex::new(None), + plan: Mutex::new(None), + tried: Mutex::new(0), + sent: Mutex::new(Vec::new()), + current: Mutex::new(VecDeque::new()), + } + } + + /// Every request the plane bound for the far end, in order. + pub fn sent(&self) -> Vec<OutboundRequest> { + self.sent.lock().unwrap().clone() + } + + fn plan(&self) -> Plan { + let c = self.constraint.lock().unwrap().clone().unwrap_or_default(); + let live: Vec<usize> = (0..self.primary.members.len()) + .filter(|i| c.keep.as_ref().is_none_or(|k| k.contains(i))) + .collect(); + let mut order: Vec<usize> = c + .order + .clone() + .unwrap_or_default() + .into_iter() + .filter(|i| live.contains(i)) + .collect(); + for i in &live { + if !order.contains(i) { + order.push(*i); + } + } + let mut plan: VecDeque<_> = order + .into_iter() + .map(|i| { + Ok(( + self.primary.name.to_string(), + self.primary.members[i].clone(), + )) + }) + .collect(); + if let Some(fb) = &self.fallback { + let tagged = fb + .members + .iter() + .enumerate() + .map(|(i, m)| (i, m.tags.as_slice())); + match c.enforce(tagged) { + Ok(kept) => plan.extend( + kept.into_iter() + .map(|i| Ok((fb.name.to_string(), fb.members[i].clone()))), + ), + Err(_) => plan.push_back(Err(( + 503, + "No upstream satisfies a required gate's restriction. Please retry shortly." + .to_string(), + ))), + } + } + plan + } +} + +fn candidate(i: usize, m: &Member) -> CandidateFacts { + CandidateFacts { + idx: i, + model: m.model.to_string(), + provider: m.provider.to_string(), + weight: 1, + tags: m.tags.clone(), + ..CandidateFacts::default() + } +} + +impl FarEnd for Far { + fn member<'a>( + &'a self, + _: &'a Pass<Route>, + _attempt_no: u32, + ) -> impl Future<Output = Pick> + Send + 'a { + let pick = { + let mut plan = self.plan.lock().unwrap(); + let plan = plan.get_or_insert_with(|| self.plan()); + match plan.pop_front() { + Some(Ok((pool, m))) => { + *self.tried.lock().unwrap() += 1; + Pick::Member { + name: m.model.to_string(), + pool, + passthrough: false, + provider: m.provider.to_string(), + } + } + Some(Err((status, text))) => Pick::Vetoed { status, text }, + None => Pick::Exhausted { + status: 503, + retry_after: None, + detail: "The service is temporarily overloaded. Please retry shortly.", + }, + } + }; + async move { pick } + } + + fn send<'a>( + &'a self, + _: &'a Pass<Route>, + request: OutboundRequest, + ) -> impl Future<Output = bool> + Send + 'a { + let m = self + .primary + .members + .iter() + .chain(self.fallback.iter().flat_map(|f| f.members.iter())) + .find(|m| m.model == request.member) + .cloned(); + self.sent.lock().unwrap().push(request); + let ok = match m { + Some(m) if !m.dead => { + *self.current.lock().unwrap() = answer_of(&m).into(); + true + } + _ => false, + }; + async move { ok } + } + + fn next<'a>( + &'a self, + _: &'a Pass<Route>, + ) -> impl Future<Output = Option<FarPiece>> + Send + 'a { + let piece = self.current.lock().unwrap().pop_front(); + async move { piece } + } + + fn remaining(&self, _: &Pass<Route>) -> Option<usize> { + self.plan.lock().unwrap().as_ref().map(VecDeque::len) + } + + fn failure(&self, _: &Pass<Route>) -> Option<&'static str> { + Some("connect") + } + + fn candidates(&self, _: &Pass<Route>) -> Option<Candidates> { + Some(Candidates { + pool: self.primary.name.to_string(), + members: self + .primary + .members + .iter() + .enumerate() + .map(|(i, m)| candidate(i, m)) + .collect(), + }) + } + + fn constrain(&self, _: &Pass<Route>, constraint: Constraint) { + *self.constraint.lock().unwrap() = Some(constraint); + } + + /// A request/response far end holds no attempt open for a later frame. + fn write<'a>( + &'a self, + _: &'a Pass<Route>, + _: busbar_kernel::plane_driver::OutboundRequest, + ) -> impl Future<Output = bool> + Send + 'a { + std::future::ready(false) + } +} + +// ── the caller and the books ───────────────────────────────────────────────────────────────────── + +/// What the caller was answered. +#[derive(Default)] +pub struct Caller { + head: Mutex<Option<Head>>, + bytes: Mutex<Vec<u8>>, +} + +impl CallerEnd for Caller { + fn head(&self, status: u32, fields: Vec<(Vec<u8>, Vec<u8>)>) { + *self.head.lock().unwrap() = Some((status, fields)); + } + + async fn write(&self, bytes: &[u8]) -> bool { + self.bytes.lock().unwrap().extend_from_slice(bytes); + true + } + + async fn write_text(&self, bytes: &[u8]) -> bool { + self.write(bytes).await + } +} + +/// A request/response caller sends nothing after its request. +impl busbar_kernel::plane_driver::SessionCaller for Caller { + fn read(&self) -> impl Future<Output = Option<Vec<u8>>> + Send + '_ { + std::future::ready(None) + } +} + +/// The money seam: nothing to settle in a hook test. +#[derive(Default)] +pub struct Book; + +impl MoneySeam for Book { + fn checkpoint(&self, _: &UnitCtx, _: &[UnitCount]) -> Checkpoint { + Checkpoint::Continue + } + fn cancelled(&self, _: &UnitCtx, _: &CancelBill) {} + fn abandoned(&self, _: &UnitCtx, _: Ended) {} + fn served(&self, _: &UnitCtx, _: &str, _: &str) {} +} + +// ── the hooks ──────────────────────────────────────────────────────────────────────────────────── + +/// What the hooks may know of the caller: its key by principal, its group, nothing else; a hook +/// handed the prompt leaves its access amendment on the kernel's node journal. +#[derive(Default)] +pub struct Callers { + pub keys: HashMap<String, CallerKey>, + /// `principal -> the groups it is in` (self and ancestors). + pub groups: HashMap<String, Vec<String>>, +} + +impl CallerFacts for Callers { + fn key(&self, principal: &str) -> Option<CallerKey> { + self.keys.get(principal).cloned() + } + fn in_groups(&self, principal: Option<&str>, groups: &[String]) -> bool { + if groups.is_empty() { + return true; + } + let Some(mine) = principal.and_then(|p| self.groups.get(p)) else { + return false; + }; + groups.iter().any(|g| mine.contains(g)) + } + fn rate_headroom(&self, _: &str, _: &str) -> Option<f64> { + None + } + fn budget(&self, _: &str) -> Vec<BudgetBucketState> { + Vec::new() + } +} + +/// The deployment's hooks, as a test states them. +#[derive(Default)] +pub struct Hooks { + pub rewrites: Vec<(Duration, Arc<dyn RoutingPolicy>)>, + pub gates: Vec<(u16, ResolvedPolicy)>, + pub pool_rewrites: Vec<(Duration, Arc<dyn RoutingPolicy>)>, + pub pool_gates: Vec<(u16, ResolvedPolicy)>, + pub policy: Option<ResolvedPolicy>, + pub taps: StageTaps, + pub callers: Callers, +} + +/// The process's one request-id counter, as the binder is handed it. +pub fn counter() -> Arc<dyn Fn() -> u64 + Send + Sync> { + static NEXT: std::sync::atomic::AtomicU64 = std::sync::atomic::AtomicU64::new(1); + Arc::new(|| NEXT.fetch_add(1, std::sync::atomic::Ordering::SeqCst)) +} + +impl Hooks { + fn bound(self, pool: &str, dialects: Vec<String>) -> BoundHooks { + let mut gates = self.gates; + gates.sort_by_key(|(p, _)| *p); + BoundHooks { + rewrites: self.rewrites, + gates, + pool_rewrites: std::iter::once((pool.to_string(), self.pool_rewrites)).collect(), + pool_gates: std::iter::once((pool.to_string(), self.pool_gates)).collect(), + pool_policies: self + .policy + .map(|p| (pool.to_string(), p)) + .into_iter() + .collect(), + taps: self.taps, + requested: RequestedSignals::default(), + next_request_id: counter(), + caller: Arc::new(self.callers), + dialects, + } + } +} + +// ── one unit ───────────────────────────────────────────────────────────────────────────────────── + +fn octets(b: &[u8]) -> Blob { + Blob { + ptr: b.as_ptr(), + len: b.len(), + fmt: BLOB_OCTETS, + flags: 0, + } +} + +/// THE LINKED DOOR SERVING THE `pools` MAP, found by what its Statement declares +/// (`LINKED_PLANE_DOORS`, the manifest's plane-door rows): the plane under proof. +pub(crate) fn pools_door() -> busbar_contract::abi::mechanism::door::DoorFn { + crate::LINKED_PLANE_DOORS + .iter() + .copied() + .find(|door| { + let dispatcher = Dispatcher::new(DispatchConfig::default()); + let row = LinkedRow::of(*door).expect("a linked door states itself"); + load_linked::<Plane>( + &row, + Bind { + instance: Arc::from("pools-door-probe"), + max_inflight_cap: 64, + sink: Arc::new(NoSink), + dispatcher: dispatcher.adopter(), + conns: busbar_plugin_loader::dispatch::ConnTable::Probe, + }, + ) + .expect("a linked door binds") + .served() + .section + == busbar_contract::section::RESERVED_POOLS_KEY + }) + .expect("the fold switch links the door serving the `pools` map") +} + +/// The plane under proof's dialects, in its tail's order, as its door states them. +pub(crate) fn dialects() -> Vec<String> { + let dispatcher = Dispatcher::new(DispatchConfig::default()); + let row = LinkedRow::of(pools_door()).expect("the plane door states its Statement"); + load_linked::<Plane>( + &row, + Bind { + instance: Arc::from("the-plane-facts"), + max_inflight_cap: 64, + sink: Arc::new(NoSink), + dispatcher: dispatcher.adopter(), + conns: busbar_plugin_loader::dispatch::ConnTable::Probe, + }, + ) + .expect("the plane door loads") + .served() + .dialects + .iter() + .map(|d| (*d).to_string()) + .collect() +} + +/// The door serving the `pools` map, linked, opened over `settings`. +fn door(dispatcher: &Dispatcher, settings: &[u8]) -> Plugin<Plane> { + let row = LinkedRow::of(pools_door()).expect("the plane door states its Statement"); + let plugin = load_linked::<Plane>( + &row, + Bind { + instance: Arc::from("the-plane"), + max_inflight_cap: 64, + sink: Arc::new(NoSink), + dispatcher: dispatcher.adopter(), + conns: busbar_plugin_loader::dispatch::ConnTable::Probe, + }, + ) + .expect("the plane door loads"); + let mut open = Frame::new( + PlaneOpenIn { + open: OpenIn { + head: in_head(), + host: std::ptr::null(), + settings: octets(settings), + secrets: std::ptr::null(), + secrets_len: 0, + generation: 1, + err_buf: std::ptr::null_mut(), + err_cap: 0, + }, + public_url: busbar_contract::abi::mechanism::call::AbiStr { + ptr: std::ptr::null(), + len: 0, + }, + owned: Blob { + ptr: std::ptr::null(), + len: 0, + fmt: busbar_contract::abi::mechanism::call::BLOB_ABSENT, + flags: 0, + }, + providers: std::ptr::null(), + providers_len: 0, + }, + PlaneOpenOut { + open: OpenOut { + head: out_head(), + instance: std::ptr::null_mut(), + err_len: 0, + }, + snapshot: std::ptr::null(), + }, + ); + let (called, _) = plugin.open(&mut open); + assert_eq!(called.outcome, AbiOutcome::Ready, "the plane door opens"); + plugin +} + +/// What one unit came to. +pub struct Answered { + pub status: u32, + pub fields: Vec<(Vec<u8>, Vec<u8>)>, + pub body: Vec<u8>, + pub sent: Vec<OutboundRequest>, + pub outcome: Outcome, +} + +impl Answered { + /// The answer's body as JSON. + pub fn json(&self) -> Value { + serde_json::from_slice(&self.body).unwrap_or(Value::Null) + } + /// The model the answer names: which member served. + pub fn model(&self) -> String { + self.json() + .get("model") + .and_then(Value::as_str) + .unwrap_or_default() + .to_string() + } + /// The answer's body as text. + pub fn text(&self) -> String { + String::from_utf8_lossy(&self.body).into_owned() + } +} + +/// THE RIG: one deployment (its pools and its hooks), ready to drive units. +pub struct Rig { + driver: PlaneDriver, + primary: Pool, + fallback: Option<Pool>, + _plugin: Plugin<Plane>, +} + +impl Rig { + pub fn new(primary: Pool, fallback: Option<Pool>, hooks: Hooks) -> Self { + let dispatcher = Arc::new(Dispatcher::new(DispatchConfig { + workers: 2, + ..DispatchConfig::default() + })); + let mut pools = vec![&primary]; + if let Some(f) = &fallback { + pools.push(f); + } + let plugin = door(&dispatcher, &settings(&pools)); + let calls = Arc::new(PlaneInstance::new(plugin.clone(), dispatcher, 1)); + let refusal_statuses = calls.refusal_statuses(); + let served = plugin.served(); + let dialects: Vec<String> = served.dialects.iter().map(|d| (*d).to_string()).collect(); + let op_classes: Vec<_> = served + .op_classes + .iter() + .map(|c| busbar_contract::caps::OpClassId::new(c)) + .collect(); + let driver = PlaneDriver::new( + calls, + DriverConfig { + caps: BufferCaps::default(), + op_classes, + status_of: refusal_status, + refusal_statuses, + caller_refs: None, + }, + Arc::new(Book), + Arc::new(busbar_kernel::host_services::KernelServices::new()), + ("pools", &serde_yaml::Value::Null), + ) + .expect("the plane instance is admitted") + .with_hooks(Arc::new(hooks.bound(primary.name, dialects))); + Rig { + driver, + primary, + fallback, + _plugin: plugin, + } + } + + /// One unit: `body` posted to `target` with `fields`, the kernel steps `steps`. + pub async fn fire_with( + &self, + steps: &TestUnits, + target: &str, + fields: &[(&str, &str)], + body: &[u8], + ) -> Answered { + let far = Far::new(self.primary.clone(), self.fallback.clone()); + let caller = Caller::default(); + let arrival = Arrival { + claim: 0, + method: b"POST".to_vec(), + target: target.as_bytes().to_vec(), + fields: fields + .iter() + .map(|(n, v)| (n.as_bytes().to_vec(), v.as_bytes().to_vec())) + .collect(), + body: Arc::from(body), + }; + let units = self.driver.unit(steps, &far, &caller, arrival, 0); + let outcome = drive(&units).await; + let rendered = units.take_rendered(); + let (status, fields, body) = match rendered { + Some(r) => (r.status, r.fields, r.body), + None => { + let head = caller.head.lock().unwrap().clone().unwrap_or_default(); + (head.0, head.1, caller.bytes.lock().unwrap().clone()) + } + }; + Answered { + status, + fields, + body, + sent: far.sent(), + outcome, + } + } + + /// One plain unit (the plane's own plain request, `wire::FIRE_TARGET` with `wire::FIRE_FIELDS`) + /// with every kernel step passing. + pub async fn fire(&self, body: &Value) -> Answered { + self.fire_with( + &TestUnits::passing(), + crate::proofs::wire::FIRE_TARGET, + crate::proofs::wire::FIRE_FIELDS, + &serde_json::to_vec(body).expect("body serializes"), + ) + .await + } + + /// One unit refused at authentication. + pub async fn fire_unauthenticated(&self, target: &str, body: &[u8]) -> Answered { + self.fire_with( + &TestUnits::refusing(StepName::Authenticate, ReasonCode::Unauthenticated), + target, + &[("content-type", "application/json")], + body, + ) + .await + } +} + +/// Run one unit through the one loop; its outcome. +async fn drive<S, F, C>(units: &busbar_kernel::plane_driver::PlaneUnits<'_, S, F, C>) -> Outcome +where + S: busbar_kernel::plane_driver::DriverSteps + Sync, + F: FarEnd, + C: busbar_kernel::plane_driver::SessionCaller, +{ + let kernel = Kernel::new(); + let (gauge, canary, leases, meter) = ( + ConcurrencyGauge::new(), + Canary::new(), + LeaseCell::new(), + AccrualMeter::new(), + ); + let cell = cell(&kernel); + let run = Run { + cell: &cell, + parent: None, + leases: &leases, + gauge: &gauge, + canary: &canary, + meter: &meter, + }; + match run_unit_async(&kernel, units, &ctx(7), run, units).await { + Ended::Settled { end, .. } => end.outcome(), + Ended::AlreadySettled => panic!("nothing else holds this unit's cell"), + } +} diff --git a/crates/busbar/tests/plane_integration.rs b/crates/busbar/tests/plane_integration.rs index 4d1e93442d..1fa81ab97f 100644 --- a/crates/busbar/tests/plane_integration.rs +++ b/crates/busbar/tests/plane_integration.rs @@ -665,6 +665,10 @@ mod metrics_scrape { } } +/// The kernel's body-ingress convenience routes (`/{name}/v1/messages`, +/// `/{provider}/{model}/v1/messages`): 1.5.5 surface, mounted where a plane installed body ingress. +const BODY_INGRESS_ROUTES: [&str; 2] = ["/{name}/v1/messages", "/{provider}/{model}/v1/messages"]; + /// THE PLANE-BOUNDARY RATCHET (1.6.0), driven by the ROUTER TABLE rather than by a hand-listed /// sample. /// @@ -823,6 +827,27 @@ async fn an_audience_bound_token_is_confined_to_its_door_plane() { RouteAuth::None — the bypass set this names is not the one the router built" ); } + // THE BODY-INGRESS CONVENIENCE ROUTES (1.5.5 surface): the kernel mounts them only where a + // plane installed body ingress (`busbar_kernel::ingress::arrival::any_body_ingress`), and this + // app's linked planes install it, as the default build's boot does. So both are mounted here, + // each guarded (a key's bar, never `RouteAuth::None`), and the walk below must reach both: an + // anonymous caller refused, a keyed caller admitted. + assert!( + busbar_kernel::ingress::arrival::any_body_ingress(), + "the linked planes installed no body ingress, so the convenience routes are not mounted \ + and this walk proves nothing about them" + ); + for pattern in BODY_INGRESS_ROUTES { + assert!( + core_routes + .iter() + .any(|(path, method, auth)| path == pattern + && method == "POST" + && *auth != busbar_contract::abi::mechanism::route::RouteAuth::None), + "{pattern} is a body-ingress convenience route, mounted guarded on POST wherever body \ + ingress is installed; the router built none: {core_routes:?}" + ); + } let boot_plugin_paths: Vec<String> = busbar_kernel::boot_route_paths_of(&app); assert!( !core_routes.is_empty(), @@ -872,6 +897,7 @@ async fn an_audience_bound_token_is_confined_to_its_door_plane() { .to_string(); let mut checked = 0usize; + let mut body_walked: Vec<String> = Vec::new(); let mut door_checked = 0usize; for route in &core_routes { let path = concrete(&route.0); @@ -999,8 +1025,29 @@ async fn an_audience_bound_token_is_confined_to_its_door_plane() { route.1, path ); + if BODY_INGRESS_ROUTES.contains(&route.0.as_str()) { + assert_eq!( + anon, 401, + "an anonymous caller is refused on the convenience route {} {}", + route.1, path + ); + assert!( + plain.status() != 401 && plain.status() != 403, + "a keyed caller is admitted on the convenience route {} {}: {}", + route.1, + path, + plain.status() + ); + body_walked.push(route.0.clone()); + } checked += 1; } + for pattern in BODY_INGRESS_ROUTES { + assert!( + body_walked.iter().any(|p| p == pattern), + "the convenience route {pattern} was never walked: {body_walked:?}" + ); + } assert_eq!( door_checked, core_routes diff --git a/crates/plugin-loader/src/dispatch/kinds/plane.rs b/crates/plugin-loader/src/dispatch/kinds/plane.rs index 1d32de1e21..5e68cabc1f 100644 --- a/crates/plugin-loader/src/dispatch/kinds/plane.rs +++ b/crates/plugin-loader/src/dispatch/kinds/plane.rs @@ -34,7 +34,7 @@ use std::sync::Arc; -use busbar_contract::abi::mechanism::call::{AbiStr, Outcome}; +use busbar_contract::abi::mechanism::call::{AbiStr, Blob, Outcome}; use busbar_contract::abi::mechanism::call::{InHead, OutHead}; use busbar_contract::abi::mechanism::check::{fault, reported, Fault, Rule}; use busbar_contract::abi::mechanism::door::{Statement, SECTION_CONSUMED, SECTION_DECLARING}; @@ -114,6 +114,9 @@ pub struct PlaneFacts { pub struct ServedFacts { /// The key of the one section the Statement declares. pub section: &'static str, + /// The keys of the sections the Statement reads beside it (`SECTION_CONSUMED`), in its order: + /// the plane opens with all of them, one `{section: value}` object (spec Part 1 §4). + pub consumed: Vec<&'static str>, /// The keys of the other sections it owns (neither declaring nor consumed): what it opens /// with beside its settings (`PlaneOpenIn::owned`). pub owns: Vec<&'static str>, @@ -159,13 +162,18 @@ pub struct ServedFacts { /// The tail's dialects, in order. pub dialects: Vec<&'static str>, /// The tail's `dialect_auth`: each dialect's default outbound style, by its dialect index - /// (THE DESIGN §6 step 2: a provider entry's `auth:`, else this). - pub dialect_auth: Vec<(u32, &'static str)>, + /// (THE DESIGN §6 step 2: a provider entry's `auth:`, else this), and the style's parameters + /// for it, a JSON object's text (empty: none; ARCHITECT RULING 2026-10-03, Q-L6-AUTHPARAMS). + pub dialect_auth: Vec<(u32, &'static str, &'static [u8])>, /// The tail's `label`, `subject_noun` and `admin_noun` (what the kernel's registry entry names /// the plane and one registration by). pub nouns: (&'static str, &'static str, &'static str), /// The tail's billable classes' unit families, parallel to [`Self::billable_classes`]. pub billable_families: Vec<&'static str>, + /// Whether the tail states `TAIL_PROBES`: the plane answers the kernel's health probe unit. + pub probes: bool, + /// Whether the tail states `TAIL_FALLBACK`: the plane is the catch-all. + pub fallback: bool, /// The tail's `caller_credential_refusal`; `""` = it states none. pub caller_credential_refusal: &'static str, /// The tail's `TAIL_*` flags (the fallback catch-all, probes, the gate-first hook order). @@ -240,6 +248,12 @@ fn tail_facts(st: &Statement) -> Result<PlaneFacts, String> { .collect(); check_refusal_statuses(&refusal_statuses, tail.dialects_len as u64) .map_err(|f| format!("the plane tail breaks {:?} at {}", f.rule, f.field))?; + // Each dialect's default style and its parameters, judged before they are kept. + busbar_contract::abi::plane::check::check_dialect_auth( + &listed(tail.dialect_auth, tail.dialect_auth_len), + tail.dialects_len as u64, + ) + .map_err(|f| format!("the plane tail breaks {:?} at {}", f.rule, f.field))?; Ok(PlaneFacts { bounds: Bounds::of(&tail), refusal_statuses, @@ -249,6 +263,11 @@ fn tail_facts(st: &Statement) -> Result<PlaneFacts, String> { .iter() .find(|s| s.flags & SECTION_DECLARING != 0) .map_or("", |s| kept(s.name)), + consumed: sections + .iter() + .filter(|s| s.flags & SECTION_CONSUMED != 0) + .map(|s| kept(s.name)) + .collect(), owns: sections .iter() .filter(|s| s.flags & (SECTION_DECLARING | SECTION_CONSUMED) == 0) @@ -329,7 +348,7 @@ fn tail_facts(st: &Statement) -> Result<PlaneFacts, String> { .collect(), dialect_auth: listed(tail.dialect_auth, tail.dialect_auth_len) .into_iter() - .map(|d| (d.dialect, kept(d.style))) + .map(|d| (d.dialect, kept(d.style), kept_blob(d.params))) .collect(), nouns: ( kept(tail.label), @@ -340,6 +359,8 @@ fn tail_facts(st: &Statement) -> Result<PlaneFacts, String> { .into_iter() .map(|c| kept(c.family)) .collect(), + probes: tail.flags & busbar_contract::abi::plane::TAIL_PROBES != 0, + fallback: tail.flags & busbar_contract::abi::plane::TAIL_FALLBACK != 0, caller_credential_refusal: kept(tail.caller_credential_refusal), tail_flags: tail.flags, stream_ceiling_secs: tail.stream_ceiling_secs, @@ -358,6 +379,17 @@ fn kept(s: AbiStr) -> &'static str { String::from_utf8_lossy(bytes).into_owned().leak() } +/// A tail blob's bytes, kept for the process (empty when absent). +fn kept_blob(b: Blob) -> &'static [u8] { + if b.ptr.is_null() || b.len == 0 { + return &[]; + } + // SAFETY: `check_dialect_auth` judged the blob (`tail_facts`): a non-NULL span of `len` bytes of + // `'static` plugin data, read here while the plugin is loaded and copied. + let bytes = unsafe { std::slice::from_raw_parts(b.ptr, b.len) }; + bytes.to_vec().leak() +} + /// A tail list of `n` `T`s at `p`, copied. fn listed<T: Copy>(p: *const T, n: usize) -> Vec<T> { if p.is_null() || n == 0 { @@ -579,6 +611,7 @@ pub fn registration( record_kinds: declared.record_kinds.clone(), trust_keys: declared.trust_keys.clone(), caller_credential_refusal: (!refusal.is_empty()).then_some(refusal), + fallback: served.fallback, validate: Arc::new(move |settings: &[u8]| validate(&judge, settings)), facing: Arc::new( move |settings: &[u8], diff --git a/qa/DESIGN-BINDINGS.md b/qa/DESIGN-BINDINGS.md index 9b478ce961..dc748cf420 100644 --- a/qa/DESIGN-BINDINGS.md +++ b/qa/DESIGN-BINDINGS.md @@ -27,12 +27,12 @@ The three words are not interchangeable: | # | Surface | Status | Verdict | Why | Checks | |---|---|---|---|---|---| | PB-0 | master rule | mapped | PASS | | gate: `xtask/src/gates/inventory_coverage.rs`<br>gate: `xtask/src/gates/construction.rs` | -| PB-1 | hook `on_empty` | mapped | PASS | | test: `enforce_restricts_reapplies_compliance_tags_across_pools`<br>test: `multi_restrict_disjoint_intersection_fails_closed` | +| PB-1 | hook `on_empty` | mapped | PASS | | test: `crates/busbar-llm/src/engine/tests/hook_seam_tests.rs::enforce_restricts_reapplies_compliance_tags_across_pools`<br>test: `crates/busbar-llm/src/engine/tests/hook_seam_tests.rs::multi_restrict_disjoint_intersection_fails_closed` | | PB-2 | per-lane `max_concurrent` | mapped | PASS | | test: `excluded_reasons_records_at_capacity`<br>test: `at_capacity_reject_sheds_503_not_queued`<br>test: `queue_dispatches_when_permit_frees_before_deadline`<br>test: `queue_times_out_to_503_when_capacity_never_frees` | | PB-3 | tripped / budget-exhausted / at-capacity lanes | mapped | PASS | | test: `ordered_walk_skips_tripped_preferred_to_next`<br>test: `ordered_walk_skips_excluded_preferred`<br>test: `at_capacity_plus_tripped_member_rejects_503`<br>test: `least_bad_never_reaches_an_excluded_member`<br>test: `strengthened_lane_availability_invariant` | | PB-4 | `on_exhausted` terminals | mapped | PASS | | oracle-cell: `crosscut.traps\|exhausted-retry-after-floor`<br>oracle-cell: `route.failover\|fo\|all-down`<br>oracle-cell: `route.failover\|lb\|member-down`<br>test: `at_capacity_default_no_on_exhausted_sheds_503`<br>test: `retry_after_reflects_cooldown_when_a_member_is_tripped`<br>test: `retry_after_has_saturation_floor_when_purely_at_capacity`<br>test: `retry_after_empty_candidate_set_uses_floor_not_one`<br>test: `least_bad_still_serves_the_only_member_after_it_was_tried`<br>test: `a_fallback_pool_applies_its_own_exclusions`<br>test: `at_capacity_fallback_chain_spills_through_to_third_pool`<br>test: `at_capacity_self_referential_fallback_stays_503`<br>test: `queue_skips_wait_and_rejects_when_no_candidate_at_capacity`<br>test: `queue_won_permit_but_breaker_now_open_never_dispatches`<br>test: `queue_two_waiters_one_freed_permit_wakes_exactly_one`<br>test: `test_scrape_gauges_pool_queued_reads_live_depth` | | PB-5 | pick order | mapped | PASS | | oracle-cell: `route.failover\|fo\|all-up`<br>test: `crates/busbar-llm/src/engine/tests/ordered_walk_tests.rs::sticky_affinity_never_selects_zero_weight_drained_member`<br>test: `sticky_fall_through_records_reason`<br>test: `test_sticky_yields_when_tripped`<br>test: `crates/busbar-llm/src/engine/tests/hook_seam_tests.rs::order_last_in_chain_wins`<br>test: `crates/busbar-llm/src/engine/tests/hook_seam_tests.rs::stale_order_filtered_against_post_restrict_set`<br>test: `crates/busbar-llm/src/engine/tests/hook_seam_tests.rs::last_order_gate_filtered_to_empty_abstains_to_base_not_to_a_lower_gate`<br>test: `ordered_walk_falls_through_to_swrr_when_no_preferred_ready`<br>test: `ordered_walk_empty_order_is_swrr` | -| PB-6 | hook seats for migrated hooks | mapped | PASS | | test: `global_gate_reject_short_circuits_the_request`<br>test: `global_request_stage_tap_fires_on_a_real_dispatched_request`<br>test: `migrated_request_hook_seats_after_admit_before_candidate` | +| PB-6 | hook seats for migrated hooks | mapped | PASS | | test: `crates/busbar-llm/src/engine/tests/hook_seam_tests.rs::global_gate_reject_short_circuits_the_request`<br>test: `crates/busbar-llm/src/engine/tests/hook_seam_tests.rs::global_request_stage_tap_fires_on_a_real_dispatched_request`<br>test: `crates/busbar-llm/src/engine/tests/hook_seat_order_tests.rs::migrated_request_hook_seats_after_admit_before_candidate` | | PB-7 | inbound shed | mapped | PASS | | test: `inbound_cap_sheds_the_excess_and_serves_the_admitted`<br>test: `a_saturated_gate_sheds_instead_of_parking_the_caller`<br>oracle-cell: `concurrency\|inbound-shed\|n8` | | PB-8 | request bounds for `http`/`sse` units | mapped | PASS | | oracle-cell: `route.failover\|fo\|primary-down`<br>test: `test_saturated_lane_respects_deadline_no_infinite_spin`<br>test: `test_context_length_failover_no_penalty`<br>test: `test_prefers_larger_context_max`<br>test: `a_changed_upstream_timeout_rebuilds_the_client_an_unrelated_apply_reuses_it`<br>test: `test_member_override_wins_over_model_default`<br>test: `test_attempt_cap_budget_floor`<br>test: `route_deadline_503_carries_detail_request_timeout`<br>test: `pick_among_none_lands_on_on_exhausted_with_overloaded_and_retry_after` | | PB-9 | revocation / rotation | mapped | PASS | | test: `rotate_invalidates_the_outstanding_signed_token`<br>test: `local_revoke_rejects_the_very_next_auth_attempt`<br>test: `peer_revoke_written_to_the_store_is_honoured_within_the_window` | @@ -54,7 +54,7 @@ The three words are not interchangeable: | PB-25 | usage absent from a stream | mapped | PASS | | test: `stream_without_usage_frame_bills_zero_on_every_dialect` | | PB-26 | pre-charge vs post-charge exits | mapped | PASS | | oracle-cell: `billing\|key-usage\|upstream-down-refunded`<br>test: `test_unpriced_passthrough_model_rejected_when_rate_card_present`<br>oracle-cell: `http.crosscut\|unknown-path\|openai-suffix`<br>test: `test_governance_pool_acl_403_openai_native_envelope`<br>test: `test_governance_pool_acl_403_bedrock_native_envelope`<br>test: `test_fallback_pool_acl_denies_key_not_allowed_on_fallback_target`<br>test: `test_finish_refunds_flat_fee_on_non_2xx_keeps_on_2xx`<br>test: `test_pre_routing_failure_does_not_refund_prior_charge`<br>test: `test_bedrock_non_object_body_is_400` | | PB-27 | zero-token ends | mapped | PASS | | oracle-cell: `billing\|key-usage\|upstream-down-refunded`<br>oracle-cell: `route.failover\|fo\|primary-cut-stream`<br>test: `test_mid_stream_transport_error_does_not_bill_partial_usage`<br>test: `test_streaming_translate_abort_trips_breaker_and_skips_billing`<br>test: `test_untranslatable_2xx_does_not_charge_tokens`<br>test: `test_cross_protocol_nonstream_over_cap_body_returns_500_uncharged`<br>test: `test_streaming_pre_first_byte_transport_error_refunds_budget`<br>test: `test_streaming_nonsse_post_first_byte_cut_refunds_the_lane_unit`<br>test: `test_streaming_nonsse_mid_body_transport_error_records_transient`<br>test: `test_cancel_drop_bills_partial_tokens`<br>test: `test_cancel_drop_mid_stream_refunds_budget`<br>test: `test_finish_refunds_flat_fee_on_non_2xx_keeps_on_2xx` | -| PB-28 | `on_empty: weighted` semantics | mapped | PASS | | test: `enforce_restricts_reapplies_compliance_tags_across_pools` | +| PB-28 | `on_empty: weighted` semantics | mapped | PASS | | test: `crates/busbar-llm/src/engine/tests/hook_seam_tests.rs::enforce_restricts_reapplies_compliance_tags_across_pools` | | PB-29 | gate vs routing-policy 503 literals | mapped | PASS | | test: `decision_gate_that_cannot_complete_has_its_own_503_body`<br>test: `decision_gate_restriction_leaving_no_lane_has_its_own_503_body`<br>test: `base_policy_that_cannot_select_has_its_own_503_body`<br>test: `base_policy_restriction_leaving_no_lane_has_its_own_503_body`<br>test: `the_four_503_literals_are_distinct` | | PB-30 | protocol detection | mapped | PASS | | oracle-cell: `http.crosscut\|api-prefix\|data`<br>oracle-cell: `http.crosscut\|unknown-path\|anthropic-beta`<br>oracle-cell: `http.crosscut\|unknown-path\|bare`<br>test: `resolver_table`<br>test: `test_api_root_unmatched_paths_speak_the_admin_envelope`<br>test: `test_fallback_bedrock_404_is_native_envelope_with_amzn_headers`<br>test: `test_fallback_openai_404_is_json_no_amzn_headers` | | PB-31 | plugin-declared routes | mapped | PASS | | test: `confinement_rejects_hook_out_of_namespace`<br>test: `confinement_export_metrics_exception_and_reserved_routes`<br>test: `route_dispatches_to_handle_http_and_resolves_live_after_swap`<br>test: `head_dispatches_to_the_declared_get_route`<br>test: `over_cap_response_headers_are_rejected_not_silently_truncated`<br>test: `under_cap_response_headers_relay_unchanged`<br>test: `paths_awaiting_restart_flags_only_newly_declared_unmounted_paths`<br>test: `collision_is_a_loud_failure_naming_the_owner` | @@ -72,7 +72,7 @@ The three words are not interchangeable: | PB-43 | operational routes | mapped | PASS | | oracle-cell: `hooks\|metrics-hooks@hooks-webrequest`<br>oracle-cell: `ops.scrape\|metricshooks\|key`<br>oracle-cell: `ops.scrape\|metrics\|key`<br>oracle-cell: `ops.scrape\|stats\|key`<br>test: `test_is_ready_any_cell_false_when_every_cell_open`<br>test: `test_is_ready_any_cell_true_when_a_pool_cell_is_ready`<br>test: `test_is_ready_is_side_effect_free`<br>test: `healthz_returns_a_real_response_not_the_default`<br>test: `healthz_with_no_model_lane_is_ready_when_a_configured_plane_opened`<br>test: `healthz_with_no_model_lane_is_unready_when_the_planes_open_refused`<br>test: `healthz_with_model_lanes_every_cell_open_stays_unready_beside_an_opened_plane`<br>test: `thread_per_core_boots_and_serves_healthz`<br>test: `counter_renders_with_hook_label_and_verbatim_name`<br>test: `histogram_renders_as_summary`<br>test: `busbar_prefix_is_reserved`<br>test: `type_conflict_for_shared_name_is_dropped`<br>test: `test_stats_reports_at_capacity_when_lane_saturated`<br>test: `test_stats_limit_is_numeric_alias_of_max_concurrent`<br>test: `test_recovery_hint_ms` | | PB-44 | `in_flight_reserve` | mapped | PASS | | test: `a_full_table_refuses_every_counted_unit_at_the_cap_and_holds_no_slot_back`<br>test: `the_administrative_listener_and_the_heartbeat_are_outside_the_cap`<br>test: `the_cap_holds_when_everything_arrives_at_once` | | PB-45 | revoke / rotate | mapped | PASS | | test: `a_failed_read_does_not_close_the_staleness_window`<br>test: `a_successful_read_unions_and_closes_the_window`<br>test: `a_hung_store_does_not_park_the_reactor`<br>test: `test_key_state_distinguishes_disable_revoke_and_tombstone`<br>test: `local_revoke_rejects_the_very_next_auth_attempt` | -| PB-46 | migrated hook seats | mapped | PASS | | test: `migrated_request_hook_seats_after_admit_before_candidate`<br>test: `candidate_tap_fires_after_an_abstaining_gate_and_a_served_request_bills_once` | +| PB-46 | migrated hook seats | mapped | PASS | | test: `crates/busbar-llm/src/engine/tests/hook_seat_order_tests.rs::migrated_request_hook_seats_after_admit_before_candidate`<br>test: `crates/busbar-llm/src/engine/tests/hook_seat_order_tests.rs::candidate_tap_fires_after_an_abstaining_gate_and_a_served_request_bills_once` | | PB-47 | scoped draws and fallback | mapped | PASS | | oracle-cell: `route.failover\|fb\|member-down`<br>test: `crates/busbar-kernel/src/governance/tests/limits_tests.rs::pool_scoped_accrual_and_refund_mirror_the_charge`<br>test: `crates/busbar-kernel/src/governance/tests/limits_tests.rs::budget_block_carries_downgrade_target`<br>test: `test_budget_exhaustion_downgrades_pool`<br>test: `test_downgrade_cycle_terminates_via_the_revisit_guard` | | PB-48 | stall sweep on `http`/`sse` | mapped | PASS | | test: `crates/busbar-kernel/tests/recovery_and_ticks.rs::a_slow_unit_is_not_a_lost_one`<br>test: `crates/busbar-kernel/tests/recovery_and_ticks.rs::a_lost_task_is_settled_within_one_tick`<br>test: `crates/busbar-llm/src/engine/engine_tests/stream_deadline_tests.rs::a_proxied_stream_is_cut_at_the_configured_second_and_by_nothing_earlier`<br>test: `crates/busbar-llm/src/engine/engine_tests/stream_deadline_tests.rs::the_total_deadline_the_stream_rides_defaults_to_five_minutes` | | PB-49 | config overlay subsystem | mapped | PASS | | test: `a_read_only_config_dir_boots_without_an_overlay_instead_of_refusing`<br>test: `f_every_persist_entry_point_refuses_a_none_locked_overlay`<br>test: `d_overlay_precedence_config_over_env_over_default`<br>test: `write_is_0600`<br>test: `merge_into_applies_tombstones`<br>test: `safe_mode_requested_matches_the_exact_flag_only` | @@ -110,7 +110,7 @@ The three words are not interchangeable: | PB-81 | plugin call deadlines | mapped | PASS | | test: `the_inflight_cap_saturates_and_fails_on_the_caller_deadline_through_the_axis`<br>test: `dlopen_slow_gate_hits_the_deadline`<br>test: `a_slow_hook_is_cut_off_at_its_budget_through_the_axis`<br>test: `a_panicking_hook_is_broken_through_the_axis_never_a_verdict`<br>lint: `xtask/src/gates/blocking_ffi.rs` | | PB-82 | usage extraction | mapped | PASS | | test: `read_response_subtracts_cached_prefix_from_prompt_tokens`<br>test: `read_response_subtracts_cached_prefix_from_input_tokens`<br>test: `test_cached_content_token_count_reads_into_cache_read`<br>test: `test_gemini_usage_counts_thinking_tokens_as_output`<br>test: `adds_include_usage_when_absent`<br>test: `test_streaming_openai_egress_without_client_opt_in_still_gets_include_usage_injected`<br>test: `strip_same_proto_usage_fires_without_object_field` | | PB-83 | breaker scope | mapped | PASS | | oracle-cell: `route.failover\|fb\|member-401`<br>test: `test_pool_breaker_isolation`<br>test: `test_record_hard_down_all_cells_trips_default_and_every_pool`<br>test: `test_budget_is_lane_global_across_pools`<br>test: `test_unbounded_lane_skips_the_semaphore_bounded_still_enforces` | -| PB-84 | response-stage taps | mapped | PASS | | test: `completion_tap_reports_ok_outcome`<br>test: `completion_tap_fires_synthetic_rejected_by_gate`<br>test: `completion_tap_fires_synthetic_rejected_by_auth`<br>oracle-cell: `hooks\|hooked-pool\|unauth@hooks-webrequest` | +| PB-84 | response-stage taps | mapped | PASS | | test: `crates/busbar-llm/src/engine/tests/hook_seam_tests.rs::completion_tap_reports_ok_outcome`<br>test: `crates/busbar-llm/src/engine/tests/hook_seam_tests.rs::completion_tap_fires_synthetic_rejected_by_gate`<br>test: `crates/busbar-llm/src/engine/tests/hook_seam_tests.rs::completion_tap_fires_synthetic_rejected_by_auth`<br>oracle-cell: `hooks\|hooked-pool\|unauth@hooks-webrequest` | | PB-85 | `max_tokens` injection | mapped | PASS | | test: `per_model_then_global_then_4096`<br>test: `test_requires_max_tokens_per_protocol`<br>test: `test_openai_explicit_max_tokens_preserved_over_lane_default`<br>test: `test_openai_omits_max_tokens_injects_fallback_for_anthropic` | | PB-86 | usage locators are plane-normalized | mapped | PASS | | lint: `xtask/src/gates/kernel_token_wire_purity.rs`<br>gate: `xtask/src/gates/construction.rs` | | PB-87 | non-chat billing classes | mapped | PASS | | test: `rerank_resp_billing_is_flat`<br>test: `rerank_resp_billing_counts_search_units_as_an_open_class`<br>test: `a_reranks_search_units_reach_the_price` | @@ -121,7 +121,7 @@ The three words are not interchangeable: | PB-92 | `VirtualKey.expires_at` | mapped | PASS | | test: `a_key_row_whose_expires_at_is_in_the_past_still_verifies` | | PB-93 | 1.6.0-only store ops on an ABI-2 store | mapped | PASS | | oracle-cell: `plugins.store-persist\|store-mysql`<br>oracle-cell: `plugins.store-persist\|store-postgres`<br>oracle-cell: `plugins.store-persist\|store-sqlite`<br>oracle-cell: `plugins.store-persist\|store-valkey`<br>test: `every_added_operation_on_a_published_schema_store_is_silent_and_never_errors` | | PB-94 | upstream credential mode | mapped | PASS | | test: `test_passthrough_forwards_caller_token`<br>test: `crates/busbar/src/root/tests/declared_credentials.rs::a_declared_static_header_follows_the_credential_verbatim`<br>test: `override_present_runs_full_lookup`<br>test: `golden_migrate_auth_upstream_credentials_moves_to_pools` | -| PB-95 | migrated tap stages | mapped | PASS | | test: `attempt_tap_carries_attempt_story`<br>test: `route_tap_reports_surviving_candidates` | +| PB-95 | migrated tap stages | mapped | PASS | | test: `crates/busbar-llm/src/engine/tests/hook_seam_tests.rs::attempt_tap_carries_attempt_story`<br>test: `crates/busbar-llm/src/engine/tests/hook_seam_tests.rs::route_tap_reports_surviving_candidates` | | PB-96 | streaming byte layout (the terminal-usage settle guarantee) | mapped | PASS | | test: `test_translate_anthropic_egress_to_openai_ingress`<br>test: `test_translate_openai_egress_to_anthropic_ingress`<br>test: `bedrock_stream_framing_emits_one_metadata_delta_then_guards_duplicate`<br>test: `test_translate_openai_include_usage_egress_to_bedrock_ingress_single_metadata`<br>test: `test_duplicate_terminal_message_delta_after_stop_is_dropped`<br>test: `test_tool_id_remap_is_a_stable_reversible_bijection`<br>test: `cohere_tool_ids_pass_through_verbatim_no_decode`<br>test: `strip_same_proto_usage_fires_without_object_field` | | PB-97 | pristine request bytes | mapped | PASS | | test: `a_same_dialect_hop_is_the_callers_bytes_with_governed_splices`<br>test: `pristine_same_proto_is_byte_identical_body_model`<br>test: `pristine_same_proto_is_byte_identical_url_model`<br>test: `claude_on_vertex_drops_model_and_injects_anthropic_version`<br>test: `invalidator_3_model_rewrite_forces_non_pristine`<br>test: `invalidator_4_same_proto_model_shim_strip_forces_non_pristine` | | PB-98 | `error_map` and lane state carry-over | mapped | PASS | | test: `test_normalize_raw_error_with_provider_override`<br>test: `restored_halfopen_state_normalizes_to_open`<br>test: `restore_does_not_clobber_new_limit_with_unlimited_sentinel`<br>test: `test_hard_down_follows_identity_across_rebuild`<br>oracle-cell: `route.cheapest\|card-swap`<br>test: `from_ranked_never_produces_reject`<br>test: `from_ranked_empty_is_abstain`<br>test: `dlopen_notify_is_fire_and_forget` | diff --git a/qa/design-bindings.json b/qa/design-bindings.json index 9a0361607e..81620591dd 100644 --- a/qa/design-bindings.json +++ b/qa/design-bindings.json @@ -63,21 +63,21 @@ "checks": [ { "kind": "test", - "ref": "enforce_restricts_reapplies_compliance_tags_across_pools", + "ref": "crates/busbar-llm/src/engine/tests/hook_seam_tests.rs::enforce_restricts_reapplies_compliance_tags_across_pools", "proves": "a Reject restrict with no eligible lane fails closed; the Weighted arm passes candidates unchanged", "source": "curated", "status": "mapped" }, { "kind": "test", - "ref": "multi_restrict_disjoint_intersection_fails_closed", + "ref": "crates/busbar-llm/src/engine/tests/hook_seam_tests.rs::multi_restrict_disjoint_intersection_fails_closed", "proves": "two restricts intersecting to empty produce a 503 (status only, not the literal body)", "source": "curated", "status": "mapped" } ], "verdict": "PASS", - "verdict_detail": "test:enforce_restricts_reapplies_compliance_tags_across_pools, test:multi_restrict_disjoint_intersection_fails_closed" + "verdict_detail": "test:crates/busbar-llm/src/engine/tests/hook_seam_tests.rs::enforce_restricts_reapplies_compliance_tags_across_pools, test:crates/busbar-llm/src/engine/tests/hook_seam_tests.rs::multi_restrict_disjoint_intersection_fails_closed" }, { "id": "PB-2", @@ -363,28 +363,28 @@ "checks": [ { "kind": "test", - "ref": "global_gate_reject_short_circuits_the_request", + "ref": "crates/busbar-llm/src/engine/tests/hook_seam_tests.rs::global_gate_reject_short_circuits_the_request", "proves": "a decision-gate reject ends the request", "source": "curated", "status": "mapped" }, { "kind": "test", - "ref": "global_request_stage_tap_fires_on_a_real_dispatched_request", + "ref": "crates/busbar-llm/src/engine/tests/hook_seam_tests.rs::global_request_stage_tap_fires_on_a_real_dispatched_request", "proves": "request-stage taps fire through fire_global_taps", "source": "curated", "status": "mapped" }, { "kind": "test", - "ref": "migrated_request_hook_seats_after_admit_before_candidate", + "ref": "crates/busbar-llm/src/engine/tests/hook_seat_order_tests.rs::migrated_request_hook_seats_after_admit_before_candidate", "proves": "a decision-gate reject consumes the requests slot and refunds the billable request; rewrite, request taps and gates all run after the admission charge", "source": "hand", "status": "mapped" } ], "verdict": "PASS", - "verdict_detail": "test:global_gate_reject_short_circuits_the_request, test:global_request_stage_tap_fires_on_a_real_dispatched_request, test:migrated_request_hook_seats_after_admit_before_candidate" + "verdict_detail": "test:crates/busbar-llm/src/engine/tests/hook_seam_tests.rs::global_gate_reject_short_circuits_the_request, test:crates/busbar-llm/src/engine/tests/hook_seam_tests.rs::global_request_stage_tap_fires_on_a_real_dispatched_request, test:crates/busbar-llm/src/engine/tests/hook_seat_order_tests.rs::migrated_request_hook_seats_after_admit_before_candidate" }, { "id": "PB-7", @@ -1351,14 +1351,14 @@ "checks": [ { "kind": "test", - "ref": "enforce_restricts_reapplies_compliance_tags_across_pools", + "ref": "crates/busbar-llm/src/engine/tests/hook_seam_tests.rs::enforce_restricts_reapplies_compliance_tags_across_pools", "proves": "for a gate, Weighted leaves the candidate set unchanged", "source": "curated", "status": "mapped" } ], "verdict": "PASS", - "verdict_detail": "test:enforce_restricts_reapplies_compliance_tags_across_pools" + "verdict_detail": "test:crates/busbar-llm/src/engine/tests/hook_seam_tests.rs::enforce_restricts_reapplies_compliance_tags_across_pools" }, { "id": "PB-29", @@ -2276,21 +2276,21 @@ "checks": [ { "kind": "test", - "ref": "migrated_request_hook_seats_after_admit_before_candidate", + "ref": "crates/busbar-llm/src/engine/tests/hook_seat_order_tests.rs::migrated_request_hook_seats_after_admit_before_candidate", "proves": "a migrated request-stage chain seats after the admission door and ahead of the candidate tap: admission → rewrite → request taps → gates → candidate tap", "source": "hand", "status": "mapped" }, { "kind": "test", - "ref": "candidate_tap_fires_after_an_abstaining_gate_and_a_served_request_bills_once", + "ref": "crates/busbar-llm/src/engine/tests/hook_seat_order_tests.rs::candidate_tap_fires_after_an_abstaining_gate_and_a_served_request_bills_once", "proves": "the candidate tap fires only after the gates decided, on a served request", "source": "hand", "status": "mapped" } ], "verdict": "PASS", - "verdict_detail": "test:migrated_request_hook_seats_after_admit_before_candidate, test:candidate_tap_fires_after_an_abstaining_gate_and_a_served_request_bills_once" + "verdict_detail": "test:crates/busbar-llm/src/engine/tests/hook_seat_order_tests.rs::migrated_request_hook_seats_after_admit_before_candidate, test:crates/busbar-llm/src/engine/tests/hook_seat_order_tests.rs::candidate_tap_fires_after_an_abstaining_gate_and_a_served_request_bills_once" }, { "id": "PB-47", @@ -3976,21 +3976,21 @@ "checks": [ { "kind": "test", - "ref": "completion_tap_reports_ok_outcome", + "ref": "crates/busbar-llm/src/engine/tests/hook_seam_tests.rs::completion_tap_reports_ok_outcome", "proves": "the response stage reports ok", "source": "curated", "status": "mapped" }, { "kind": "test", - "ref": "completion_tap_fires_synthetic_rejected_by_gate", + "ref": "crates/busbar-llm/src/engine/tests/hook_seam_tests.rs::completion_tap_fires_synthetic_rejected_by_gate", "proves": "the synthetic outcome rejected_by_gate", "source": "curated", "status": "mapped" }, { "kind": "test", - "ref": "completion_tap_fires_synthetic_rejected_by_auth", + "ref": "crates/busbar-llm/src/engine/tests/hook_seam_tests.rs::completion_tap_fires_synthetic_rejected_by_auth", "proves": "OWNER DECISION: an auth refusal on a hooked pool fires the completion tap with outcome rejected_by_auth and the protocol-native status (binding amended)", "source": "hand", "status": "mapped" @@ -4004,7 +4004,7 @@ } ], "verdict": "PASS", - "verdict_detail": "test:completion_tap_reports_ok_outcome, test:completion_tap_fires_synthetic_rejected_by_gate, test:completion_tap_fires_synthetic_rejected_by_auth, oracle-cell:hooks|hooked-pool|unauth@hooks-webrequest" + "verdict_detail": "test:crates/busbar-llm/src/engine/tests/hook_seam_tests.rs::completion_tap_reports_ok_outcome, test:crates/busbar-llm/src/engine/tests/hook_seam_tests.rs::completion_tap_fires_synthetic_rejected_by_gate, test:crates/busbar-llm/src/engine/tests/hook_seam_tests.rs::completion_tap_fires_synthetic_rejected_by_auth, oracle-cell:hooks|hooked-pool|unauth@hooks-webrequest" }, { "id": "PB-85", @@ -4344,21 +4344,21 @@ "checks": [ { "kind": "test", - "ref": "attempt_tap_carries_attempt_story", + "ref": "crates/busbar-llm/src/engine/tests/hook_seam_tests.rs::attempt_tap_carries_attempt_story", "proves": "the routing-stage payload carries a 1-based attempt_number", "source": "curated", "status": "mapped" }, { "kind": "test", - "ref": "route_tap_reports_surviving_candidates", + "ref": "crates/busbar-llm/src/engine/tests/hook_seam_tests.rs::route_tap_reports_surviving_candidates", "proves": "remaining_candidates on the candidate stage", "source": "curated", "status": "mapped" } ], "verdict": "PASS", - "verdict_detail": "test:attempt_tap_carries_attempt_story, test:route_tap_reports_surviving_candidates" + "verdict_detail": "test:crates/busbar-llm/src/engine/tests/hook_seam_tests.rs::attempt_tap_carries_attempt_story, test:crates/busbar-llm/src/engine/tests/hook_seam_tests.rs::route_tap_reports_surviving_candidates" }, { "id": "PB-96", diff --git a/testing/shadow-oracle/accepted-differences.json b/testing/shadow-oracle/accepted-differences.json index 3e77ad8d45..413c090ce6 100644 --- a/testing/shadow-oracle/accepted-differences.json +++ b/testing/shadow-oracle/accepted-differences.json @@ -2412,6 +2412,53 @@ "by": "2026-12-31" } }, + { + "id": "PLANELOG the llm plane's warnings are written to its own plugin log file", + "kind": "improvement", + "changelog": "1.6.0 Changed: a plugin's log lines, the LLM plane's included (it is a plugin in 1.6.0), are written to that plugin's own log file under `plugins.logs.dir` (for example `logs/plugins/llm.log`), not to the Busbar log.", + "by": "owner (Matthew Jackson), signed 2026-10-04 (PLANELOG, OWNER SIGNED on the p3-flip-llm park); 2026-09-28 PLUGIN LOGGING amendment (\"all plugins should log to their plugin's log file\"), #85", + "cells": "^egress\\.image-detail\\|high\\|(anthropic|bedrock|gemini)$", + "expected_cells": 3, + "classes": [ + "body" + ], + "premise": { + "all": [ + { + "cell": "egress.image-detail|high|anthropic", + "body_contains": "\"status\":200" + }, + { + "not": { + "cell": "egress.image-detail|high|anthropic", + "body_contains": "dropping image_url.detail" + } + }, + { + "not": { + "cell": "egress.image-detail|high|bedrock", + "body_contains": "dropping image_url.detail" + } + }, + { + "not": { + "cell": "egress.image-detail|high|gemini", + "body_contains": "dropping image_url.detail" + } + } + ] + }, + "signoff": { + "who": "owner (Matthew Jackson)", + "when": "2026-10-04", + "ruling": "OWNER SIGNED 2026-10-04 (PLANELOG): the llm plane's warnings move to its own plugin log file per the 2026-09-28 PLUGIN LOGGING amendment, #85 'log changes from 1.5.5' (BUSBAR-1.6.0.md)" + }, + "expires": { + "baseline": "1.5.5", + "by": "2026-12-31" + }, + "rationale": "#85 (OWNER 2026-09-28 PLUGIN LOGGING: one log file per plugin instance, not the host log) moves the llm plane's log lines, as every plugin's, to its own plugin log file. MEASURED 2026-10-03 (lane-dg-llm, door build): the three cells read busbar.log after a request whose image_url.detail the target dialect cannot carry. 1.5.5 wrote the plane's warning `dropping image_url.detail: ... detail=\"high\"` into busbar.log; with the llm plane served through its door the host writes it to the plane's own plugin log file (recorded: work/capture/logs/plugins/llm.log holds `WARN llm plane busbar_plane_llm::codec::chat_handle: dropping image_url.detail: no cross-protocol carrier exists for this cost/latency hint (not even on a same-protocol OpenAI round-trip) detail=\"high\"`), so busbar.log lacks that one line. The upstream request, the status and every money class are unchanged; only the read_file step's body differs." + }, { "id": "Q-STORE migrate names the store", "kind": "breaking", diff --git a/xtask/src/gates/design_bindings/tables.rs b/xtask/src/gates/design_bindings/tables.rs index 9ed84d1cbe..7bec326f94 100644 --- a/xtask/src/gates/design_bindings/tables.rs +++ b/xtask/src/gates/design_bindings/tables.rs @@ -19,8 +19,8 @@ pub static SEED: &[(&str, &[SeededCheck])] = &[ ("gate", "xtask/src/gates/inventory_coverage.rs", "every qa/evidence/inventory/*.md row id either has a citing shadow-oracle cell that PASSes on the golden ledger or is a NAMED gap in qa/inventory-gaps.json; a row with neither turns the check red, and so does a gap entry naming a row that no longer exists or has since been covered"), ]), ("PB-1", &[ - ("test", "enforce_restricts_reapplies_compliance_tags_across_pools", "a Reject restrict with no eligible lane fails closed; the Weighted arm passes candidates unchanged"), - ("test", "multi_restrict_disjoint_intersection_fails_closed", "two restricts intersecting to empty produce a 503 (status only, not the literal body)"), + ("test", "crates/busbar-llm/src/engine/tests/hook_seam_tests.rs::enforce_restricts_reapplies_compliance_tags_across_pools", "a Reject restrict with no eligible lane fails closed; the Weighted arm passes candidates unchanged"), + ("test", "crates/busbar-llm/src/engine/tests/hook_seam_tests.rs::multi_restrict_disjoint_intersection_fails_closed", "two restricts intersecting to empty produce a 503 (status only, not the literal body)"), ]), ("PB-2", &[ ("test", "excluded_reasons_records_at_capacity", "a saturated lane's try_admit yields no pick and records AtCapacity"), @@ -60,8 +60,8 @@ pub static SEED: &[(&str, &[SeededCheck])] = &[ ("test", "ordered_walk_empty_order_is_swrr", "no ordering gate means the SWRR floor"), ]), ("PB-6", &[ - ("test", "global_gate_reject_short_circuits_the_request", "a decision-gate reject ends the request"), - ("test", "global_request_stage_tap_fires_on_a_real_dispatched_request", "request-stage taps fire through fire_global_taps"), + ("test", "crates/busbar-llm/src/engine/tests/hook_seam_tests.rs::global_gate_reject_short_circuits_the_request", "a decision-gate reject ends the request"), + ("test", "crates/busbar-llm/src/engine/tests/hook_seam_tests.rs::global_request_stage_tap_fires_on_a_real_dispatched_request", "request-stage taps fire through fire_global_taps"), ]), ("PB-8", &[ ("test", "test_saturated_lane_respects_deadline_no_infinite_spin", "the pick_among guard is bounded by failover.timeout_secs and resolves 503"), @@ -177,7 +177,7 @@ pub static SEED: &[(&str, &[SeededCheck])] = &[ ("test", "test_finish_refunds_flat_fee_on_non_2xx_keeps_on_2xx", "the flat fee is kept after 2xx headers"), ]), ("PB-28", &[ - ("test", "enforce_restricts_reapplies_compliance_tags_across_pools", "for a gate, Weighted leaves the candidate set unchanged"), + ("test", "crates/busbar-llm/src/engine/tests/hook_seam_tests.rs::enforce_restricts_reapplies_compliance_tags_across_pools", "for a gate, Weighted leaves the candidate set unchanged"), ]), ("PB-30", &[ ("test", "resolver_table", "the detect ladder rungs: headers, SigV4 prefix, path suffixes, catch-all"), @@ -482,8 +482,8 @@ pub static SEED: &[(&str, &[SeededCheck])] = &[ ("test", "test_unbounded_lane_skips_the_semaphore_bounded_still_enforces", "max_concurrent is lane-global"), ]), ("PB-84", &[ - ("test", "completion_tap_reports_ok_outcome", "the response stage reports ok"), - ("test", "completion_tap_fires_synthetic_rejected_by_gate", "the synthetic outcome rejected_by_gate"), + ("test", "crates/busbar-llm/src/engine/tests/hook_seam_tests.rs::completion_tap_reports_ok_outcome", "the response stage reports ok"), + ("test", "crates/busbar-llm/src/engine/tests/hook_seam_tests.rs::completion_tap_fires_synthetic_rejected_by_gate", "the synthetic outcome rejected_by_gate"), ]), ("PB-85", &[ ("test", "per_model_then_global_then_4096", "default_max_tokens is injected only when the IR carries none; a caller value survives"), @@ -518,8 +518,8 @@ pub static SEED: &[(&str, &[SeededCheck])] = &[ ("test", "golden_migrate_auth_upstream_credentials_moves_to_pools", "the 1.5.5 key lands under pools"), ]), ("PB-95", &[ - ("test", "attempt_tap_carries_attempt_story", "the routing-stage payload carries a 1-based attempt_number"), - ("test", "route_tap_reports_surviving_candidates", "remaining_candidates on the candidate stage"), + ("test", "crates/busbar-llm/src/engine/tests/hook_seam_tests.rs::attempt_tap_carries_attempt_story", "the routing-stage payload carries a 1-based attempt_number"), + ("test", "crates/busbar-llm/src/engine/tests/hook_seam_tests.rs::route_tap_reports_surviving_candidates", "remaining_candidates on the candidate stage"), ]), ("PB-96", &[ ("test", "test_translate_anthropic_egress_to_openai_ingress", "[DONE] for an openai ingress"), diff --git a/xtask/src/gates/population.rs b/xtask/src/gates/population.rs index 1b7b25a92d..bf230c3806 100644 --- a/xtask/src/gates/population.rs +++ b/xtask/src/gates/population.rs @@ -43,12 +43,14 @@ use crate::ctx::{Ctx, Overlay, SourceFile, WalkSpec}; /// dispatch/inline.rs, dispatch/io_slots.rs) and remove three (`busbar-core-connector` stream.rs, /// `busbar-transport-stdio` carrier.rs and conn.rs, whose work moved to the host's I/O). Lowered /// from 948 to 947 when #703 (the MCP-FIX stack) deleted `busbar-plane-mcp/src/tool_plane.rs` and -/// `outputschema.rs` and added `door_sessions.rs`, a reviewed net removal of one file. A drop -/// below 947 is refused until a reviewed diff re-measures. The selftest plant ([`one_file_short`]) -/// cuts the live population to `FLOOR - 1` whatever its size, so predev growing a file no longer -/// forces a re-pin for the plant's sake. Lowering the floor is how a gate stops reading the -/// repository without saying so, so a diff that lowers it is the diff to refuse. -pub const FLOOR: usize = 947; +/// `outputschema.rs` and added `door_sessions.rs`, a reviewed net removal of one file. Raised from +/// 947 to 949 when FLIP-LLM (#464) added `busbar/src/root/model_egress.rs` and +/// `busbar-plane-llm/src/exchange/project.rs` (it had raised 941 to 943 over predev d7929b661d). A +/// drop below 949 is refused until a reviewed diff re-measures. The selftest plant +/// ([`one_file_short`]) cuts the live population to `FLOOR - 1` whatever its size, so predev +/// growing a file no longer forces a re-pin for the plant's sake. Lowering the floor is how a gate +/// stops reading the repository without saying so, so a diff that lowers it is the diff to refuse. +pub const FLOOR: usize = 949; /// The `.rs` under `crates/` that are not test scaffolding, plus the accounting to refuse a /// population that cannot support a verdict. diff --git a/xtask/src/gates/tracing.rs b/xtask/src/gates/tracing.rs index acda234fc6..8c5d896cef 100644 --- a/xtask/src/gates/tracing.rs +++ b/xtask/src/gates/tracing.rs @@ -44,19 +44,21 @@ pub const ROW_CLOSES: &str = "tracing:attribute-closes"; const SCAN_ROOT: &str = "crates"; const EXCLUDE_TESTS_DIR: &str = "/tests/"; -/// The denominator floor, pinned AT the measured count: the crates walk reads 977 files after #703 -/// (the MCP-FIX stack) deleted busbar-plane-mcp's tool_plane.rs and outputschema.rs and added -/// door_sessions.rs, merged with predev 1c586876f8 (978 there); 977 with transport stage B (#502) -/// merged with predev 13a9ecbfb2, which adds the host's I/O table and the carriers (nine files -/// outside `/tests/`) and removes three (`busbar-core-connector` stream.rs, +/// The denominator floor, pinned AT the measured count: the crates walk reads 979 files once +/// FLIP-LLM (#464) adds `busbar/src/root/model_egress.rs` and +/// `busbar-plane-llm/src/exchange/project.rs` (973 over predev d7929b661d's 971) to predev's 977; +/// 977 after #703 (the MCP-FIX stack) deleted busbar-plane-mcp's tool_plane.rs and outputschema.rs +/// and added door_sessions.rs, merged with predev 1c586876f8 (978 there); 977 with transport stage +/// B (#502) merged with predev 13a9ecbfb2, which adds the host's I/O table and the carriers (nine +/// files outside `/tests/`) and removes three (`busbar-core-connector` stream.rs, /// `busbar-transport-stdio` carrier.rs and conn.rs) — 971 on predev d7929b661d, after /// busbar-kernel-ledger's usage/series.rs left (972 on predev 3bc48823df; 971 on predev /// 39374ec00e; 980 on 5e672d125d, before the egress_auth / auth_cache / ingress_sigv4 files left -/// the kernel crates; it was 130 against a tree that had grown past 700). A drop below 977 is +/// the kernel crates; it was 130 against a tree that had grown past 700). A drop below 979 is /// refused until a reviewed diff re-measures; the selftest plant cuts the walk to one file under /// the floor (see [`floor_plant`]), so the tree growing past it needs no re-pin. It is a `const` /// here and has no environment override: the only way to lower one is a reviewable source edit. -const SCAN_FLOOR: usize = 977; +const SCAN_FLOOR: usize = 979; /// THE SUBJECT FLOOR (item 228). The file floor above proves the walk opened the crates; it says /// nothing about whether the thing this gate judges is still there. "Every `#[instrument]` has a