diff --git a/src/harness/message/mod.rs b/src/harness/message/mod.rs index e68f2411..95293e69 100644 --- a/src/harness/message/mod.rs +++ b/src/harness/message/mod.rs @@ -88,6 +88,10 @@ impl ContentBlock { ContentBlock::Thinking { text, .. } => text.chars().count(), ContentBlock::RedactedThinking { data } => data.chars().count(), ContentBlock::ProviderExtension(value) => value.to_string().chars().count(), + // A zero-width marker weighs nothing. It must not contribute here: + // this figure gates compaction, and inflating it would trigger + // summarization earlier for a block that carries no information. + ContentBlock::CacheBreakpoint => 0, } } } diff --git a/src/harness/message/types.rs b/src/harness/message/types.rs index a0f63a12..7c2a0885 100644 --- a/src/harness/message/types.rs +++ b/src/harness/message/types.rs @@ -50,6 +50,26 @@ pub enum ContentBlock { }, /// An opaque provider-specific block preserved verbatim. ProviderExtension(Value), + /// A zero-width marker: every block before this point is a stable prefix + /// the provider may cache. + /// + /// Providers differ in what they need to be told. Backends with automatic + /// longest-prefix caching (OpenAI, DeepSeek, most vLLM deployments) infer + /// the prefix from the bytes and need nothing; **Anthropic caches nothing + /// at all unless the request marks a breakpoint explicitly**. A host that + /// freezes its system prompt for the life of a session is therefore paying + /// full price on Anthropic and full price only there, which is exactly the + /// kind of bug that never shows up as an error. + /// + /// This block is the provider-neutral way to say it. It carries no content: + /// [`ContentBlock::as_text`] returns `None`, so a provider that has no use + /// for the marker drops it as it already drops thinking blocks, and the + /// wire bytes for that provider are unchanged. + /// + /// Place it *after* the content it makes cacheable. A message of + /// `[Text(stable), CacheBreakpoint, Text(volatile)]` asks the provider to + /// cache through the end of `stable`. + CacheBreakpoint, } /// A reference to an image, either by URL or inline base64 data. diff --git a/src/harness/providers/openai/convert.rs b/src/harness/providers/openai/convert.rs index bcb5a7f0..b1457010 100644 --- a/src/harness/providers/openai/convert.rs +++ b/src/harness/providers/openai/convert.rs @@ -97,9 +97,14 @@ fn normalize_tool_call_id(id: &str) -> String { /// [`TinyAgentsError::Validation`] instead of being discarded. pub(super) fn translate_message(message: &Message) -> Result { let wire = match message { - Message::System(_) => ChatMessageWire { + // Routed through the same content translation as user messages rather + // than `message.text()`. `text()` concatenates only the text blocks, so + // it silently discards a `CacheBreakpoint` — which is precisely the + // message that carries one, since the system prompt is the stable + // prefix worth caching. + Message::System(system) => ChatMessageWire { role: "system".to_string(), - content: Some(MessageContentWire::Text(message.text())), + content: Some(translate_user_content(&system.content)?), tool_calls: Vec::new(), tool_call_id: None, }, @@ -158,11 +163,17 @@ pub(super) fn translate_message(message: &Message) -> Result { /// representation, so it fails closed with a validation error rather than being /// silently dropped. pub(super) fn translate_user_content(blocks: &[ContentBlock]) -> Result { - let has_image = blocks - .iter() - .any(|block| matches!(block, ContentBlock::Image(_))); - - if !has_image { + // Two things force the content-parts shape: an image, which has no string + // representation, and a declared cache breakpoint, which is an attribute of + // a *part* and therefore cannot be expressed on a bare string. + let needs_parts = blocks.iter().any(|block| { + matches!( + block, + ContentBlock::Image(_) | ContentBlock::CacheBreakpoint + ) + }); + + if !needs_parts { // No image: render as a single string, but still fail closed on blocks // that cannot be represented. let mut text = String::new(); @@ -170,7 +181,11 @@ pub(super) fn translate_user_content(blocks: &[ContentBlock]) -> Result text.push_str(t), ContentBlock::Json(value) => text.push_str(&value.to_string()), - ContentBlock::Image(_) => unreachable!("guarded by has_image"), + ContentBlock::Image(_) => unreachable!("guarded by needs_parts"), + // Unreachable for the same reason, and deliberately not folded + // into the drop arm below: a breakpoint that silently vanished + // would leave the caller believing the request was cached. + ContentBlock::CacheBreakpoint => unreachable!("guarded by needs_parts"), // OpenAI-compatible requests have no representation for // reasoning blocks; they are dropped rather than failing the // request (matching the assistant path, which serializes via @@ -187,9 +202,13 @@ pub(super) fn translate_user_content(blocks: &[ContentBlock]) -> Result parts.push(ContentPartWire::Text { text: t.clone() }), + ContentBlock::Text(t) => parts.push(ContentPartWire::Text { + text: t.clone(), + cache_control: None, + }), ContentBlock::Json(value) => parts.push(ContentPartWire::Text { text: value.to_string(), + cache_control: None, }), ContentBlock::Image(image) => parts.push(ContentPartWire::ImageUrl { image_url: ImageUrlWire { @@ -199,14 +218,73 @@ pub(super) fn translate_user_content(blocks: &[ContentBlock]) -> Result {} + // The marker attaches to the part it follows: the provider caches + // through the end of that block. A leading breakpoint (nothing to + // mark) is a caller mistake with no safe interpretation, so it is + // dropped rather than guessed at — caching the empty prefix and + // caching the whole message are both wrong. + ContentBlock::CacheBreakpoint => match parts.last_mut() { + Some(ContentPartWire::Text { cache_control, .. }) => { + *cache_control = Some(CacheControlWire::Ephemeral); + } + Some(ContentPartWire::ImageUrl { .. }) | None => { + tracing::warn!( + "[openai] ignoring a CacheBreakpoint with no preceding text part; \ + place it after the content it should make cacheable" + ); + } + }, ContentBlock::ProviderExtension(_) => { return Err(unrepresentable_block_error()); } } } + enforce_breakpoint_limit(&mut parts); Ok(MessageContentWire::Parts(parts)) } +/// Anthropic accepts at most four `cache_control` breakpoints per request and +/// rejects the whole request with a 400 beyond that. +/// +/// Keep the **last** four. Each breakpoint caches from the start of the request +/// through its own block, so the later ones cover strictly longer prefixes and +/// are strictly more valuable; dropping from the front loses the least. A +/// caller declaring more than four has a prompt-assembly bug, so this warns +/// rather than trimming silently — but it trims, because a 400 on every turn is +/// a worse failure than a smaller cache. +fn enforce_breakpoint_limit(parts: &mut [ContentPartWire]) { + const MAX_BREAKPOINTS: usize = 4; + let marked: Vec = parts + .iter() + .enumerate() + .filter(|(_, part)| { + matches!( + part, + ContentPartWire::Text { + cache_control: Some(_), + .. + } + ) + }) + .map(|(index, _)| index) + .collect(); + if marked.len() <= MAX_BREAKPOINTS { + return; + } + let drop_count = marked.len() - MAX_BREAKPOINTS; + tracing::warn!( + declared = marked.len(), + kept = MAX_BREAKPOINTS, + "[openai] more cache breakpoints than the provider accepts; keeping the \ + last {MAX_BREAKPOINTS} (longest prefixes) and dropping the earliest {drop_count}" + ); + for &index in &marked[..drop_count] { + if let Some(ContentPartWire::Text { cache_control, .. }) = parts.get_mut(index) { + *cache_control = None; + } + } +} + /// Error returned when a content block cannot be represented in an OpenAI /// request. Failing closed keeps the block from being silently dropped. pub(super) fn unrepresentable_block_error() -> TinyAgentsError { diff --git a/src/harness/providers/openai/test.rs b/src/harness/providers/openai/test.rs index a2b21d4c..1d86b506 100644 --- a/src/harness/providers/openai/test.rs +++ b/src/harness/providers/openai/test.rs @@ -2975,3 +2975,117 @@ async fn a_transport_failure_is_reported_as_a_structured_provider_error() { "the failing URL is what makes this diagnosable, got: {rendered}" ); } + +// --------------------------------------------------------------------------- +// Prompt-cache breakpoints (ContentBlock::CacheBreakpoint) +// --------------------------------------------------------------------------- +// +// The point of these is the *first* one: a request that declares no breakpoint +// must serialize exactly as it did before the feature existed. Every host on +// the OpenAI-compatible path shares this code, and most of them talk to +// backends that cache automatically and would be actively harmed by a content +// array appearing where a string used to be. + +mod cache_breakpoints { + use crate::harness::message::{ContentBlock, Message, SystemMessage}; + use super::super::convert::translate_message; + + fn wire(message: &Message) -> serde_json::Value { + serde_json::to_value(translate_message(message).expect("translates")).expect("serializes") + } + + fn system(blocks: Vec) -> Message { + Message::System(SystemMessage { content: blocks }) + } + + #[test] + fn a_system_message_without_a_breakpoint_is_still_a_bare_string() { + let got = wire(&system(vec![ContentBlock::Text("be helpful".into())])); + assert_eq!( + got["content"], + serde_json::json!("be helpful"), + "declaring no breakpoint must leave the wire bytes untouched" + ); + } + + #[test] + fn a_breakpoint_marks_the_part_it_follows() { + let got = wire(&system(vec![ + ContentBlock::Text("stable".into()), + ContentBlock::CacheBreakpoint, + ContentBlock::Text("volatile".into()), + ])); + assert_eq!( + got["content"], + serde_json::json!([ + {"type": "text", "text": "stable", "cache_control": {"type": "ephemeral"}}, + {"type": "text", "text": "volatile"} + ]) + ); + } + + #[test] + fn several_breakpoints_are_all_emitted() { + let got = wire(&system(vec![ + ContentBlock::Text("a".into()), + ContentBlock::CacheBreakpoint, + ContentBlock::Text("b".into()), + ContentBlock::CacheBreakpoint, + ContentBlock::Text("c".into()), + ])); + let parts = got["content"].as_array().expect("parts"); + assert_eq!(parts.len(), 3); + assert!(parts[0].get("cache_control").is_some()); + assert!(parts[1].get("cache_control").is_some()); + assert!(parts[2].get("cache_control").is_none()); + } + + #[test] + fn breakpoints_beyond_the_providers_limit_keep_the_longest_prefixes() { + // Anthropic 400s past four. Five are declared; the earliest is dropped + // because it covers the shortest prefix and is worth the least. + let mut blocks = Vec::new(); + for i in 0..5 { + blocks.push(ContentBlock::Text(format!("block{i}"))); + blocks.push(ContentBlock::CacheBreakpoint); + } + let got = wire(&system(blocks)); + let parts = got["content"].as_array().expect("parts"); + assert_eq!(parts.len(), 5); + assert!( + parts[0].get("cache_control").is_none(), + "the earliest breakpoint is the one to drop" + ); + assert_eq!( + parts[1..] + .iter() + .filter(|p| p.get("cache_control").is_some()) + .count(), + 4 + ); + } + + #[test] + fn a_leading_breakpoint_is_dropped_rather_than_guessed_at() { + let got = wire(&system(vec![ + ContentBlock::CacheBreakpoint, + ContentBlock::Text("body".into()), + ])); + let parts = got["content"].as_array().expect("parts"); + assert_eq!(parts.len(), 1); + assert!(parts[0].get("cache_control").is_none()); + } + + #[test] + fn a_breakpoint_does_not_change_the_messages_text() { + // `Message::text()` feeds token budgeting, summarization and every + // provider that does not implement caching. A marker must be invisible + // to all of them. + let message = system(vec![ + ContentBlock::Text("one ".into()), + ContentBlock::CacheBreakpoint, + ContentBlock::Text("two".into()), + ]); + assert_eq!(message.text(), "one two"); + } +} diff --git a/src/harness/providers/openai/types.rs b/src/harness/providers/openai/types.rs index da0db37b..3a49c454 100644 --- a/src/harness/providers/openai/types.rs +++ b/src/harness/providers/openai/types.rs @@ -245,6 +245,13 @@ pub enum ContentPartWire { Text { /// The text content. text: String, + /// Marks this part as the end of a cacheable prefix. + /// + /// Absent for every part that is not a declared breakpoint, so a + /// request with no [`crate::harness::message::ContentBlock::CacheBreakpoint`] + /// serializes exactly as it did before this field existed. + #[serde(skip_serializing_if = "Option::is_none")] + cache_control: Option, }, /// An image reference, by URL or data URI. ImageUrl { @@ -253,6 +260,25 @@ pub enum ContentPartWire { }, } +/// A prompt-cache breakpoint on a content part. +/// +/// Anthropic's wire format (and the OpenAI-compatible surfaces that proxy to +/// it, notably OpenRouter) reads `cache_control` on a content block and caches +/// everything from the start of the request through that block. Providers that +/// do not implement it ignore an unknown key, and providers that cache +/// automatically never receive one because the caller does not declare +/// breakpoints for them. +/// +/// Only `ephemeral` exists today. Modelled as an enum rather than a bare string +/// so a future lifetime is a compile error at every construction site instead of +/// a typo that silently disables caching. +#[derive(Clone, Copy, Debug, Serialize)] +#[serde(tag = "type", rename_all = "snake_case")] +pub enum CacheControlWire { + /// The provider's default (short) cache lifetime. + Ephemeral, +} + /// The `image_url` payload of a [`ContentPartWire::ImageUrl`]. #[derive(Clone, Debug, Serialize)] pub struct ImageUrlWire { diff --git a/src/harness/summarization/render.rs b/src/harness/summarization/render.rs index c052875c..19d5f5c3 100644 --- a/src/harness/summarization/render.rs +++ b/src/harness/summarization/render.rs @@ -76,6 +76,8 @@ fn render_content(content: &[ContentBlock]) -> Vec { "", image.mime_type.as_deref().unwrap_or("unknown") )), + // Carries no information a summary could keep. + ContentBlock::CacheBreakpoint => None, ContentBlock::Thinking { text, .. } if text.trim().is_empty() => None, ContentBlock::Thinking { text, .. } => { Some(format!("{}", elide(text))) diff --git a/src/harness/tool/prompt.rs b/src/harness/tool/prompt.rs index 05a8183e..0b1fe437 100644 --- a/src/harness/tool/prompt.rs +++ b/src/harness/tool/prompt.rs @@ -175,6 +175,10 @@ fn is_resolvable_user_query(message: &Message) -> bool { ContentBlock::Thinking { .. } | ContentBlock::RedactedThinking { .. } | ContentBlock::ProviderExtension(_) => false, + // Nor is a cache marker. Counting it as user input would let a message + // that is only a breakpoint satisfy the "has a user turn" check and + // hand a chat template an empty turn to render. + ContentBlock::CacheBreakpoint => false, }) } diff --git a/vendor/tinytools b/vendor/tinytools index 47db6d45..095aff78 160000 --- a/vendor/tinytools +++ b/vendor/tinytools @@ -1 +1 @@ -Subproject commit 47db6d45b7691c6d4ab6f3d2b5967e631865376d +Subproject commit 095aff787ebf37e8675d9b578f8c784d46eca0ce