From edda5522c42db6c60bb6de270f8adaaad411624f Mon Sep 17 00:00:00 2001 From: Steven Enamakel Date: Mon, 31 Aug 2026 15:26:41 +0300 Subject: [PATCH 1/8] feat(message): add cache breakpoint content block Add a provider-neutral marker for explicitly identifying stable prompt prefixes for caching. Providers that do not support explicit breakpoints can ignore it without changing their wire format. Auto-committed-on: macbook Co-authored-by: Medulla --- src/harness/message/types.rs | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/src/harness/message/types.rs b/src/harness/message/types.rs index a0f63a12..7c2a0885 100644 --- a/src/harness/message/types.rs +++ b/src/harness/message/types.rs @@ -50,6 +50,26 @@ pub enum ContentBlock { }, /// An opaque provider-specific block preserved verbatim. ProviderExtension(Value), + /// A zero-width marker: every block before this point is a stable prefix + /// the provider may cache. + /// + /// Providers differ in what they need to be told. Backends with automatic + /// longest-prefix caching (OpenAI, DeepSeek, most vLLM deployments) infer + /// the prefix from the bytes and need nothing; **Anthropic caches nothing + /// at all unless the request marks a breakpoint explicitly**. A host that + /// freezes its system prompt for the life of a session is therefore paying + /// full price on Anthropic and full price only there, which is exactly the + /// kind of bug that never shows up as an error. + /// + /// This block is the provider-neutral way to say it. It carries no content: + /// [`ContentBlock::as_text`] returns `None`, so a provider that has no use + /// for the marker drops it as it already drops thinking blocks, and the + /// wire bytes for that provider are unchanged. + /// + /// Place it *after* the content it makes cacheable. A message of + /// `[Text(stable), CacheBreakpoint, Text(volatile)]` asks the provider to + /// cache through the end of `stable`. + CacheBreakpoint, } /// A reference to an image, either by URL or inline base64 data. From 21bfda61158571800ea8ee2f5907a05ae704b7c4 Mon Sep 17 00:00:00 2001 From: Steven Enamakel Date: Mon, 31 Aug 2026 15:26:58 +0300 Subject: [PATCH 2/8] feat(openai): support prompt cache breakpoints Add optional cache control metadata to text content parts, enabling declared prompt-cache breakpoints while preserving the existing serialization for requests without them. Model the currently supported ephemeral cache lifetime as a typed wire enum. Auto-committed-on: macbook Co-authored-by: Medulla --- src/harness/providers/openai/types.rs | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/src/harness/providers/openai/types.rs b/src/harness/providers/openai/types.rs index da0db37b..3a49c454 100644 --- a/src/harness/providers/openai/types.rs +++ b/src/harness/providers/openai/types.rs @@ -245,6 +245,13 @@ pub enum ContentPartWire { Text { /// The text content. text: String, + /// Marks this part as the end of a cacheable prefix. + /// + /// Absent for every part that is not a declared breakpoint, so a + /// request with no [`crate::harness::message::ContentBlock::CacheBreakpoint`] + /// serializes exactly as it did before this field existed. + #[serde(skip_serializing_if = "Option::is_none")] + cache_control: Option, }, /// An image reference, by URL or data URI. ImageUrl { @@ -253,6 +260,25 @@ pub enum ContentPartWire { }, } +/// A prompt-cache breakpoint on a content part. +/// +/// Anthropic's wire format (and the OpenAI-compatible surfaces that proxy to +/// it, notably OpenRouter) reads `cache_control` on a content block and caches +/// everything from the start of the request through that block. Providers that +/// do not implement it ignore an unknown key, and providers that cache +/// automatically never receive one because the caller does not declare +/// breakpoints for them. +/// +/// Only `ephemeral` exists today. Modelled as an enum rather than a bare string +/// so a future lifetime is a compile error at every construction site instead of +/// a typo that silently disables caching. +#[derive(Clone, Copy, Debug, Serialize)] +#[serde(tag = "type", rename_all = "snake_case")] +pub enum CacheControlWire { + /// The provider's default (short) cache lifetime. + Ephemeral, +} + /// The `image_url` payload of a [`ContentPartWire::ImageUrl`]. #[derive(Clone, Debug, Serialize)] pub struct ImageUrlWire { From f169f37bbc2e1b412ad20351bddaba41fd6bd50a Mon Sep 17 00:00:00 2001 From: Steven Enamakel Date: Mon, 31 Aug 2026 15:27:28 +0300 Subject: [PATCH 3/8] fix(openai): preserve cache breakpoints in message content Route system content through block translation and represent cache breakpoints on OpenAI content parts. This preserves cache directives instead of silently dropping them while validating unsupported breakpoint placement. Auto-committed-on: macbook Co-authored-by: Medulla --- src/harness/providers/openai/convert.rs | 54 ++++++++++++++++++++----- 1 file changed, 45 insertions(+), 9 deletions(-) diff --git a/src/harness/providers/openai/convert.rs b/src/harness/providers/openai/convert.rs index bcb5a7f0..92cf5f3b 100644 --- a/src/harness/providers/openai/convert.rs +++ b/src/harness/providers/openai/convert.rs @@ -97,9 +97,14 @@ fn normalize_tool_call_id(id: &str) -> String { /// [`TinyAgentsError::Validation`] instead of being discarded. pub(super) fn translate_message(message: &Message) -> Result { let wire = match message { - Message::System(_) => ChatMessageWire { + // Routed through the same content translation as user messages rather + // than `message.text()`. `text()` concatenates only the text blocks, so + // it silently discards a `CacheBreakpoint` — which is precisely the + // message that carries one, since the system prompt is the stable + // prefix worth caching. + Message::System(system) => ChatMessageWire { role: "system".to_string(), - content: Some(MessageContentWire::Text(message.text())), + content: Some(translate_user_content(&system.content)?), tool_calls: Vec::new(), tool_call_id: None, }, @@ -158,11 +163,17 @@ pub(super) fn translate_message(message: &Message) -> Result { /// representation, so it fails closed with a validation error rather than being /// silently dropped. pub(super) fn translate_user_content(blocks: &[ContentBlock]) -> Result { - let has_image = blocks - .iter() - .any(|block| matches!(block, ContentBlock::Image(_))); - - if !has_image { + // Two things force the content-parts shape: an image, which has no string + // representation, and a declared cache breakpoint, which is an attribute of + // a *part* and therefore cannot be expressed on a bare string. + let needs_parts = blocks.iter().any(|block| { + matches!( + block, + ContentBlock::Image(_) | ContentBlock::CacheBreakpoint + ) + }); + + if !needs_parts { // No image: render as a single string, but still fail closed on blocks // that cannot be represented. let mut text = String::new(); @@ -170,7 +181,11 @@ pub(super) fn translate_user_content(blocks: &[ContentBlock]) -> Result text.push_str(t), ContentBlock::Json(value) => text.push_str(&value.to_string()), - ContentBlock::Image(_) => unreachable!("guarded by has_image"), + ContentBlock::Image(_) => unreachable!("guarded by needs_parts"), + // Unreachable for the same reason, and deliberately not folded + // into the drop arm below: a breakpoint that silently vanished + // would leave the caller believing the request was cached. + ContentBlock::CacheBreakpoint => unreachable!("guarded by needs_parts"), // OpenAI-compatible requests have no representation for // reasoning blocks; they are dropped rather than failing the // request (matching the assistant path, which serializes via @@ -187,9 +202,13 @@ pub(super) fn translate_user_content(blocks: &[ContentBlock]) -> Result parts.push(ContentPartWire::Text { text: t.clone() }), + ContentBlock::Text(t) => parts.push(ContentPartWire::Text { + text: t.clone(), + cache_control: None, + }), ContentBlock::Json(value) => parts.push(ContentPartWire::Text { text: value.to_string(), + cache_control: None, }), ContentBlock::Image(image) => parts.push(ContentPartWire::ImageUrl { image_url: ImageUrlWire { @@ -199,11 +218,28 @@ pub(super) fn translate_user_content(blocks: &[ContentBlock]) -> Result {} + // The marker attaches to the part it follows: the provider caches + // through the end of that block. A leading breakpoint (nothing to + // mark) is a caller mistake with no safe interpretation, so it is + // dropped rather than guessed at — caching the empty prefix and + // caching the whole message are both wrong. + ContentBlock::CacheBreakpoint => match parts.last_mut() { + Some(ContentPartWire::Text { cache_control, .. }) => { + *cache_control = Some(CacheControlWire::Ephemeral); + } + Some(ContentPartWire::ImageUrl { .. }) | None => { + tracing::warn!( + "[openai] ignoring a CacheBreakpoint with no preceding text part; \ + place it after the content it should make cacheable" + ); + } + }, ContentBlock::ProviderExtension(_) => { return Err(unrepresentable_block_error()); } } } + enforce_breakpoint_limit(&mut parts); Ok(MessageContentWire::Parts(parts)) } From 7daaad2b30aac603d1a12b1e62a61dea48088586 Mon Sep 17 00:00:00 2001 From: Steven Enamakel Date: Mon, 31 Aug 2026 15:27:44 +0300 Subject: [PATCH 4/8] fix(openai): limit cache breakpoints to provider maximum MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Trim excess cache breakpoints to the provider’s maximum of four while retaining the longest cached prefixes. Emit a warning when prompt assembly declares too many breakpoints instead of allowing the request to fail. Auto-committed-on: macbook Co-authored-by: Medulla --- src/harness/providers/openai/convert.rs | 42 +++++++++++++++++++++++++ 1 file changed, 42 insertions(+) diff --git a/src/harness/providers/openai/convert.rs b/src/harness/providers/openai/convert.rs index 92cf5f3b..b1457010 100644 --- a/src/harness/providers/openai/convert.rs +++ b/src/harness/providers/openai/convert.rs @@ -243,6 +243,48 @@ pub(super) fn translate_user_content(blocks: &[ContentBlock]) -> Result = parts + .iter() + .enumerate() + .filter(|(_, part)| { + matches!( + part, + ContentPartWire::Text { + cache_control: Some(_), + .. + } + ) + }) + .map(|(index, _)| index) + .collect(); + if marked.len() <= MAX_BREAKPOINTS { + return; + } + let drop_count = marked.len() - MAX_BREAKPOINTS; + tracing::warn!( + declared = marked.len(), + kept = MAX_BREAKPOINTS, + "[openai] more cache breakpoints than the provider accepts; keeping the \ + last {MAX_BREAKPOINTS} (longest prefixes) and dropping the earliest {drop_count}" + ); + for &index in &marked[..drop_count] { + if let Some(ContentPartWire::Text { cache_control, .. }) = parts.get_mut(index) { + *cache_control = None; + } + } +} + /// Error returned when a content block cannot be represented in an OpenAI /// request. Failing closed keeps the block from being silently dropped. pub(super) fn unrepresentable_block_error() -> TinyAgentsError { From adc94dfc67f0fb7c055189ec0b731e712b81dea4 Mon Sep 17 00:00:00 2001 From: Steven Enamakel Date: Mon, 31 Aug 2026 15:29:55 +0300 Subject: [PATCH 5/8] fix(harness): ignore cache breakpoints in message processing Treat cache breakpoints as zero-length, omit them from summaries, and prevent them from satisfying user-query checks so they do not trigger premature compaction or render empty turns. Auto-committed-on: macbook Co-authored-by: Medulla --- src/harness/message/mod.rs | 4 ++++ src/harness/summarization/render.rs | 2 ++ src/harness/tool/prompt.rs | 4 ++++ 3 files changed, 10 insertions(+) diff --git a/src/harness/message/mod.rs b/src/harness/message/mod.rs index e68f2411..95293e69 100644 --- a/src/harness/message/mod.rs +++ b/src/harness/message/mod.rs @@ -88,6 +88,10 @@ impl ContentBlock { ContentBlock::Thinking { text, .. } => text.chars().count(), ContentBlock::RedactedThinking { data } => data.chars().count(), ContentBlock::ProviderExtension(value) => value.to_string().chars().count(), + // A zero-width marker weighs nothing. It must not contribute here: + // this figure gates compaction, and inflating it would trigger + // summarization earlier for a block that carries no information. + ContentBlock::CacheBreakpoint => 0, } } } diff --git a/src/harness/summarization/render.rs b/src/harness/summarization/render.rs index c052875c..19d5f5c3 100644 --- a/src/harness/summarization/render.rs +++ b/src/harness/summarization/render.rs @@ -76,6 +76,8 @@ fn render_content(content: &[ContentBlock]) -> Vec { "", image.mime_type.as_deref().unwrap_or("unknown") )), + // Carries no information a summary could keep. + ContentBlock::CacheBreakpoint => None, ContentBlock::Thinking { text, .. } if text.trim().is_empty() => None, ContentBlock::Thinking { text, .. } => { Some(format!("{}", elide(text))) diff --git a/src/harness/tool/prompt.rs b/src/harness/tool/prompt.rs index 05a8183e..0b1fe437 100644 --- a/src/harness/tool/prompt.rs +++ b/src/harness/tool/prompt.rs @@ -175,6 +175,10 @@ fn is_resolvable_user_query(message: &Message) -> bool { ContentBlock::Thinking { .. } | ContentBlock::RedactedThinking { .. } | ContentBlock::ProviderExtension(_) => false, + // Nor is a cache marker. Counting it as user input would let a message + // that is only a breakpoint satisfy the "has a user turn" check and + // hand a chat template an empty turn to render. + ContentBlock::CacheBreakpoint => false, }) } From 96053b75e26c3d2c8c467f9422a571873f8c9b86 Mon Sep 17 00:00:00 2001 From: Steven Enamakel Date: Mon, 31 Aug 2026 15:30:17 +0300 Subject: [PATCH 6/8] test(openai): cover prompt-cache breakpoint serialization Add coverage for preserving bare string content without breakpoints, emitting cache controls at valid breakpoints, enforcing provider limits, and keeping markers out of message text. Auto-committed-on: macbook Co-authored-by: Medulla --- src/harness/providers/openai/test.rs | 114 +++++++++++++++++++++++++++ 1 file changed, 114 insertions(+) diff --git a/src/harness/providers/openai/test.rs b/src/harness/providers/openai/test.rs index a2b21d4c..84c77def 100644 --- a/src/harness/providers/openai/test.rs +++ b/src/harness/providers/openai/test.rs @@ -2975,3 +2975,117 @@ async fn a_transport_failure_is_reported_as_a_structured_provider_error() { "the failing URL is what makes this diagnosable, got: {rendered}" ); } + +// --------------------------------------------------------------------------- +// Prompt-cache breakpoints (ContentBlock::CacheBreakpoint) +// --------------------------------------------------------------------------- +// +// The point of these is the *first* one: a request that declares no breakpoint +// must serialize exactly as it did before the feature existed. Every host on +// the OpenAI-compatible path shares this code, and most of them talk to +// backends that cache automatically and would be actively harmed by a content +// array appearing where a string used to be. + +mod cache_breakpoints { + use crate::harness::message::{ContentBlock, Message, SystemMessage}; + use crate::harness::providers::openai::convert::translate_message; + + fn wire(message: &Message) -> serde_json::Value { + serde_json::to_value(translate_message(message).expect("translates")).expect("serializes") + } + + fn system(blocks: Vec) -> Message { + Message::System(SystemMessage { content: blocks }) + } + + #[test] + fn a_system_message_without_a_breakpoint_is_still_a_bare_string() { + let got = wire(&system(vec![ContentBlock::Text("be helpful".into())])); + assert_eq!( + got["content"], + serde_json::json!("be helpful"), + "declaring no breakpoint must leave the wire bytes untouched" + ); + } + + #[test] + fn a_breakpoint_marks_the_part_it_follows() { + let got = wire(&system(vec![ + ContentBlock::Text("stable".into()), + ContentBlock::CacheBreakpoint, + ContentBlock::Text("volatile".into()), + ])); + assert_eq!( + got["content"], + serde_json::json!([ + {"type": "text", "text": "stable", "cache_control": {"type": "ephemeral"}}, + {"type": "text", "text": "volatile"} + ]) + ); + } + + #[test] + fn several_breakpoints_are_all_emitted() { + let got = wire(&system(vec![ + ContentBlock::Text("a".into()), + ContentBlock::CacheBreakpoint, + ContentBlock::Text("b".into()), + ContentBlock::CacheBreakpoint, + ContentBlock::Text("c".into()), + ])); + let parts = got["content"].as_array().expect("parts"); + assert_eq!(parts.len(), 3); + assert!(parts[0].get("cache_control").is_some()); + assert!(parts[1].get("cache_control").is_some()); + assert!(parts[2].get("cache_control").is_none()); + } + + #[test] + fn breakpoints_beyond_the_providers_limit_keep_the_longest_prefixes() { + // Anthropic 400s past four. Five are declared; the earliest is dropped + // because it covers the shortest prefix and is worth the least. + let mut blocks = Vec::new(); + for i in 0..5 { + blocks.push(ContentBlock::Text(format!("block{i}"))); + blocks.push(ContentBlock::CacheBreakpoint); + } + let got = wire(&system(blocks)); + let parts = got["content"].as_array().expect("parts"); + assert_eq!(parts.len(), 5); + assert!( + parts[0].get("cache_control").is_none(), + "the earliest breakpoint is the one to drop" + ); + assert_eq!( + parts[1..] + .iter() + .filter(|p| p.get("cache_control").is_some()) + .count(), + 4 + ); + } + + #[test] + fn a_leading_breakpoint_is_dropped_rather_than_guessed_at() { + let got = wire(&system(vec![ + ContentBlock::CacheBreakpoint, + ContentBlock::Text("body".into()), + ])); + let parts = got["content"].as_array().expect("parts"); + assert_eq!(parts.len(), 1); + assert!(parts[0].get("cache_control").is_none()); + } + + #[test] + fn a_breakpoint_does_not_change_the_messages_text() { + // `Message::text()` feeds token budgeting, summarization and every + // provider that does not implement caching. A marker must be invisible + // to all of them. + let message = system(vec![ + ContentBlock::Text("one ".into()), + ContentBlock::CacheBreakpoint, + ContentBlock::Text("two".into()), + ]); + assert_eq!(message.text(), "one two"); + } +} From f6d6496ff1a3c5dba94dc09ad38449f5390aa04f Mon Sep 17 00:00:00 2001 From: Steven Enamakel Date: Mon, 31 Aug 2026 15:30:47 +0300 Subject: [PATCH 7/8] refactor(openai): use a relative import for message translation Update the cache breakpoint tests to reference the translation helper through the local module hierarchy without changing behavior. Auto-committed-on: macbook Co-authored-by: Medulla --- src/harness/providers/openai/test.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/harness/providers/openai/test.rs b/src/harness/providers/openai/test.rs index 84c77def..1d86b506 100644 --- a/src/harness/providers/openai/test.rs +++ b/src/harness/providers/openai/test.rs @@ -2988,7 +2988,7 @@ async fn a_transport_failure_is_reported_as_a_structured_provider_error() { mod cache_breakpoints { use crate::harness::message::{ContentBlock, Message, SystemMessage}; - use crate::harness::providers::openai::convert::translate_message; + use super::super::convert::translate_message; fn wire(message: &Message) -> serde_json::Value { serde_json::to_value(translate_message(message).expect("translates")).expect("serializes") From 485703fc0e6331da35a825bf1ab40b5a2cfdd2bd Mon Sep 17 00:00:00 2001 From: Steven Enamakel Date: Mon, 31 Aug 2026 17:35:59 +0300 Subject: [PATCH 8/8] chore(vendor): bump tinytools for ToolExposure Co-authored-by: Medulla --- vendor/tinytools | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/vendor/tinytools b/vendor/tinytools index 47db6d45..095aff78 160000 --- a/vendor/tinytools +++ b/vendor/tinytools @@ -1 +1 @@ -Subproject commit 47db6d45b7691c6d4ab6f3d2b5967e631865376d +Subproject commit 095aff787ebf37e8675d9b578f8c784d46eca0ce