From af0023f3d685f6a532aa65bf61cf1791b3511fb0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=EC=B5=9C=EC=9E=AC=EC=98=81?= <144660874+Jaeyeong-CHOI@users.noreply.github.com> Date: Wed, 7 Oct 2026 00:27:45 +0900 Subject: [PATCH 1/2] fix: preserve text in typed-content chat templates Normalize eligible string messages at the checkpoint template boundary. Preserve raw conversation fields and explicit template input contracts, with conservative probes and focused regression fixtures. Fixes #2135 --- src/server/chat_request.rs | 4 + src/server/chat_request_content_tests.rs | 249 ++++++++ src/server/chat_template.rs | 95 +-- src/server/chat_template_content.rs | 325 +++++++++++ .../chat_template_content_cache_tests.rs | 136 +++++ src/server/chat_template_content_tests.rs | 334 +++++++++++ src/server/chat_template_policy_tests.rs | 543 ++++++++++++++++++ src/server/chat_template_raw_content_tests.rs | 241 ++++++++ tests/fixtures/granite_4_vision/SOURCE.md | 10 + .../granite_4_vision/chat_template.jinja | 180 ++++++ tests/fixtures/granite_vision_3_2/SOURCE.md | 13 + .../granite_vision_3_2/chat_template.jinja | 40 ++ .../granite_vision_3_2/tokenizer_config.json | 3 + 13 files changed, 2130 insertions(+), 43 deletions(-) create mode 100644 src/server/chat_request_content_tests.rs create mode 100644 src/server/chat_template_content.rs create mode 100644 src/server/chat_template_content_cache_tests.rs create mode 100644 src/server/chat_template_content_tests.rs create mode 100644 src/server/chat_template_policy_tests.rs create mode 100644 src/server/chat_template_raw_content_tests.rs create mode 100644 tests/fixtures/granite_4_vision/SOURCE.md create mode 100644 tests/fixtures/granite_4_vision/chat_template.jinja create mode 100644 tests/fixtures/granite_vision_3_2/SOURCE.md create mode 100644 tests/fixtures/granite_vision_3_2/chat_template.jinja create mode 100644 tests/fixtures/granite_vision_3_2/tokenizer_config.json diff --git a/src/server/chat_request.rs b/src/server/chat_request.rs index f8f039c93..0921396be 100644 --- a/src/server/chat_request.rs +++ b/src/server/chat_request.rs @@ -2175,3 +2175,7 @@ mod tests; #[cfg(test)] #[path = "chat_request_reasoning_content_tests.rs"] mod reasoning_content_tests; + +#[cfg(test)] +#[path = "chat_request_content_tests.rs"] +mod content_tests; diff --git a/src/server/chat_request_content_tests.rs b/src/server/chat_request_content_tests.rs new file mode 100644 index 000000000..9ba4bd12a --- /dev/null +++ b/src/server/chat_request_content_tests.rs @@ -0,0 +1,249 @@ +// Copyright 2025-2026 Lablup Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! Model-free request routing regressions for typed-only checkpoint templates. +//! +//! Embedded data URI resolves local bytes only, as in the existing request tests. + +use std::path::PathBuf; + +use serde_json::{Value, json}; + +use super::prepare_chat_request_with_cache; +use crate::server::chat_template::ChatTemplateProcessor; +use crate::server::types::ChatCompletionRequest; +use crate::tokenizer::ThinkingMarkers; + +const JSON_OBSERVER: &str = concat!( + "{% for m in messages %}", + "{% if m.content is not string %}{{ m | tojson }}{{ '\\n' }}{% endif %}", + "{% endfor %}", + "{% if add_generation_prompt %}GENERATE{% endif %}", +); + +fn checkpoint(template: &str) -> ChatTemplateProcessor { + let dir = tempfile::tempdir().unwrap(); + std::fs::write( + dir.path().join("tokenizer_config.json"), + json!({"chat_template": template}).to_string(), + ) + .unwrap(); + ChatTemplateProcessor::from_model_path(dir.path()) + .unwrap() + .unwrap() +} + +fn observed_messages(rendered: &str) -> Vec { + rendered + .strip_suffix("GENERATE") + .unwrap_or(rendered) + .lines() + .map(|line| serde_json::from_str(line).expect("template, not fallback, must render")) + .collect() +} + +fn request(messages: Value) -> ChatCompletionRequest { + serde_json::from_value(json!({"model": "test", "messages": messages})).unwrap() +} + +#[tokio::test] +async fn image_routing_keeps_system_assistant_and_plain_followup_text() { + let processor = ChatTemplateProcessor::from_model_path( + &PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("tests/fixtures/granite_vision_3_2"), + ) + .unwrap() + .unwrap(); + let req = request(json!([ + {"role": "system", "content": "SYSTEM"}, + {"role": "user", "content": [ + {"type": "text", "text": "IMAGE_QUESTION"}, + {"type": "image_url", "image_url": {"url": "data:image/png;base64,aGVsbG8="}}, + ]}, + {"role": "assistant", "content": "ANSWER"}, + {"role": "user", "content": "FOLLOWUP"}, + ])); + let prepared = prepare_chat_request_with_cache( + &processor, + &req, + None, + true, + true, + false, + &ThinkingMarkers::default(), + ) + .await + .unwrap(); + assert_eq!( + prepared.prompt, + concat!( + "<|system|>\nSYSTEM\n", + "<|user|>\n\nIMAGE_QUESTION\n", + "<|assistant|>\nANSWER<|end_of_text|>", + "<|user|>\nFOLLOWUP\n<|assistant|>\n", + ) + ); + assert_eq!(prepared.image_data, vec![b"hello".to_vec()]); + assert!( + prepared.history_prompt.is_none(), + "media history snapshot stays disabled" + ); +} + +#[tokio::test] +async fn reasoning_routing_preserves_strings_and_both_reasoning_spellings() { + let processor = checkpoint(JSON_OBSERVER); + let req = request(json!([ + {"role": "user", "content": "QUESTION"}, + {"role": "assistant", "content": "ANSWER", "reasoning_content": "TRACE"}, + {"role": "user", "content": "FOLLOWUP"}, + ])); + let prepared = prepare_chat_request_with_cache( + &processor, + &req, + None, + true, + true, + false, + &ThinkingMarkers::default(), + ) + .await + .unwrap(); + let history = prepared + .history_prompt + .as_ref() + .expect("raw text history boundary"); + assert_eq!(prepared.prompt, format!("{history}GENERATE")); + let rows = observed_messages(&prepared.prompt); + assert_eq!( + rows, + vec![ + json!({"role": "user", "content": [{"type": "text", "text": "QUESTION"}]}), + json!({ + "role": "assistant", "content": [{"type": "text", "text": "ANSWER"}], + "reasoning": "TRACE", "reasoning_content": "TRACE", + }), + json!({"role": "user", "content": [{"type": "text", "text": "FOLLOWUP"}]}), + ] + ); +} + +#[tokio::test] +async fn raw_normalization_does_not_reintroduce_stripped_old_thinking() { + let processor = checkpoint(JSON_OBSERVER); + let mut req = request(json!([ + {"role": "user", "content": "QUESTION"}, + { + "role": "assistant", + "content": [{"type": "text", "text": "PRIVATE_INLINEANSWER"}], + "reasoning_content": "PRIVATE_PARALLEL", + }, + {"role": "user", "content": "FOLLOWUP"}, + ])); + req.chat_template_kwargs = + Some(serde_json::from_value(json!({"preserve_thinking": false})).unwrap()); + let prepared = prepare_chat_request_with_cache( + &processor, + &req, + None, + true, + true, + false, + &ThinkingMarkers::default(), + ) + .await + .unwrap(); + let rows = observed_messages(&prepared.prompt); + assert_eq!(rows.len(), 3); + assert_eq!( + rows[1], + json!({ + "role": "assistant", + "content": [{"type": "text", "text": "ANSWER"}], + }) + ); + assert!(!prepared.prompt.contains("PRIVATE_")); + assert!(!prepared.history_prompt.unwrap().contains("PRIVATE_")); +} + +#[tokio::test] +async fn tool_routing_preserves_call_ids_names_arguments_and_empty_content() { + let processor = checkpoint(JSON_OBSERVER); + let req = request(json!([ + {"role": "user", "content": "QUESTION"}, + { + "role": "assistant", + "content": null, + "name": "planner", + "tool_calls": [{ + "id": "call-1", + "type": "function", + "function": {"name": "weather", "arguments": "{\"city\":\"Seoul\"}"}, + }], + }, + { + "role": "tool", + "content": "SUNNY", + "tool_call_id": "call-1", + "name": "weather", + }, + {"role": "user", "content": "FOLLOWUP"}, + ])); + let prepared = prepare_chat_request_with_cache( + &processor, + &req, + None, + true, + true, + false, + &ThinkingMarkers::default(), + ) + .await + .unwrap(); + let rows = observed_messages(&prepared.prompt); + assert_eq!(rows.len(), 4); + assert_eq!( + rows[0]["content"], + json!([{"type": "text", "text": "QUESTION"}]) + ); + assert_eq!( + rows[1], + json!({ + "role": "assistant", + "content": [{"type": "text", "text": ""}], + "name": "planner", + "tool_calls": [{ + "id": "call-1", + "type": "function", + "function": {"name": "weather", "arguments": {"city": "Seoul"}}, + }], + }) + ); + assert_eq!( + rows[2], + json!({ + "role": "tool", + "content": [{"type": "text", "text": "SUNNY"}], + "tool_call_id": "call-1", + "name": "weather", + }) + ); + assert_eq!( + rows[3]["content"], + json!([{"type": "text", "text": "FOLLOWUP"}]) + ); + let history = prepared + .history_prompt + .expect("tool-history boundary retained"); + assert_eq!(prepared.prompt, format!("{history}GENERATE")); +} diff --git a/src/server/chat_template.rs b/src/server/chat_template.rs index c7df1d113..491deb7cf 100644 --- a/src/server/chat_template.rs +++ b/src/server/chat_template.rs @@ -52,6 +52,9 @@ use super::chat_template_kwargs::ChatTemplateKwargs; use super::tool_calls::{self, ToolCallFormat}; use super::types::request::Tool; +#[path = "chat_template_content.rs"] +mod content; + /// A message in the conversation #[derive(Debug, Clone, Serialize, Deserialize)] pub struct ChatMessage { @@ -178,6 +181,10 @@ pub struct ChatTemplateProcessor { bos_token: String, eos_token: String, add_generation_prompt: bool, + /// Checkpoint templates may need typed text parts. Decisions use the + /// effective render context and string-content roles, with a bounded cache. + /// Custom template strings keep their caller-controlled content contract. + content_normalizer: Option, /// Cached result of `supports_tools()` introspection. /// `None` means not yet computed. supports_tools_cached: Option, @@ -318,7 +325,10 @@ impl std::fmt::Display for ChatTemplateCompileError { impl std::error::Error for ChatTemplateCompileError {} impl ChatTemplateProcessor { - /// Create a new processor by loading template from tokenizer_config.json or chat_template.jinja + /// Create a new processor by loading template from tokenizer_config.json or chat_template.jinja. + /// + /// Used by: CLI and server model loading across families, including + /// Granite Vision, Llama and Qwen. pub fn from_model_path(model_path: &Path) -> Result> { let config_path = model_path.join("tokenizer_config.json"); let config: Option = if config_path.exists() { @@ -412,6 +422,7 @@ impl ChatTemplateProcessor { bos_token, eos_token, add_generation_prompt: true, + content_normalizer: Some(content::ContentNormalizer::default()), supports_tools_cached: None, forced_tool_call_format, default_enable_thinking: false, @@ -433,6 +444,7 @@ impl ChatTemplateProcessor { bos_token: String::new(), eos_token: String::new(), add_generation_prompt: true, + content_normalizer: None, supports_tools_cached: None, forced_tool_call_format, default_enable_thinking: false, @@ -904,6 +916,11 @@ impl ChatTemplateProcessor { /// `[{"type": "image"}, {"type": "text", "text": "..."}]`) that Jinja2 /// templates like Gemma3 VLM can iterate over. /// + /// Checkpoint-loaded processors conservatively adapt eligible string + /// content to typed text parts in the effective render context. Existing + /// arrays, null content and message metadata are untouched. Processors + /// created with with_template retain the caller's original raw contract. + /// /// When `tools` is `Some`, the tool definitions are passed to the Jinja2 /// template context, enabling tool-calling prompt formatting. /// @@ -928,11 +945,12 @@ impl ChatTemplateProcessor { /// plumbing path used's `preserve_thinking` feature and /// generalizes to any future kwarg a HuggingFace chat template expects. /// - /// When a kwarg key duplicates an already-provided context entry - /// (`messages`, `bos_token`, `eos_token`, `add_generation_prompt`, - /// `tools`), the kwarg value wins — this lets operators override defaults - /// if a template ships with unusual expectations. - // Used by: chat_request, routes/chat + /// Canonical server keys (messages, bos_token, eos_token, + /// add_generation_prompt and tools) are reserved; attempts to override + /// them through kwargs are ignored. enable_thinking remains overridable. + /// Other kwargs enter the effective context used for both adaptation and + /// rendering. + // Used by: chat_request, routes/chat, checkpoint-backed prompt inspection pub fn apply_raw_with_kwargs( &self, messages: &serde_json::Value, @@ -970,9 +988,6 @@ impl ChatTemplateProcessor { kwargs: &ChatTemplateKwargs, add_generation_prompt: bool, ) -> Result { - // Convert serde_json::Value to minijinja::Value - let messages_val = minijinja::Value::from_serialize(messages); - // Leave `tools` UNDEFINED when the request carries no tools, and count // an explicit empty list as no tools (issue #1597). Three guard shapes // ship in the local template corpus and only an absent key satisfies @@ -1003,7 +1018,7 @@ impl ChatTemplateProcessor { .map(minijinja::Value::from_serialize); let context = build_template_context( - messages_val, + Value::UNDEFINED, &self.bos_token, &self.eos_token, add_generation_prompt, @@ -1014,11 +1029,16 @@ impl ChatTemplateProcessor { self.thinking_mode_sentinel().as_deref(), ); - // Render directly: minijinja reproduces the template's own - // enable_thinking branches faithfully (issue #686), so no post-render - // Gemma-4 patching is applied. The `enable_thinking` value reaches the - // template through `build_template_context` above. - let rendered = self.render_template(context)?; + // Both ordinary and raw messages use this one adaptation boundary. + // Probes call the unchecked renderer, so they cannot normalize again. + // Existing arrays, null content and message metadata remain untouched. + let messages = match &self.content_normalizer { + Some(normalizer) => { + normalizer.normalize(messages, &context, |probe| self.render_template(probe)) + } + None => std::borrow::Cow::Borrowed(messages), + }; + let rendered = self.render_template(content::with_messages(&context, messages.as_ref()))?; Ok(self.complete_generation_prompt(rendered, add_generation_prompt)) } @@ -1064,6 +1084,8 @@ impl ChatTemplateProcessor { self.apply_inner(messages, tools, kwargs, false) } + // Used by: text-only CLI and server generation/history renders across + // model families. Share adaptation, kwargs and rendering with the raw path. fn apply_inner( &self, messages: &[ChatMessage], @@ -1071,34 +1093,9 @@ impl ChatTemplateProcessor { kwargs: &ChatTemplateKwargs, add_generation_prompt: bool, ) -> Result { - // Same `tools` rule as `apply_raw_inner`: the key is absent when the - // request carries no tools, and an explicit empty list counts as no - // tools (issue #1597). See that function for why undefined rather than - // `none` or an empty list is the value that every shipped guard shape - // agrees on. - let tools_val = tools - .filter(|t| !t.is_empty()) - .map(minijinja::Value::from_serialize); - - let messages_val = minijinja::Value::from_serialize(messages); - let context = build_template_context( - messages_val, - &self.bos_token, - &self.eos_token, - add_generation_prompt, - tools_val, - kwargs, - self.default_enable_thinking, - self.wants_bare_thinking_alias(), - self.thinking_mode_sentinel().as_deref(), - ); - - // Render directly: minijinja reproduces the template's own - // enable_thinking branches faithfully (issue #686), so no post-render - // Gemma-4 patching is applied. The `enable_thinking` value reaches the - // template through `build_template_context` above. - let rendered = self.render_template(context)?; - Ok(self.complete_generation_prompt(rendered, add_generation_prompt)) + let messages = + serde_json::to_value(messages).context("Failed to serialize chat messages")?; + self.apply_raw_inner(&messages, tools, kwargs, add_generation_prompt) } } @@ -4997,3 +4994,15 @@ TOOL assert!(out.starts_with("")); } } + +#[cfg(test)] +#[path = "chat_template_content_tests.rs"] +mod content_tests; + +#[cfg(test)] +#[path = "chat_template_raw_content_tests.rs"] +mod raw_content_tests; + +#[cfg(test)] +#[path = "chat_template_policy_tests.rs"] +mod policy_tests; diff --git a/src/server/chat_template_content.rs b/src/server/chat_template_content.rs new file mode 100644 index 000000000..e2e56f7e6 --- /dev/null +++ b/src/server/chat_template_content.rs @@ -0,0 +1,325 @@ +// Copyright 2025-2026 Lablup Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! Conservative, context-sensitive adaptation for typed-content-only templates. +//! +//! Probes establish a limited compatibility heuristic, not a proof about every +//! possible conversation. A successful user-only probe never licenses changing +//! an already string-compatible system or assistant branch. Decisions are keyed +//! by the effective render context and the roles being converted, not messages. + +use std::borrow::Cow; +use std::collections::{BTreeSet, VecDeque}; +use std::io::Write; +use std::sync::{Arc, Mutex}; + +use anyhow::Result; +use minijinja::Value; +use serde_json::{Value as JsonValue, json}; + +const CACHE_ENTRIES: usize = 16; +const MAX_CACHE_KEY_BYTES: usize = 16 * 1024; +// Different lengths, initial/final characters and case catch more transformations +// than equal-length markers sharing a prefix (uppercasing, slicing, length, …). +const PROBE_A: &str = "a7!mlx_probe_Z"; +const PROBE_B: &str = "Q92_DIFFERENT_text_payload?end6"; +const USER_BEFORE: &str = "u3_CONTEXT_before!M"; +const USER_AFTER: &str = "v84_CONTEXT_after?N"; + +type DecisionCache = VecDeque<(Vec, bool)>; + +#[derive(Clone, Default)] +pub(super) struct ContentNormalizer { + // Clones share bounded decisions keyed by effective context and roles. + // Keys may retain kwargs/tools, but not message objects or rendered output. + // No template code runs while this mutex is held. + cache: Arc>, +} + +impl ContentNormalizer { + /// Normalize string content only, preserving every other message field. + /// + /// The context must be the effective context with messages undefined. + /// The render callback is unchecked, never another normalization entry. + /// Used by: checkpoint-backed CLI/server generation and history renders, + /// including their raw multimodal/tool-message counterparts. + pub(super) fn normalize<'a>( + &self, + messages: &'a JsonValue, + context: &Value, + render: impl Fn(Value) -> Result, + ) -> Cow<'a, JsonValue> { + let Some(roles) = string_roles(messages) else { + return Cow::Borrowed(messages); + }; + if roles.is_empty() { + return Cow::Borrowed(messages); + } + + // Metadata can select a different content schema even for the same + // role. Never retain it in keys or reuse a plain-message decision. + let has_metadata = has_message_metadata(messages); + let key = if has_metadata { + None + } else { + cache_key(context, &roles) + }; + let cached = key.as_ref().and_then(|key| { + self.cache.lock().ok().and_then(|cache| { + cache + .iter() + .find_map(|(entry, decision)| (entry == key).then_some(*decision)) + }) + }); + let should_wrap = cached.unwrap_or_else(|| { + let decision = infer(context, &roles, messages, has_metadata, &render); + if let Some(key) = key + && let Ok(mut cache) = self.cache.lock() + { + // Another clone may have populated this key while we rendered. + if !cache.iter().any(|(entry, _)| entry == &key) { + if cache.len() == CACHE_ENTRIES { + cache.pop_front(); + } + cache.push_back((key, decision)); + } + } + decision + }); + if !should_wrap { + return Cow::Borrowed(messages); + } + + let mut normalized = messages.clone(); + // The array and role/content shape were checked by string_roles. + if let Some(messages) = normalized.as_array_mut() { + for message in messages { + if let Some(content) = message.get_mut("content") + && let JsonValue::String(text) = content + { + *content = typed_text(std::mem::take(text)); + } + } + } + Cow::Owned(normalized) + } +} + +fn string_roles(messages: &JsonValue) -> Option> { + let mut roles = BTreeSet::new(); + for message in messages.as_array()? { + if message.get("content").is_some_and(JsonValue::is_string) { + roles.insert(message.get("role")?.as_str()?.to_owned()); + } + } + Some(roles) +} + +fn has_message_metadata(messages: &JsonValue) -> bool { + messages.as_array().is_some_and(|messages| { + messages.iter().any(|message| { + message + .as_object() + .is_some_and(|fields| fields.keys().any(|key| key != "role" && key != "content")) + }) + }) +} + +/// Replace only the messages slot without rebuilding or logging kwargs. +pub(super) fn with_messages(context: &Value, messages: &JsonValue) -> Value { + minijinja::context! { + messages => Value::from_serialize(messages), + ..context.clone() + } +} + +fn infer( + context: &Value, + roles: &BTreeSet, + messages: &JsonValue, + has_metadata: bool, + render: &impl Fn(Value) -> Result, +) -> bool { + let render_messages = |messages: &JsonValue| render(with_messages(context, messages)); + + // A transformation is not a dropped message: require two dissimilar + // string payloads to produce the same successful output. In particular, + // rejecting strings must not itself enable normalization. + let string_a = render_messages(&JsonValue::Array(vec![message("user", PROBE_A, false)])); + let string_b = render_messages(&JsonValue::Array(vec![message("user", PROBE_B, false)])); + let (Ok(string_a), Ok(string_b)) = (string_a, string_b) else { + return false; + }; + if string_a != string_b { + return false; + } + for (marker, other) in [(PROBE_A, PROBE_B), (PROBE_B, PROBE_A)] { + let Ok(typed) = render_messages(&JsonValue::Array(vec![message("user", marker, true)])) + else { + return false; + }; + if !typed.contains(marker) || typed.contains(other) { + return false; + } + } + + if has_metadata { + // Inspect every string-message prototype, including user messages: + // name, tool calls, reasoning and other values can select a schema. + // Only the content changes in these probes; actual metadata survives. + for prototype in messages.as_array().into_iter().flatten() { + if prototype.get("content").is_some_and(JsonValue::is_string) + && !prototype_can_wrap(prototype, &render_messages) + { + return false; + } + } + } else { + // Absent roles do not block an otherwise valid user-only template. + for role in roles.iter().filter(|role| role.as_str() != "user") { + if !prototype_can_wrap(&message(role, "", false), &render_messages) { + return false; + } + } + } + true +} + +fn prototype_can_wrap( + prototype: &JsonValue, + render: &impl Fn(&JsonValue) -> Result, +) -> bool { + for (marker, other) in [(PROBE_A, PROBE_B), (PROBE_B, PROBE_A)] { + let (messages, required) = prototype_scaffold(prototype, marker, true); + let Ok(typed) = render(&messages) else { + return false; + }; + if typed.contains(other) || required.iter().any(|text| !typed.contains(text.as_str())) { + return false; + } + } + + let string_a = render(&prototype_scaffold(prototype, PROBE_A, false).0); + let string_b = render(&prototype_scaffold(prototype, PROBE_B, false).0); + match (string_a, string_b) { + (Ok(a), Ok(b)) => a == b, + // Granite Vision 3.2 indexes system/assistant content[0].text, so a + // string in those roles is an engine error. Accept this only AFTER + // the successful user silent-drop gate and both typed-role renders + // preserved all markers. Explicit rejections still veto adaptation, + // as do inconsistent success/error outcomes. + (Err(a), Err(b)) => { + super::template_rejection_message(&a).is_none() + && super::template_rejection_message(&b).is_none() + } + _ => false, + } +} + +// Used by: checkpoint-backed string normalization and compatibility probes. +fn typed_text(text: String) -> JsonValue { + // Build the part from owned values: json! would serialize and clone text. + let part = serde_json::Map::from_iter([ + ("type".to_owned(), JsonValue::String("text".to_owned())), + ("text".to_owned(), JsonValue::String(text)), + ]); + JsonValue::Array(vec![JsonValue::Object(part)]) +} + +fn message(role: &str, text: &str, typed: bool) -> JsonValue { + let content = if typed { + typed_text(text.to_owned()) + } else { + json!(text) + }; + json!({"role": role, "content": content}) +} + +fn prototype_scaffold( + prototype: &JsonValue, + marker: &str, + typed: bool, +) -> (JsonValue, Vec) { + // Used by: checkpoint-backed text/raw generation and history probes. + // string_roles has already validated each converted message's role. + // Preserve metadata without cloning the potentially large, discarded body. + let target = JsonValue::Object( + prototype + .as_object() + .expect("string-content messages have validated roles") + .iter() + .map(|(key, value)| { + let value = if key == "content" { + if typed { + typed_text(marker.to_owned()) + } else { + JsonValue::String(marker.to_owned()) + } + } else { + value.clone() + }; + (key.clone(), value) + }) + .collect(), + ); + match prototype.get("role").and_then(JsonValue::as_str) { + Some("user") => (JsonValue::Array(vec![target]), vec![marker.to_owned()]), + Some("system") => ( + JsonValue::Array(vec![target, message("user", USER_AFTER, true)]), + vec![marker.to_owned(), USER_AFTER.to_owned()], + ), + _ => ( + JsonValue::Array(vec![ + message("user", USER_BEFORE, true), + target, + message("user", USER_AFTER, true), + ]), + vec![ + USER_BEFORE.to_owned(), + marker.to_owned(), + USER_AFTER.to_owned(), + ], + ), + } +} + +fn cache_key(context: &Value, roles: &BTreeSet) -> Option> { + // Bound serialized key bytes, not allocator/container overhead. Oversized + // contexts are still checked; their decisions are simply not cached. + let mut writer = KeyWriter(Vec::new()); + serde_json::to_writer(&mut writer, &(context, roles)).ok()?; + Some(writer.0) +} + +struct KeyWriter(Vec); + +impl Write for KeyWriter { + fn write(&mut self, bytes: &[u8]) -> std::io::Result { + if bytes.len() > MAX_CACHE_KEY_BYTES.saturating_sub(self.0.len()) { + return Err(std::io::Error::other( + "content-adaptation cache key too large", + )); + } + self.0.extend_from_slice(bytes); + Ok(bytes.len()) + } + + fn flush(&mut self) -> std::io::Result<()> { + Ok(()) + } +} + +#[cfg(test)] +#[path = "chat_template_content_cache_tests.rs"] +mod cache_tests; diff --git a/src/server/chat_template_content_cache_tests.rs b/src/server/chat_template_content_cache_tests.rs new file mode 100644 index 000000000..b95fb2f9e --- /dev/null +++ b/src/server/chat_template_content_cache_tests.rs @@ -0,0 +1,136 @@ +// Copyright 2025-2026 Lablup Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use std::cell::Cell; + +use super::*; + +fn typed_only(context: Value, calls: &Cell) -> Result { + calls.set(calls.get() + 1); + let mut output = String::new(); + for message in context.get_attr("messages")?.try_iter()? { + let content = message.get_attr("content")?; + if content.as_str().is_none() { + for part in content.try_iter()? { + if let Some(text) = part.get_attr("text")?.as_str() { + output.push_str(text); + } + } + } + } + Ok(output) +} + +#[test] +fn cloned_cache_reuses_decisions_without_retaining_messages_or_metadata() { + let normalizer = ContentNormalizer::default(); + let context = minijinja::context! { messages => Value::UNDEFINED }; + let calls = Cell::new(0); + let first = json!([{"role": "user", "content": "first"}]); + assert!( + normalizer.normalize(&first, &context, |ctx| typed_only(ctx, &calls))[0]["content"] + .is_array() + ); + assert_eq!(calls.get(), 4); + let second = json!([{"role": "user", "content": "different private text"}]); + assert!( + normalizer + .clone() + .normalize(&second, &context, |ctx| typed_only(ctx, &calls))[0]["content"] + .is_array() + ); + assert_eq!(calls.get(), 4, "a warm clone must run no probes"); + + let metadata = json!([{ + "role": "user", "content": "question", "reasoning_content": "private metadata" + }]); + for expected in [12, 20] { + assert!( + normalizer.normalize(&metadata, &context, |ctx| typed_only(ctx, &calls))[0]["content"] + .is_array() + ); + assert_eq!(calls.get(), expected, "metadata is checked without caching"); + } + let cache = normalizer.cache.lock().unwrap(); + assert_eq!(cache.len(), 1); + let key = String::from_utf8_lossy(&cache[0].0); + assert!(!key.contains("different private text")); + assert!(!key.contains("private metadata")); +} + +#[test] +fn poisoned_cache_is_bypassed_without_changing_normalization() { + let normalizer = ContentNormalizer::default(); + let cache = normalizer.cache.clone(); + let worker = std::thread::spawn(move || { + let _guard = cache.lock().unwrap(); + panic!("deliberately poison the decision cache"); + }); + assert!(worker.join().is_err()); + assert!(normalizer.cache.is_poisoned()); + + let messages = json!([{"role": "user", "content": "still normalize"}]); + let calls = Cell::new(0); + let normalized = normalizer.normalize( + &messages, + &minijinja::context! { messages => Value::UNDEFINED }, + |ctx| typed_only(ctx, &calls), + ); + assert_eq!(normalized[0]["content"][0]["text"], "still normalize"); + assert_eq!(calls.get(), 4); +} + +#[test] +fn cache_bounds_entries_and_key_allocations_but_still_checks_large_contexts() { + let normalizer = ContentNormalizer::default(); + let messages = json!([{"role": "user", "content": "bounded"}]); + let calls = Cell::new(0); + for mode in 0..CACHE_ENTRIES + 4 { + let context = minijinja::context! { messages => Value::UNDEFINED, mode }; + assert!( + normalizer.normalize(&messages, &context, |ctx| typed_only(ctx, &calls))[0]["content"] + .is_array() + ); + } + let cache = normalizer.cache.lock().unwrap(); + assert_eq!(cache.len(), CACHE_ENTRIES); + assert!( + cache + .iter() + .all(|(key, _)| key.len() <= MAX_CACHE_KEY_BYTES) + ); + drop(cache); + + let large = minijinja::context! { + messages => Value::UNDEFINED, + hint => "x".repeat(MAX_CACHE_KEY_BYTES + 1) + }; + let roles = string_roles(&messages).unwrap(); + assert!(cache_key(&large, &roles).is_none()); + let before = calls.get(); + for _ in 0..2 { + assert!( + normalizer.normalize(&messages, &large, |ctx| typed_only(ctx, &calls))[0]["content"] + .is_array() + ); + } + assert_eq!(calls.get(), before + 8, "oversized contexts run uncached"); + assert_eq!(normalizer.cache.lock().unwrap().len(), CACHE_ENTRIES); + let mut writer = KeyWriter(Vec::new()); + assert!(writer.write(&[0; MAX_CACHE_KEY_BYTES + 1]).is_err()); + assert!( + writer.0.is_empty(), + "oversized write must not allocate its payload" + ); +} diff --git a/src/server/chat_template_content_tests.rs b/src/server/chat_template_content_tests.rs new file mode 100644 index 000000000..86c0dbefb --- /dev/null +++ b/src/server/chat_template_content_tests.rs @@ -0,0 +1,334 @@ +// Copyright 2025-2026 Lablup Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! Checkpoint-selected templates that require typed text content (#2135). + +use std::path::PathBuf; + +use super::{ChatMessage, ChatTemplateProcessor}; +use serde_json::{Value, json}; + +const QUESTION: &str = "What is the capital of France?"; +const GRANITE_SYSTEM: &str = "<|system|>\nA chat between a curious user and an artificial intelligence assistant. The assistant gives helpful, detailed, and polite answers to the user's questions.\n"; +const GRANITE_3: &str = include_str!("../../tests/fixtures/granite_vision_3_2/chat_template.jinja"); +const GRANITE_3_CONFIG: &str = + include_str!("../../tests/fixtures/granite_vision_3_2/tokenizer_config.json"); + +fn message(role: &str, content: &str) -> ChatMessage { + ChatMessage { + role: role.into(), + content: content.into(), + } +} + +fn granite_3() -> ChatTemplateProcessor { + let path = PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("tests/fixtures/granite_vision_3_2"); + ChatTemplateProcessor::from_model_path(&path) + .unwrap() + .unwrap() +} + +fn checkpoint(template: &str) -> ChatTemplateProcessor { + let dir = tempfile::tempdir().unwrap(); + std::fs::write( + dir.path().join("tokenizer_config.json"), + json!({"chat_template": template}).to_string(), + ) + .unwrap(); + ChatTemplateProcessor::from_model_path(dir.path()) + .unwrap() + .unwrap() +} + +#[test] +fn granite_3_from_model_path_preserves_the_exact_user_prompt() { + // The standalone typed template must win over the competing string one, + // then receive typed text. This assertion fails against unmodified main. + let processor = granite_3(); + let actual = processor.apply(&[message("user", QUESTION)], None).unwrap(); + assert_eq!( + actual, + format!("{GRANITE_SYSTEM}<|user|>\n{QUESTION}\n<|assistant|>\n") + ); +} + +#[test] +fn granite_3_wraps_system_assistant_and_history_content() { + let processor = granite_3(); + let messages = [ + message("system", "Answer concisely."), + message("user", QUESTION), + message("assistant", "Paris."), + message("user", "And Germany?"), + ]; + let expected = concat!( + "<|system|>\nAnswer concisely.\n", + "<|user|>\nWhat is the capital of France?\n", + "<|assistant|>\nParis.<|end_of_text|>", + "<|user|>\nAnd Germany?\n" + ); + assert_eq!( + processor.apply(&messages, None).unwrap(), + format!("{expected}<|assistant|>\n") + ); + assert_eq!( + processor + .apply_history_with_kwargs(&messages, None, &Default::default()) + .unwrap(), + expected + ); + let mut no_generation_prompt = processor.clone(); + no_generation_prompt.set_add_generation_prompt(false); + assert_eq!( + no_generation_prompt.apply(&messages, None).unwrap(), + expected + ); +} + +#[test] +fn empty_text_remains_one_empty_text_part_in_every_role() { + let processor = granite_3(); + let messages = [ + message("system", ""), + message("user", ""), + message("assistant", ""), + ]; + assert_eq!( + processor + .apply_history_with_kwargs(&messages, None, &Default::default()) + .unwrap(), + "<|system|>\n\n<|user|>\n\n<|assistant|>\n<|end_of_text|>" + ); + + let count_parts = checkpoint( + "{% for m in messages %}{% if m.content is not string %}{{ m.content | length }}:{% for p in m.content %}{{ p.type }}={{ p.text }}{% endfor %}{% endif %}{% endfor %}", + ); + assert_eq!( + count_parts.apply(&[message("user", "")], None).unwrap(), + "1:text=" + ); +} + +#[test] +fn granite_3_raw_image_is_unchanged_and_strings_are_normalized() { + let processor = granite_3(); + let raw = json!([{ + "role": "user", + "content": [{"type": "image"}, {"type": "text", "text": QUESTION}], + }]); + assert_eq!( + processor.apply_raw(&raw, None).unwrap(), + format!("{GRANITE_SYSTEM}<|user|>\n\n{QUESTION}\n<|assistant|>\n") + ); + assert_eq!( + processor + .apply_raw_history_with_kwargs(&raw, None, &Default::default()) + .unwrap(), + format!("{GRANITE_SYSTEM}<|user|>\n\n{QUESTION}\n") + ); + // Raw strings use the same normalization as ordinary messages. + let raw_string = json!([{"role": "user", "content": QUESTION}]); + assert_eq!( + processor.apply_raw(&raw_string, None).unwrap(), + format!("{GRANITE_SYSTEM}<|user|>\n{QUESTION}\n<|assistant|>\n") + ); +} + +#[test] +fn raw_message_metadata_survives_on_a_wrapping_processor() { + let processor = checkpoint( + "{% for m in messages %}{{ m.role }}|{{ m.name | default('') }}|{{ m.tool_call_id | default('') }}|{{ m.reasoning | default('') }}|{% if m.tool_calls is defined %}{{ m.tool_calls[0].function.name }}{% endif %}|{% for p in m.content | selectattr('type', 'equalto', 'text') %}{{ p.text }}{% endfor %}{% endfor %}", + ); + let raw = json!([{ + "role": "assistant", + "content": [{"type": "text", "text": "answer"}], + "name": "speaker", + "tool_call_id": "call-1", + "reasoning": "reason", + "tool_calls": [{"function": {"name": "weather"}}], + }]); + assert_eq!( + processor.apply_raw(&raw, None).unwrap(), + "assistant|speaker|call-1|reason|weather|answer" + ); +} + +#[test] +fn granite_3_config_only_string_template_is_not_wrapped() { + let dir = tempfile::tempdir().unwrap(); + std::fs::write(dir.path().join("tokenizer_config.json"), GRANITE_3_CONFIG).unwrap(); + let processor = ChatTemplateProcessor::from_model_path(dir.path()) + .unwrap() + .unwrap(); + assert_eq!( + processor.apply(&[message("user", QUESTION)], None).unwrap(), + format!("{GRANITE_SYSTEM}<|user|>\n{QUESTION}\n<|assistant|>\n") + ); +} + +#[test] +fn granite_4_accepts_both_shapes_without_wrapping() { + let path = PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("tests/fixtures/granite_4_vision"); + let processor = ChatTemplateProcessor::from_model_path(&path) + .unwrap() + .unwrap(); + let messages = [ + message("system", "Answer concisely."), + message("user", QUESTION), + message("assistant", "Paris."), + ]; + let raw: Vec = messages + .iter() + .map(|m| json!({"role": m.role, "content": [{"type": "text", "text": m.content}]})) + .collect(); + // The checkpoint's render_content macro preserves eight indentation + // spaces for strings, but trims them for typed text parts. Both forms + // retain their original prompt contract; they need not render identically. + let expected_string = concat!( + "<|start_of_role|>system<|end_of_role|>Answer concisely.<|end_of_text|>\n", + "<|start_of_role|>user<|end_of_role|> What is the capital of France?<|end_of_text|>\n", + "<|start_of_role|>assistant<|end_of_role|> Paris.<|end_of_text|>\n", + "<|start_of_role|>assistant<|end_of_role|>" + ); + let expected_typed = concat!( + "<|start_of_role|>system<|end_of_role|>Answer concisely.<|end_of_text|>\n", + "<|start_of_role|>user<|end_of_role|>What is the capital of France?<|end_of_text|>\n", + "<|start_of_role|>assistant<|end_of_role|>Paris.<|end_of_text|>\n", + "<|start_of_role|>assistant<|end_of_role|>" + ); + let original = ChatTemplateProcessor::with_template( + std::fs::read_to_string(path.join("chat_template.jinja")).unwrap(), + ); + // Custom-template and raw rendering keep the pre-fix content contract. + assert_eq!(original.apply(&messages, None).unwrap(), expected_string); + assert_eq!( + original.apply_raw(&json!(raw), None).unwrap(), + expected_typed + ); + assert_eq!(processor.apply(&messages, None).unwrap(), expected_string); + assert_eq!( + processor.apply_raw(&json!(messages), None).unwrap(), + expected_string + ); + assert_eq!( + processor.apply_raw(&json!(raw), None).unwrap(), + expected_typed + ); +} + +#[test] +fn default_and_template_override_do_not_opt_into_wrapping() { + let default = ChatTemplateProcessor::default(); + assert!( + default + .apply(&[message("user", QUESTION)], None) + .unwrap() + .contains(QUESTION) + ); + + let explicit = ChatTemplateProcessor::with_template(GRANITE_3.into()); + assert!( + !explicit + .apply(&[message("user", QUESTION)], None) + .unwrap() + .contains(QUESTION) + ); +} + +#[test] +fn probe_uses_the_selected_processors_special_tokens_and_preprocessing() { + let dir = tempfile::tempdir().unwrap(); + let template = concat!( + "{% if bos_token != '' or eos_token != '' %}", + "{{ raise_exception('missing checkpoint tokens') }}{% endif %}", + "{% generation %}{% for p in messages[0].content | selectattr('type', 'equalto', 'text') %}", + "{{ bos_token }}{{ p.text }}{{ eos_token }}{% endfor %}{% endgeneration %}" + ); + std::fs::write( + dir.path().join("tokenizer_config.json"), + json!({ + "chat_template": template, + "bos_token": {"content": ""}, + "eos_token": "", + }) + .to_string(), + ) + .unwrap(); + let processor = ChatTemplateProcessor::from_model_path(dir.path()) + .unwrap() + .unwrap(); + assert_eq!( + processor.apply(&[message("user", QUESTION)], None).unwrap(), + format!("{QUESTION}") + ); + assert_eq!( + processor.template_compile_count(), + 1, + "probes reuse the compiled template" + ); +} + +#[test] +fn either_probe_render_error_leaves_wrapping_disabled() { + for (template, string_succeeds) in [ + ( + "{% if messages[0].content is string %}{{ raise_exception('string rejected') }}{% else %}{{ messages[0].content[0].text }}{% endif %}", + false, + ), + ( + "{% if messages[0].content is string %}{{ messages[0].content }}{% else %}{{ raise_exception('typed rejected') }}{% endif %}", + true, + ), + ( + "{% if messages[0].content is not string %}{{ raise_exception('typed rejected') }}{% endif %}", + true, + ), + ( + "{% if messages[0].content is string %}{{ unavailable_probe_function() }}{% else %}{{ messages[0].content[0].text }}{% endif %}", + false, + ), + ] { + let processor = checkpoint(template); + assert_eq!( + processor.apply(&[message("user", QUESTION)], None).is_ok(), + string_succeeds, + "{template}" + ); + } +} + +#[test] +fn parse_errors_and_missing_typed_markers_do_not_enable_wrapping() { + let malformed = checkpoint("{% for m in messages %}"); + assert!(malformed.apply(&[message("user", QUESTION)], None).is_err()); + + let drops_both = checkpoint("fixed prompt"); + assert_eq!( + drops_both + .apply(&[message("user", QUESTION)], None) + .unwrap(), + "fixed prompt" + ); +} + +#[test] +fn a_string_accepting_template_keeps_the_original_content_shape() { + let processor = checkpoint( + "{% for m in messages %}{% if m.content is string %}string:{{ m.content }}{% else %}typed:{{ m.content[0].text }}{% endif %}{% endfor %}", + ); + assert_eq!( + processor.apply(&[message("user", QUESTION)], None).unwrap(), + format!("string:{QUESTION}") + ); +} diff --git a/src/server/chat_template_policy_tests.rs b/src/server/chat_template_policy_tests.rs new file mode 100644 index 000000000..ea2d34a62 --- /dev/null +++ b/src/server/chat_template_policy_tests.rs @@ -0,0 +1,543 @@ +// Copyright 2025-2026 Lablup Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! Context-sensitive compatibility policy regressions through public APIs. + +use serde_json::json; + +use super::{ChatMessage, ChatTemplateProcessor}; +use crate::server::chat_template_kwargs::ChatTemplateKwargs; +use crate::server::types::request::Tool; + +fn checkpoint(template: &str) -> ChatTemplateProcessor { + let dir = tempfile::tempdir().unwrap(); + std::fs::write( + dir.path().join("tokenizer_config.json"), + json!({"chat_template": template}).to_string(), + ) + .unwrap(); + ChatTemplateProcessor::from_model_path(dir.path()) + .unwrap() + .unwrap() +} + +fn message(role: &str, content: &str) -> ChatMessage { + ChatMessage { + role: role.into(), + content: content.into(), + } +} + +fn kwargs(value: serde_json::Value) -> ChatTemplateKwargs { + ChatTemplateKwargs::from_json_object(value.as_object().unwrap().clone()) +} + +const THINKING_SCHEMA: &str = concat!( + "{% if enable_thinking %}STRING:{% else %}TYPED:{% endif %}", + "{% for m in messages %}", + "{% if enable_thinking %}{{ m.content }}", + "{% else %}{% for p in m.content | selectattr('type', 'equalto', 'text') %}", + "{{ p.text }}{% endfor %}{% endif %}{% endfor %}", +); + +#[test] +fn transformed_string_content_is_not_mistaken_for_a_dropped_question() { + let processor = checkpoint(concat!( + "{% for m in messages %}", + "{% if m.content is string %}{{ m.content | upper }}", + "{% else %}{% for p in m.content | selectattr('type', 'equalto', 'text') %}", + "{{ p.text }}{% endfor %}{% endif %}{% endfor %}", + )); + let messages = [message("user", "Mixed_CASE question")]; + assert_eq!( + processor.apply(&messages, None).unwrap(), + "MIXED_CASE QUESTION" + ); + assert_eq!( + processor.apply_raw(&json!(messages), None).unwrap(), + "MIXED_CASE QUESTION" + ); +} + +#[test] +fn hybrid_roles_preserve_original_strings_without_poisoning_user_only_policy() { + let processor = checkpoint(concat!( + "{% for m in messages %}{{ m.role }}:", + "{% if m.role == 'user' %}", + "{% for p in m.content | selectattr('type', 'equalto', 'text') %}{{ p.text }}{% endfor %}", + "{% else %}{{ m.content }}{% endif %}|{% endfor %}", + )); + let user = [message("user", "QUESTION")]; + let conversation = [ + message("system", "SYSTEM"), + message("user", "QUESTION"), + message("assistant", "ANSWER"), + message("user", "FOLLOWUP"), + ]; + // User-only input can be repaired. If a real string system/assistant role + // is incompatible with wrapping, preserve the whole original conversation + // instead of corrupting previously correct instructions/history. + assert_eq!(processor.apply(&user, None).unwrap(), "user:QUESTION|"); + let original = "system:SYSTEM|user:|assistant:ANSWER|user:|"; + assert_eq!(processor.apply(&conversation, None).unwrap(), original); + assert_eq!( + processor.apply_raw(&json!(conversation), None).unwrap(), + original + ); + assert_eq!(processor.apply(&user, None).unwrap(), "user:QUESTION|"); +} + +#[test] +fn changing_default_thinking_rechecks_the_effective_content_contract() { + let mut processor = checkpoint(THINKING_SCHEMA); + let messages = [message("user", "QUESTION")]; + assert_eq!(processor.apply(&messages, None).unwrap(), "TYPED:QUESTION"); + processor.set_default_enable_thinking(true); + assert_eq!(processor.apply(&messages, None).unwrap(), "STRING:QUESTION"); + processor.set_default_enable_thinking(false); + assert_eq!(processor.apply(&messages, None).unwrap(), "TYPED:QUESTION"); +} + +#[test] +fn explicit_thinking_kwargs_override_the_default_without_cache_cross_contamination() { + let mut processor = checkpoint(THINKING_SCHEMA); + let messages = [message("user", "QUESTION")]; + let enabled = kwargs(json!({"enable_thinking": true})); + let disabled = kwargs(json!({"enable_thinking": false})); + assert_eq!(processor.apply(&messages, None).unwrap(), "TYPED:QUESTION"); + assert_eq!( + processor + .apply_with_kwargs(&messages, None, &enabled) + .unwrap(), + "STRING:QUESTION" + ); + processor.set_default_enable_thinking(true); + assert_eq!( + processor + .apply_with_kwargs(&messages, None, &disabled) + .unwrap(), + "TYPED:QUESTION" + ); + assert_eq!(processor.apply(&messages, None).unwrap(), "STRING:QUESTION"); + assert_eq!( + processor + .apply_with_kwargs(&messages, None, &enabled) + .unwrap(), + "STRING:QUESTION" + ); +} + +#[test] +fn generation_and_history_use_their_own_content_contract() { + let mut processor = checkpoint(concat!( + "{% for m in messages %}", + "{% if add_generation_prompt %}", + "{% for p in m.content | selectattr('type', 'equalto', 'text') %}{{ p.text }}{% endfor %}", + "{% else %}{{ m.content }}{% endif %}{% endfor %}", + "{% if add_generation_prompt %}|GENERATE{% endif %}", + )); + let messages = [message("user", "QUESTION")]; + let raw = json!(messages); + assert_eq!( + processor.apply(&messages, None).unwrap(), + "QUESTION|GENERATE" + ); + assert_eq!( + processor + .apply_history_with_kwargs(&messages, None, &Default::default()) + .unwrap(), + "QUESTION" + ); + assert_eq!( + processor + .apply_raw_history_with_kwargs(&raw, None, &Default::default()) + .unwrap(), + "QUESTION" + ); + processor.set_add_generation_prompt(false); + assert_eq!(processor.apply(&messages, None).unwrap(), "QUESTION"); + processor.set_add_generation_prompt(true); + assert_eq!( + processor.apply_raw(&raw, None).unwrap(), + "QUESTION|GENERATE" + ); +} + +#[test] +fn preserve_thinking_kwargs_participate_in_content_policy() { + let processor = checkpoint(concat!( + "{% for m in messages %}{% if preserve_thinking | default(false) %}", + "KEPT:{{ m.content }}{% else %}STRIPPED:", + "{% for p in m.content | selectattr('type', 'equalto', 'text') %}{{ p.text }}{% endfor %}", + "{% endif %}{% endfor %}", + )); + let messages = [message("user", "QUESTION")]; + let kept = kwargs(json!({"preserve_thinking": true})); + let stripped = kwargs(json!({"preserve_thinking": false})); + assert_eq!( + processor.apply(&messages, None).unwrap(), + "STRIPPED:QUESTION" + ); + assert_eq!( + processor.apply_with_kwargs(&messages, None, &kept).unwrap(), + "KEPT:QUESTION" + ); + assert_eq!( + processor + .apply_with_kwargs(&messages, None, &stripped) + .unwrap(), + "STRIPPED:QUESTION" + ); +} + +#[test] +fn arbitrary_nested_kwargs_participate_in_content_policy() { + let processor = checkpoint(concat!( + "{% set mode = render_mode | default({'schema': 'typed'}) %}", + "{% for m in messages %}{% if mode.schema == 'string' %}", + "STRING:{{ m.content }}{% else %}TYPED:", + "{% for p in m.content | selectattr('type', 'equalto', 'text') %}{{ p.text }}{% endfor %}", + "{% endif %}{% endfor %}", + )); + let messages = [message("user", "QUESTION")]; + let string = kwargs(json!({"render_mode": {"schema": "string", "revision": 1}})); + let typed = kwargs(json!({"render_mode": {"schema": "typed", "revision": 1}})); + assert_eq!(processor.apply(&messages, None).unwrap(), "TYPED:QUESTION"); + assert_eq!( + processor + .apply_with_kwargs(&messages, None, &string) + .unwrap(), + "STRING:QUESTION" + ); + assert_eq!( + processor + .apply_with_kwargs(&messages, None, &typed) + .unwrap(), + "TYPED:QUESTION" + ); + assert_eq!( + processor + .apply_raw_with_kwargs(&json!(messages), None, &string) + .unwrap(), + "STRING:QUESTION" + ); +} + +#[test] +fn tool_definitions_not_just_tool_presence_participate_in_content_policy() { + let processor = checkpoint(concat!( + "{% for m in messages %}", + "{% if tools is defined and tools[0].function.name == 'strings' %}", + "STRING:{{ m.content }}{% else %}TYPED:", + "{% for p in m.content | selectattr('type', 'equalto', 'text') %}{{ p.text }}{% endfor %}", + "{% endif %}{% endfor %}", + )); + let make_tool = |name: &str| -> Tool { + serde_json::from_value(json!({ + "type": "function", + "function": {"name": name, "parameters": {"type": "object"}}, + })) + .unwrap() + }; + let strings = [make_tool("strings")]; + let typed = [make_tool("typed")]; + let messages = [message("user", "QUESTION")]; + assert_eq!(processor.apply(&messages, None).unwrap(), "TYPED:QUESTION"); + assert_eq!( + processor.apply(&messages, Some(&strings)).unwrap(), + "STRING:QUESTION" + ); + assert_eq!( + processor.apply(&messages, Some(&typed)).unwrap(), + "TYPED:QUESTION" + ); + assert_eq!( + processor.apply(&messages, Some(&[])).unwrap(), + "TYPED:QUESTION" + ); + assert_eq!( + processor.apply(&messages, Some(&strings)).unwrap(), + "STRING:QUESTION" + ); +} + +#[test] +fn cloned_processors_share_cache_without_sharing_effective_thinking_defaults() { + let processor = checkpoint(THINKING_SCHEMA); + let mut thinking = processor.clone(); + thinking.set_default_enable_thinking(true); + let messages = [message("user", "QUESTION")]; + for _ in 0..3 { + assert_eq!(processor.apply(&messages, None).unwrap(), "TYPED:QUESTION"); + assert_eq!(thinking.apply(&messages, None).unwrap(), "STRING:QUESTION"); + } +} + +#[test] +fn many_distinct_contexts_and_revisited_entries_keep_their_content_contract() { + let processor = checkpoint(THINKING_SCHEMA); + let messages = [message("user", "QUESTION")]; + // More than the bounded decision cache can retain: evicting a decision + // must only cause another probe, never change rendering semantics. + for nonce in 0..48 { + let enabled = nonce % 2 == 0; + let context = kwargs(json!({ + "enable_thinking": enabled, + "cache_test_nonce": nonce, + })); + let expected = if enabled { + "STRING:QUESTION" + } else { + "TYPED:QUESTION" + }; + assert_eq!( + processor + .apply_with_kwargs(&messages, None, &context) + .unwrap(), + expected + ); + } + for nonce in [0, 1, 47, 0, 46, 1] { + let enabled = nonce % 2 == 0; + let context = kwargs(json!({ + "enable_thinking": enabled, + "cache_test_nonce": nonce, + })); + let expected = if enabled { + "STRING:QUESTION" + } else { + "TYPED:QUESTION" + }; + assert_eq!( + processor + .apply_with_kwargs(&messages, None, &context) + .unwrap(), + expected + ); + } +} + +#[test] +fn oversized_context_keys_still_render_correctly_between_cached_contexts() { + let processor = checkpoint(THINKING_SCHEMA); + let messages = [message("user", "QUESTION")]; + let padding = "context-only-padding-".repeat(2048); + for (oversized, enabled) in [ + (false, false), + (true, true), + (true, false), + (false, true), + (true, true), + (false, false), + ] { + let context = kwargs(json!({ + "enable_thinking": enabled, + "cache_test_padding": if oversized { padding.as_str() } else { "" }, + })); + let expected = if enabled { + "STRING:QUESTION" + } else { + "TYPED:QUESTION" + }; + assert_eq!( + processor + .apply_with_kwargs(&messages, None, &context) + .unwrap(), + expected + ); + } +} + +#[test] +fn concurrent_clones_render_different_contexts_without_cache_cross_contamination() { + let processor = checkpoint(THINKING_SCHEMA); + let barrier = std::sync::Arc::new(std::sync::Barrier::new(6)); + std::thread::scope(|scope| { + let handles: Vec<_> = (0..6) + .map(|worker| { + let processor = processor.clone(); + let barrier = std::sync::Arc::clone(&barrier); + scope.spawn(move || { + let messages = [message("user", "QUESTION")]; + let raw = json!(messages); + barrier.wait(); + for turn in 0..32 { + let enabled = (worker + turn) % 2 == 0; + let context = kwargs(json!({ + "enable_thinking": enabled, + "cache_test_partition": worker % 3, + })); + let expected = if enabled { + "STRING:QUESTION" + } else { + "TYPED:QUESTION" + }; + let actual = if turn % 2 == 0 { + processor.apply_with_kwargs(&messages, None, &context) + } else { + processor.apply_raw_with_kwargs(&raw, None, &context) + } + .unwrap(); + assert_eq!(actual, expected, "worker={worker}, turn={turn}"); + } + }) + }) + .collect(); + for handle in handles { + handle.join().expect("concurrent renderer must not panic"); + } + }); +} + +#[test] +fn tool_call_metadata_schema_does_not_reuse_plain_role_decisions() { + let processor = checkpoint(concat!( + "{% for m in messages %}{{ m.role }}:", + "{% if m.role == 'assistant' and m.tool_calls is defined ", + "and m.tool_calls | length > 0 and m.tool_calls[0].function.name == 'strings' %}", + "{{ m.content }}{% else %}", + "{% for p in m.content | selectattr('type', 'equalto', 'text') %}{{ p.text }}{% endfor %}", + "{% endif %}|{% endfor %}{% if add_generation_prompt %}GENERATE{% endif %}", + )); + let plain = json!([ + {"role": "user", "content": "QUESTION"}, + {"role": "assistant", "content": "ANSWER"}, + {"role": "user", "content": "FOLLOWUP"}, + ]); + let adapted = "user:QUESTION|assistant:ANSWER|user:FOLLOWUP|"; + let original_mixed = "user:|assistant:ANSWER|user:|"; + let call = |name: &str| { + json!([{ + "id": "call-1", "type": "function", + "function": {"name": name, "arguments": {"city": "Seoul"}}, + }]) + }; + // Prime a plain-role decision, then alternate values of the same metadata + // field. Guarding presence alone, or caching only context+roles, is unsafe. + for calls in [ + None, + Some(json!([])), + Some(call("strings")), + Some(call("typed")), + None, + Some(call("strings")), + ] { + let mut raw = plain.clone(); + let expected = if let Some(calls) = calls { + let strings = calls + .get(0) + .and_then(|v| v.pointer("/function/name")) + .is_some_and(|name| name == "strings"); + raw[1]["tool_calls"] = calls; + if strings { original_mixed } else { adapted } + } else { + adapted + }; + assert_eq!( + processor.apply_raw(&raw, None).unwrap(), + format!("{expected}GENERATE") + ); + assert_eq!( + processor + .apply_raw_history_with_kwargs(&raw, None, &Default::default()) + .unwrap(), + expected + ); + } +} + +#[test] +fn reasoning_metadata_schema_is_checked_without_retaining_a_stale_role_policy() { + let processor = checkpoint(concat!( + "{% for m in messages %}{{ m.role }}:", + "{% if m.role == 'assistant' and m.reasoning_content | default('') == 'STRING_MODE' %}", + "{{ m.content }}{% else %}", + "{% for p in m.content | selectattr('type', 'equalto', 'text') %}{{ p.text }}{% endfor %}", + "{% endif %}|{% endfor %}", + )); + for reasoning in ["TYPED_MODE", "STRING_MODE", "TYPED_MODE", "STRING_MODE"] { + let raw = json!([ + {"role": "user", "content": "QUESTION"}, + {"role": "assistant", "content": "ANSWER", "reasoning_content": reasoning}, + {"role": "user", "content": "FOLLOWUP"}, + ]); + let expected = if reasoning == "STRING_MODE" { + "user:|assistant:ANSWER|user:|" + } else { + "user:QUESTION|assistant:ANSWER|user:FOLLOWUP|" + }; + assert_eq!(processor.apply_raw(&raw, None).unwrap(), expected); + } +} + +#[test] +fn string_user_metadata_also_participates_in_the_safety_guard() { + let processor = checkpoint(concat!( + "{% for m in messages %}", + "{% if m.name | default('') == 'string-reader' %}{{ m.content }}", + "{% else %}{% for p in m.content | selectattr('type', 'equalto', 'text') %}", + "{{ p.text }}{% endfor %}{% endif %}{% endfor %}", + )); + for name in ["typed-reader", "string-reader", "typed-reader"] { + let raw = json!([{"role": "user", "content": "QUESTION", "name": name}]); + assert_eq!(processor.apply_raw(&raw, None).unwrap(), "QUESTION"); + } +} + +#[test] +fn explicit_assistant_rejection_is_not_bypassed_by_content_adaptation() { + let processor = checkpoint(concat!( + "{% for m in messages %}", + "{% if m.role == 'assistant' and m.content is string %}", + "{{ raise_exception('assistant string policy rejected') }}", + "{% endif %}", + "{% for p in m.content | selectattr('type', 'equalto', 'text') %}", + "{{ p.text }}{% endfor %}{% endfor %}", + )); + // A successful user-only decision must not override a later role's + // explicit policy rejection, even when typed assistant content would render. + assert_eq!( + processor + .apply(&[message("user", "QUESTION")], None) + .unwrap(), + "QUESTION" + ); + let conversation = [ + message("user", "QUESTION"), + message("assistant", "ANSWER"), + message("user", "FOLLOWUP"), + ]; + let raw = json!(conversation); + let typed = json!([ + {"role": "user", "content": [{"type": "text", "text": "QUESTION"}]}, + {"role": "assistant", "content": [{"type": "text", "text": "ANSWER"}]}, + {"role": "user", "content": [{"type": "text", "text": "FOLLOWUP"}]}, + ]); + assert_eq!( + processor.apply_raw(&typed, None).unwrap(), + "QUESTIONANSWERFOLLOWUP" + ); + for result in [ + processor.apply(&conversation, None), + processor.apply_history_with_kwargs(&conversation, None, &Default::default()), + processor.apply_raw(&raw, None), + processor.apply_raw_history_with_kwargs(&raw, None, &Default::default()), + ] { + let error = result.expect_err("explicit template policy must remain an error"); + assert_eq!( + super::template_rejection_message(&error), + Some("assistant string policy rejected") + ); + } +} diff --git a/src/server/chat_template_raw_content_tests.rs b/src/server/chat_template_raw_content_tests.rs new file mode 100644 index 000000000..5a8ba2ee8 --- /dev/null +++ b/src/server/chat_template_raw_content_tests.rs @@ -0,0 +1,241 @@ +// Copyright 2025-2026 Lablup Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! Raw-message content adaptation without losing non-content fields. +//! + +use std::path::PathBuf; + +use serde_json::{Value, json}; + +use super::{ChatMessage, ChatTemplateProcessor}; + +const JSON_OBSERVER: &str = concat!( + "{% for m in messages %}", + "{% if m.content is not string %}{{ m | tojson }}{{ '\\n' }}{% endif %}", + "{% endfor %}", + "{% if add_generation_prompt %}GENERATE{% endif %}", +); + +fn checkpoint(template: &str) -> ChatTemplateProcessor { + let dir = tempfile::tempdir().unwrap(); + std::fs::write( + dir.path().join("tokenizer_config.json"), + json!({"chat_template": template}).to_string(), + ) + .unwrap(); + ChatTemplateProcessor::from_model_path(dir.path()) + .unwrap() + .unwrap() +} + +fn granite() -> ChatTemplateProcessor { + ChatTemplateProcessor::from_model_path( + &PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("tests/fixtures/granite_vision_3_2"), + ) + .unwrap() + .unwrap() +} + +fn observed_messages(rendered: &str) -> Vec { + rendered + .strip_suffix("GENERATE") + .unwrap_or(rendered) + .lines() + .map(|line| serde_json::from_str(line).expect("template emits one JSON message per line")) + .collect() +} + +#[test] +fn granite_raw_mixed_image_history_preserves_all_string_roles() { + let processor = granite(); + let raw = json!([ + {"role": "system", "content": "SYSTEM"}, + {"role": "user", "content": [ + {"type": "text", "text": "IMAGE_QUESTION"}, + {"type": "image"}, + ]}, + {"role": "assistant", "content": "ANSWER"}, + {"role": "user", "content": "FOLLOWUP"}, + ]); + let original = raw.clone(); + let history = concat!( + "<|system|>\nSYSTEM\n", + "<|user|>\n\nIMAGE_QUESTION\n", + "<|assistant|>\nANSWER<|end_of_text|>", + "<|user|>\nFOLLOWUP\n", + ); + assert_eq!( + processor.apply_raw(&raw, None).unwrap(), + format!("{history}<|assistant|>\n") + ); + assert_eq!( + processor + .apply_raw_history_with_kwargs(&raw, None, &Default::default()) + .unwrap(), + history + ); + assert_eq!( + raw, original, + "normalization must not mutate caller-owned JSON" + ); +} + +#[test] +fn raw_and_ordinary_text_share_generation_and_history_representation() { + let processor = granite(); + let messages = [ + ChatMessage { + role: "system".into(), + content: "SYSTEM".into(), + }, + ChatMessage { + role: "user".into(), + content: "QUESTION".into(), + }, + ChatMessage { + role: "assistant".into(), + content: "ANSWER".into(), + }, + ChatMessage { + role: "user".into(), + content: "FOLLOWUP".into(), + }, + ]; + let raw = serde_json::to_value(&messages).unwrap(); + assert_eq!( + processor.apply_raw(&raw, None).unwrap(), + processor.apply(&messages, None).unwrap() + ); + assert_eq!( + processor + .apply_raw_history_with_kwargs(&raw, None, &Default::default()) + .unwrap(), + processor + .apply_history_with_kwargs(&messages, None, &Default::default()) + .unwrap() + ); +} + +#[test] +fn raw_string_conversion_preserves_tool_reasoning_and_extension_metadata() { + let processor = checkpoint(JSON_OBSERVER); + let raw = json!([ + { + "role": "assistant", + "content": "ANSWER", + "name": "speaker", + "reasoning": "TRACE", + "reasoning_content": "TRACE", + "tool_calls": [{ + "id": "call-1", + "type": "function", + "function": {"name": "weather", "arguments": {"city": "Seoul"}}, + }], + "extension": {"nested": [null, false, 17, "kept"]}, + }, + { + "role": "tool", + "content": "", + "tool_call_id": "call-1", + "name": "weather", + }, + ]); + let original = raw.clone(); + let rows = observed_messages(&processor.apply_raw(&raw, None).unwrap()); + assert_eq!(rows.len(), 2); + let mut expected_assistant = raw[0].clone(); + expected_assistant["content"] = json!([{"type": "text", "text": "ANSWER"}]); + let mut expected_tool = raw[1].clone(); + expected_tool["content"] = json!([{"type": "text", "text": ""}]); + assert_eq!(rows, [expected_assistant, expected_tool]); + assert_eq!(raw, original); +} + +#[test] +fn raw_nonstring_content_is_not_flattened_coerced_or_reordered() { + let processor = checkpoint(JSON_OBSERVER); + let raw = json!([ + {"role": "assistant", "content": null, "tool_calls": []}, + {"role": "assistant", "name": "missing-content"}, + {"role": "user", "content": [ + {"type": "text", "text": "before"}, + {"type": "image", "extension": {"position": 1}}, + {"type": "text", "text": "after"}, + ]}, + {"role": "assistant", "content": []}, + {"role": "assistant", "content": false}, + {"role": "assistant", "content": 17}, + ]); + let generation = processor.apply_raw(&raw, None).unwrap(); + let history = processor + .apply_raw_history_with_kwargs(&raw, None, &Default::default()) + .unwrap(); + assert_eq!(observed_messages(&generation), *raw.as_array().unwrap()); + assert_eq!(observed_messages(&history), *raw.as_array().unwrap()); + assert_eq!(generation, format!("{history}GENERATE")); +} + +#[test] +fn raw_normalization_remains_disabled_for_string_templates_and_explicit_overrides() { + let raw = json!([{"role": "user", "content": "QUESTION", "name": "speaker"}]); + let string_compatible = checkpoint("{{ messages | tojson }}"); + assert_eq!( + serde_json::from_str::(&string_compatible.apply_raw(&raw, None).unwrap()).unwrap(), + raw + ); + + // Operator-supplied templates keep their explicit content contract. + let explicit = ChatTemplateProcessor::with_template(JSON_OBSERVER.into()); + assert_eq!(explicit.apply_raw(&raw, None).unwrap(), "GENERATE"); + assert_eq!( + explicit + .apply_raw_history_with_kwargs(&raw, None, &Default::default()) + .unwrap(), + "" + ); +} + +#[test] +fn mixed_raw_normalization_wraps_only_strings_and_preserves_other_values() { + let processor = checkpoint(JSON_OBSERVER); + let raw = json!([ + {"role": "assistant", "content": null, "tool_calls": []}, + {"role": "assistant", "name": "missing-content"}, + {"role": "user", "content": [ + {"type": "text", "text": "before"}, + {"type": "image", "extension": {"position": 1}}, + {"type": "text", "text": "after"}, + ]}, + {"role": "assistant", "content": []}, + {"role": "assistant", "content": false}, + {"role": "assistant", "content": 17}, + {"role": "user", "content": "FOLLOWUP"}, + ]); + let original = raw.clone(); + let mut expected = raw.clone(); + expected[6]["content"] = json!([{"type": "text", "text": "FOLLOWUP"}]); + + let generation = processor.apply_raw(&raw, None).unwrap(); + let history = processor + .apply_raw_history_with_kwargs(&raw, None, &Default::default()) + .unwrap(); + assert_eq!( + observed_messages(&generation), + *expected.as_array().unwrap() + ); + assert_eq!(observed_messages(&history), *expected.as_array().unwrap()); + assert_eq!(generation, format!("{history}GENERATE")); + assert_eq!(raw, original, "caller-owned JSON must remain unchanged"); +} diff --git a/tests/fixtures/granite_4_vision/SOURCE.md b/tests/fixtures/granite_4_vision/SOURCE.md new file mode 100644 index 000000000..5f750b586 --- /dev/null +++ b/tests/fixtures/granite_4_vision/SOURCE.md @@ -0,0 +1,10 @@ +# granite_4_vision chat-template fixture + +Source: [mlx-community/granite-4.0-3b-vision-4bit](https://huggingface.co/mlx-community/granite-4.0-3b-vision-4bit/tree/70fe1d89f42c71b5635cd945ed304bed438fa1e3) + +Pinned revision: 70fe1d89f42c71b5635cd945ed304bed438fa1e3 + +License: apache-2.0 (declared by the source model card). +The template is copied verbatim; no checkpoint weights are included. + +- Source [chat_template.jinja](https://huggingface.co/mlx-community/granite-4.0-3b-vision-4bit/resolve/70fe1d89f42c71b5635cd945ed304bed438fa1e3/chat_template.jinja), SHA-256: 29825b7fcc5ce44c7fd3f9977d8f2cc1b9ccef713ea2fe4046ab0d57b72c84a1. diff --git a/tests/fixtures/granite_4_vision/chat_template.jinja b/tests/fixtures/granite_4_vision/chat_template.jinja new file mode 100644 index 000000000..55d26aa26 --- /dev/null +++ b/tests/fixtures/granite_4_vision/chat_template.jinja @@ -0,0 +1,180 @@ +{#- ===== Task tag prompt constants ===== -#} +{%- set chart2code_prompt = "Generate code that recreates the chart as best as possible." -%} +{%- set chart2csv_prompt = "Please examine this chart image. Consider you are a data visualization expert, and extract the data into a CSV table.\n\nYour CSV should:\n- Include a header row with clear column names\n- Represent all data series/categories shown in the chart\n- Use numeric values that match the chart as closely as possible\n\nOutput only the CSV data, nothing else." -%} +{%- set chart2summary_prompt = "Can you describe this chart image?" -%} +{%- set tables_json_prompt = "Identify and extract the tabls schema\n Extruct the schema of all the tables in the image sorted according to the reading order.\nThe output must be a valid JSON object containing a list of dictionaries with the following structure:\n\n {\n \"dimensions\": {\n \"rows\": ,\n \"columns\": ,\n \"header_rows\": ,\n \"total_rows\": \n },\n \"cells\": [\n {\n \"row\": ,\n \"col\": ,\n \"colspan\": ,\n \"rowspan\": ,\n \"type\": \"<'header' or 'data'>\",\n \"header_level\":
,\n \"content\": \"\"\n },\n ...\n ]\n }" -%} +{%- set tables_html_prompt = "Identify and extract the tabls schema\n Extruct the schema of all the tables in the image sorted according to the reading order.\nThe output must be a list of valid HTML tables" -%} +{%- set tables_otsl_prompt = "Identify and extract the tabls schema\n Extruct the schema of all the tables in the image sorted according to the reading order.\nThe output must be a list of valid OTSL objects, each consists of the following fields: \n - a cell with content in it\n - an empty cell\n - a cell that is merged with the cell to its left\n - a cell that is merged with the cell above it\n - a cell that is merged with both the cell above it and the cell to its left\n - a new line\n - a clumn header\n - the beginning of the OTSL table\n - the end of the OTSL table\n\n An example for an output:\n [\n first table header1first table header2data1data2data with horizontal spandata with vertical spandata3,\n second table header1second table header2data1data2data with horizontal spandata with vertical spandata3\n ]" -%} + + +{#- ===== Tag expansion dispatcher ===== -#} +{%- macro expand_tags(text) -%} +{%- set has_image = "" in text -%} +{#- Determine image position: prefix if appears before the tag, suffix if after -#} +{%- if has_image -%} + {%- set img_idx = text.index("") -%} + {%- if "" in text -%}{%- set tag_idx = text.index("") -%} + {%- elif "" in text -%}{%- set tag_idx = text.index("") -%} + {%- elif "" in text -%}{%- set tag_idx = text.index("") -%} + {%- elif "" in text -%}{%- set tag_idx = text.index("") -%} + {%- elif "" in text -%}{%- set tag_idx = text.index("") -%} + {%- elif "" in text -%}{%- set tag_idx = text.index("") -%} + {%- else -%}{%- set tag_idx = 999999 -%} + {%- endif -%} + {%- set img_prefix = "\n" if img_idx < tag_idx else "" -%} + {%- set img_suffix = "\n" if img_idx >= tag_idx else "" -%} +{%- else -%} + {%- set img_prefix = "" -%} + {%- set img_suffix = "" -%} +{%- endif -%} +{%- if "" in text -%} + {{- img_prefix + chart2code_prompt + img_suffix -}} +{%- elif "" in text -%} + {{- img_prefix + chart2csv_prompt + img_suffix -}} +{%- elif "" in text -%} + {{- img_prefix + chart2summary_prompt + img_suffix -}} +{%- elif "" in text -%} + {{- img_prefix + tables_json_prompt + img_suffix -}} +{%- elif "" in text -%} + {{- img_prefix + tables_html_prompt + img_suffix -}} +{%- elif "" in text -%} + {{- img_prefix + tables_otsl_prompt + img_suffix -}} +{%- else -%} + {{- text -}} +{%- endif -%} +{%- endmacro -%} + +{#- ===== Original chat template ===== -#} +{% macro render_content(x) %} + {%- if x is string %} + {{ x }} + {%- else %} + {%- for chunk in x %} + {%- if chunk['type'] == 'text' -%} + {{ chunk['text']}} + {%- elif chunk['type'] == 'image' -%} + {{- " +" }} + {%- endif -%} + {%- endfor -%} + {%- endif -%} + {% endmacro %} + + {%- set tools_system_message_prefix = 'You are a helpful assistant with access to the following tools. You may call one or more tools to assist with the user query.\n\nYou are provided with function signatures within XML tags:\n' %} +{%- set tools_system_message_suffix = '\n\n\nFor each tool call, return a json object with function name and arguments within XML tags:\n\n{\"name\": , \"arguments\": }\n. If a tool does not exist in the provided list of tools, notify the user that you do not have the ability to fulfill the request.' %} +{%- set documents_system_message_prefix = 'You are a helpful assistant with access to the following documents. You may use one or more documents to assist with the user query.\n\nYou are given a list of documents within XML tags:\n' %} +{%- set documents_system_message_suffix = '\n\n\nWrite the response to the user\'s input by strictly aligning with the facts in the provided documents. If the information needed to answer the question is not available in the documents, inform the user that the question cannot be answered based on the available data.' %} +{%- set g4_default_system_message = 'You are a helpful assistant. Please ensure responses are professional, accurate, and safe.' %} +{%- if available_tools is defined and available_tools %} + {%- set tools = available_tools %} +{%- endif %} +{%- set ns = namespace(tools_system_message=tools_system_message_prefix, + documents_system_message=documents_system_message_prefix, + default_system_message=g4_default_system_message, + system_message='' + ) %} +{%- if tools %} + {%- for tool in tools %} + {%- set ns.tools_system_message = ns.tools_system_message + '\n' + (tool | tojson) %} + {%- endfor %} + {%- set ns.tools_system_message = ns.tools_system_message + tools_system_message_suffix %} +{%- else %} + {%- set ns.tools_system_message = '' %} +{%- endif %} +{%- if documents %} + {%- for document in documents %} + {%- set ns.documents_system_message = ns.documents_system_message + '\n' + (document | tojson) %} + {%- endfor %} + {%- set ns.documents_system_message = ns.documents_system_message + documents_system_message_suffix %} +{%- else %} + {%- set ns.documents_system_message = '' %} +{%- endif %} +{%- if messages[0].role == 'system' %} + {%- if messages[0].content is string %} + {%- set ns.system_message = messages[0].content %} + {%- elif messages[0].content is iterable %} + {%- for entry in messages[0].content %} + {%- if entry.type== 'text' %} + {%- if ns.system_message != '' %} + {%- set ns.system_message = ns.system_message + '\n' %} + {%- endif %} + {%- set ns.system_message = ns.system_message + entry.text %} + {%- endif %} + {%- endfor %} + {%- endif %} + {%- if tools and documents %} + {%- set ns.system_message = ns.system_message + '\n\n' + ns.tools_system_message + '\n\n' + ns.documents_system_message %} + {%- elif tools %} + {%- set ns.system_message = ns.system_message + '\n\n' + ns.tools_system_message %} + {%- elif documents %} + {%- set ns.system_message = ns.system_message + '\n\n' + ns.documents_system_message %} + {%- endif %} +{%- else %} + {%- if tools and documents %} + {%- set ns.system_message = ns.tools_system_message + '\n\n' + ns.documents_system_message %} + {%- elif tools %} + {%- set ns.system_message = ns.tools_system_message %} + {%- elif documents %} + {%- set ns.system_message = ns.documents_system_message %} + {%- endif %} +{%- endif %} +{%- if ns.system_message %} + {{- '<|start_of_role|>system<|end_of_role|>' + ns.system_message + '<|end_of_text|>\n' }} +{%- else %} + {{- '<|start_of_role|>system<|end_of_role|>' + ns.default_system_message + '<|end_of_text|>\n' }} +{%- endif %} +{%- for message in messages %} + {%- set content = namespace(val='') %} + {%- if render_content(message['content']) is string %} + {%- set content.val = render_content(message['content']) %} + {%- else %} + {%- if render_content(message['content']) is iterable %} + {%- for entry in render_content(message['content']) %} + {%- if entry.type== 'text' %} + {%- if content.val != '' %} + {%- set content.val = content.val + '\n' %} + {%- endif %} + {%- set content.val = content.val + entry.text %} + {%- endif %} + {%- endfor %} + {%- endif %} + {%- endif %} + {%- if (message.role == 'user') or (message.role == 'system' and not loop.first) %} + {{- '<|start_of_role|>' + message.role + '<|end_of_role|>' + expand_tags(content.val) + '<|end_of_text|>\n' }} + {%- elif message.role == 'assistant' %} + {{- '<|start_of_role|>' + message.role + '<|end_of_role|>' + content.val }} + {%- if message.tool_calls %} + {%- for tool_call in message.tool_calls %} + {%- if (loop.first and content.val) or (not loop.first) %} + {{- '\n' }} + {%- endif %} + {%- if tool_call.function %} + {%- set tool_call = tool_call.function %} + {%- endif %} + {{- '\n{"name": "' }} + {{- tool_call.name }} + {{- '", "arguments": ' }} + {%- if tool_call.arguments is string %} + {{- tool_call.arguments }} + {%- else %} + {{- tool_call.arguments | tojson }} + {%- endif %} + {{- '}\n' }} + {%- endfor %} + {%- endif %} + {{- '<|end_of_text|>\n' }} + {%- elif message.role == 'tool' %} + {%- if loop.first or (messages[loop.index0 - 1].role != 'tool') %} + {{- '<|start_of_role|>user<|end_of_role|>' }} + {%- endif %} + {{- '\n\n' }} + {{- content.val }} + {{- '\n' }} + {%- if loop.last or (messages[loop.index0 + 1].role != 'tool') %} + {{- '<|end_of_text|>\n' }} + {%- endif %} + {%- endif %} +{%- endfor %} +{%- if add_generation_prompt %} + {{- '<|start_of_role|>assistant<|end_of_role|>' }} +{%- endif %} \ No newline at end of file diff --git a/tests/fixtures/granite_vision_3_2/SOURCE.md b/tests/fixtures/granite_vision_3_2/SOURCE.md new file mode 100644 index 000000000..b5873cc94 --- /dev/null +++ b/tests/fixtures/granite_vision_3_2/SOURCE.md @@ -0,0 +1,13 @@ +# granite_vision_3_2 chat-template fixture + +Source: [mlx-community/granite-vision-3.2-2b-4bit](https://huggingface.co/mlx-community/granite-vision-3.2-2b-4bit/tree/351831ab5218b1f66bbfdfd57bd8f20d81ce8baf) + +Pinned revision: 351831ab5218b1f66bbfdfd57bd8f20d81ce8baf + +License: apache-2.0 (declared by the source model card). +The template is copied verbatim; no checkpoint weights are included. + +- Source [chat_template.jinja](https://huggingface.co/mlx-community/granite-vision-3.2-2b-4bit/resolve/351831ab5218b1f66bbfdfd57bd8f20d81ce8baf/chat_template.jinja), SHA-256: 80693db11342b45ab8393957fcfb29531ce7f61a0ba102a6fb6758fb29fea3fc. +- Source [tokenizer_config.json](https://huggingface.co/mlx-community/granite-vision-3.2-2b-4bit/resolve/351831ab5218b1f66bbfdfd57bd8f20d81ce8baf/tokenizer_config.json), SHA-256: 88a1891e9bc06ab0dc259a2c99ca3aee603c09ca214714f0f686a1cceee2b7aa. + +The checked-in tokenizer_config.json contains only the source file's chat_template value, preserving the competing string template used by the loading regression. All other metadata is omitted. diff --git a/tests/fixtures/granite_vision_3_2/chat_template.jinja b/tests/fixtures/granite_vision_3_2/chat_template.jinja new file mode 100644 index 000000000..b50f63e71 --- /dev/null +++ b/tests/fixtures/granite_vision_3_2/chat_template.jinja @@ -0,0 +1,40 @@ +{%- if tools %} + {{- '<|start_of_role|>available_tools<|end_of_role|> +' }} + {%- for tool in tools %} + {{- tool | tojson(indent=4) }} + {%- if not loop.last %} + {{- ' + +' }} + {%- endif %} + {%- endfor %} + {{- '<|end_of_text|> +' }} +{%- endif %} +{%- for message in messages if message['role'] == 'system'%}{% else %}<|system|> +A chat between a curious user and an artificial intelligence assistant. The assistant gives helpful, detailed, and polite answers to the user's questions. +{% endfor %}{%- for message in messages %} + {%- if message['role'] == 'system' %} + {{- '<|system|> +' + message['content'][0]['text'] + ' +' }} + {%- elif message['role'] == 'user' %}<|user|> + {# Render all images first #}{% for content in message['content'] | selectattr('type', 'equalto', 'image') %}{{ ' +' }}{% endfor %}{# Render all text next #}{% for content in message['content'] | selectattr('type', 'equalto', 'text') %}{{ content['text'] + ' +' }}{% endfor %} +{%- elif message['role'] == 'assistant' %} + {{- '<|assistant|> +' + message['content'][0]['text'] + '<|end_of_text|>' }} + {%- elif message['role'] == 'assistant_tool_call' %} + {{- '<|start_of_role|>assistant<|end_of_role|><|tool_call|>' + message['content'][0]['text'] + '<|end_of_text|> +' }} + {%- elif message['role'] == 'tool_response' %} + {{- '<|start_of_role|>tool_response<|end_of_role|>' + message['content'][0]['text'] + '<|end_of_text|> +' }} + {%- endif %} + {%- if loop.last and add_generation_prompt %} + {{- '<|assistant|> +' }} + {%- endif %} +{%- endfor %} \ No newline at end of file diff --git a/tests/fixtures/granite_vision_3_2/tokenizer_config.json b/tests/fixtures/granite_vision_3_2/tokenizer_config.json new file mode 100644 index 000000000..6aa6cffdf --- /dev/null +++ b/tests/fixtures/granite_vision_3_2/tokenizer_config.json @@ -0,0 +1,3 @@ +{ + "chat_template": "{%- if tools %}\n {{- '<|start_of_role|>available_tools<|end_of_role|>\n' }}\n {%- for tool in tools %}\n {{- tool | tojson(indent=4) }}\n {%- if not loop.last %}\n {{- '\n\n' }}\n {%- endif %}\n {%- endfor %}\n {{- '<|end_of_text|>\n' }}\n{%- endif %}\n{%- for message in messages if message['role'] == 'system'%}{% else %}<|system|>\nA chat between a curious user and an artificial intelligence assistant. The assistant gives helpful, detailed, and polite answers to the user's questions.\n{% endfor %}{%- for message in messages %}\n {%- if message['role'] == 'system' %}\n {{- '<|system|>\n' + message['content'] + '\n' }}\n {%- elif message['role'] == 'user' %}\n {{- '<|user|>\n' + message['content'] + '\n' }}\n {%- elif message['role'] == 'assistant' %}\n {{- '<|assistant|>\n' + message['content'] + '<|end_of_text|>' }}\n {%- elif message['role'] == 'assistant_tool_call' %}\n {{- '<|start_of_role|>assistant<|end_of_role|><|tool_call|>' + message['content'] + '<|end_of_text|>\n' }}\n {%- elif message['role'] == 'tool_response' %}\n {{- '<|start_of_role|>tool_response<|end_of_role|>' + message['content'] + '<|end_of_text|>\n' }}\n {%- endif %}\n {%- if loop.last and add_generation_prompt %}\n {{- '<|assistant|>\n' }}\n {%- endif %}\n{%- endfor %}" +} From 5daecfaa075f07931da258f5a12935910ce0e4d8 Mon Sep 17 00:00:00 2001 From: Jeongkyu Shin Date: Tue, 13 Oct 2026 01:43:19 +0900 Subject: [PATCH 2/2] fix: preserve text for typed-content templates Detect selected checkpoint templates that render typed text but drop strings, then adapt typed ChatMessage renders through a borrowed text-part view. Keep raw JSON and string-compatible template contracts unchanged, and cover Granite Vision 3.2 and Granite 4 with pinned fixtures. Validated with eight focused CUDA-profile tests, repository contract checks, and real Granite CLI and server requests on GB10. Fixes #2135 --- .../2165-typed-chat-text-20261013.en.md | 59 ++ .../2165-typed-chat-text-20261013.ko.md | 59 ++ src/server/chat_request.rs | 4 - src/server/chat_request_content_tests.rs | 249 -------- src/server/chat_template.rs | 177 ++++-- src/server/chat_template_content.rs | 325 ----------- .../chat_template_content_cache_tests.rs | 136 ----- src/server/chat_template_content_tests.rs | 334 ----------- src/server/chat_template_policy_tests.rs | 543 ------------------ src/server/chat_template_raw_content_tests.rs | 241 -------- src/server/chat_template_text_wrap_tests.rs | 196 +++++++ 11 files changed, 439 insertions(+), 1884 deletions(-) create mode 100644 TECHNICAL_REPORTS/2165-typed-chat-text-20261013.en.md create mode 100644 TECHNICAL_REPORTS/2165-typed-chat-text-20261013.ko.md delete mode 100644 src/server/chat_request_content_tests.rs delete mode 100644 src/server/chat_template_content.rs delete mode 100644 src/server/chat_template_content_cache_tests.rs delete mode 100644 src/server/chat_template_content_tests.rs delete mode 100644 src/server/chat_template_policy_tests.rs delete mode 100644 src/server/chat_template_raw_content_tests.rs create mode 100644 src/server/chat_template_text_wrap_tests.rs diff --git a/TECHNICAL_REPORTS/2165-typed-chat-text-20261013.en.md b/TECHNICAL_REPORTS/2165-typed-chat-text-20261013.en.md new file mode 100644 index 000000000..24afb44d3 --- /dev/null +++ b/TECHNICAL_REPORTS/2165-typed-chat-text-20261013.en.md @@ -0,0 +1,59 @@ +# Technical Report: PR #2165 - Typed Chat Text + +**Date**: 2026-10-13 +**Status**: Ready for review +**Languages**: Rust +**Risk Level**: Medium + +## Executive Summary + +Checkpoint-selected chat templates that accept typed text parts but silently discard string content are now detected once during model loading. Typed `ChatMessage` renders use a borrowed text-part view only for those templates, while raw JSON rendering and string-compatible templates retain their existing contracts. + +## 1. Problem Statement + +The Granite Vision 3.2 checkpoint ships a standalone multimodal template that is correctly preferred for image requests, but that template reads text from typed content parts. CLI and server text-only paths supplied strings, so the question was omitted from the prompt: the real checkpoint produced a 47-token prompt and an unrelated greeting instead of answering the question. + +Falling back to the tokenizer configuration's string template would split image and text behavior across templates and would make CLI, server, and prompt-cache history rendering disagree. + +## 3. Technical Decisions + +### Detect compatibility once at selection time + +After selecting the checkpoint template, the loader renders one string probe and one typed-text probe through the unchanged raw renderer. It stores `wrap_string_content = true` only when the typed probe preserves the marker and the string probe does not; either render error leaves adaptation disabled. + +### Adapt only typed messages without copying text + +`apply_inner` serializes a small borrowed wrapper containing the original role and a single typed text part. The wrapper holds `&str` references and is passed directly to MiniJinja, avoiding an intermediate JSON tree and duplicate string allocation. Exhaustive `ChatMessage` destructuring makes a future field addition fail compilation until the rendering view preserves it. + +### Preserve existing contracts + +`apply_raw*`, explicit `with_template` processors, the default processor, string-compatible templates, typed image ordering, and probe-error behavior remain unchanged. The fix does not alter CLI or server call sites and adds no dependency. + +## 7. Change Summary + +| Category | Files | Summary | +|---|---:|---| +| Production | 1 | Selection-time compatibility flag and borrowed `apply_inner` rendering view | +| Regression tests | 1 | Eight tests for exact Granite prompts, roles, empty text, raw behavior, compatible templates, constructors, and probe errors | +| Pinned fixtures | 5 | Granite Vision 3.2 and Granite 4 templates with source provenance | +| Reports | 2 | English and Korean implementation records | + +## Appendix + +### Test Results + +- `cargo test --locked -p mlxcel -p mlxcel-surgery --lib --profile test-fast --features cuda -j4 text_wrap_tests -- --test-threads=1`: 8 passed, 0 failed; 9,105 unrelated mlxcel tests and 160 surgery tests were filtered out. +- Repository contract gate passed: workspace versions, kernel dtype keys, kernel port dispatch, llama compatibility, ROCm overlay checks, 123 Python tooling tests, and 49 WebUI contract fixtures. +- On NVIDIA GB10/CUDA with `mlx-community/granite-vision-3.2-2b-4bit` revision `351831ab5218b1f66bbfdfd57bd8f20d81ce8baf`, CLI answered “The capital of France is Paris.” in 8 tokens. Server string and typed-text requests returned the same answer with 55 prompt tokens and 8 completion tokens. +- The frozen baseline returned an unrelated 20-token greeting for both server shapes with 47 prompt tokens, reproducing the defect. +- Implementation review, security rereview, and static finalization found no remaining issues. + +### Provenance + +- Frozen source diff SHA-256: `1d088cbdc926b50a1af5fc8810435dd2cccb3b846bf5477a00b363ea6d9ceaa4`. +- Candidate binary SHA-256: CLI `706fcf0864c6adbd251e2725db1f623044b49e6a33436617925dc9b1da4d09f8`; server `496534a54f6ea92ceaea5e3cfaec20237b11f753bd5fef114927208607a2998c`. +- Baseline binary SHA-256: CLI `592c33a141ccd7a497732e132ac6da302661e22400e27fbcdb2cb3ca858ae8ff`; server `bc076e4d0b2fde769eb15ababe241e72828c5f9c0b3ec2142137cef688c5a656`. + +### Validation Limits + +The issue unit ran focused CUDA tests and one real checkpoint on Linux/GB10. It did not run the full workspace test or Clippy suites, ROCm inference, macOS inference, or additional model families; the parent integration gate and hosted CI own the broad checks. diff --git a/TECHNICAL_REPORTS/2165-typed-chat-text-20261013.ko.md b/TECHNICAL_REPORTS/2165-typed-chat-text-20261013.ko.md new file mode 100644 index 000000000..a312e08b3 --- /dev/null +++ b/TECHNICAL_REPORTS/2165-typed-chat-text-20261013.ko.md @@ -0,0 +1,59 @@ +# 기술 보고서: PR #2165 - Typed Chat Text + +**작성일**: 2026-10-13 +**상태**: 리뷰 준비 완료 +**언어**: Rust +**위험도**: 중간 + +## 요약 + +typed text part는 처리하지만 문자열 content를 조용히 버리는 checkpoint 선택 템플릿을 모델 로드 시 한 번 감지하도록 수정했습니다. 해당 템플릿의 typed `ChatMessage` 렌더만 빌려온 text-part view를 사용하며, raw JSON 렌더와 문자열 호환 템플릿의 기존 계약은 그대로 유지합니다. + +## 1. 문제 정의 + +Granite Vision 3.2 checkpoint는 이미지 요청에 필요하여 우선 선택되는 standalone multimodal 템플릿을 제공하지만, 이 템플릿은 typed content part에서만 텍스트를 읽습니다. CLI와 서버의 text-only 경로는 문자열을 전달했으므로 질문이 prompt에서 사라졌고, 실제 checkpoint는 47-token prompt와 질문에 무관한 인사말을 생성했습니다. + +tokenizer configuration의 문자열 템플릿으로 fallback하면 이미지와 텍스트 요청이 서로 다른 템플릿을 사용하게 되고 CLI, 서버, prompt-cache history 렌더가 달라집니다. + +## 3. 기술적 선택과 그 이유 + +### 선택 시점에 호환성을 한 번 감지 + +checkpoint 템플릿을 선택한 뒤 변경하지 않은 raw renderer를 통해 문자열 probe와 typed-text probe를 각각 한 번 렌더합니다. typed probe만 marker를 보존할 때 `wrap_string_content = true`를 저장하며, 어느 probe에서든 렌더 오류가 발생하면 변환을 활성화하지 않습니다. + +### 텍스트 복사 없이 typed message만 변환 + +`apply_inner`는 원래 role과 하나의 typed text part를 담은 작은 borrowed wrapper를 직렬화합니다. wrapper는 `&str` 참조를 유지한 채 MiniJinja에 직접 전달되므로 중간 JSON tree와 문자열 중복 할당이 없습니다. `ChatMessage`를 빠짐없이 destructuring하여 향후 필드가 추가되면 렌더 view가 해당 필드를 보존하도록 갱신하기 전까지 컴파일이 실패합니다. + +### 기존 계약 유지 + +`apply_raw*`, 명시적인 `with_template` processor, default processor, 문자열 호환 템플릿, typed image 순서, probe 오류 동작은 바뀌지 않습니다. CLI와 서버 호출부를 수정하지 않았고 새 의존성도 없습니다. + +## 7. 변경 요약 + +| 분류 | 파일 수 | 내용 | +|---|---:|---| +| Production | 1 | 선택 시점 호환성 flag와 borrowed `apply_inner` 렌더 view | +| 회귀 테스트 | 1 | 정확한 Granite prompt, role, 빈 문자열, raw 동작, 호환 템플릿, constructor, probe 오류를 검증하는 8개 테스트 | +| 고정 fixture | 5 | 출처가 기록된 Granite Vision 3.2 및 Granite 4 템플릿 | +| 보고서 | 2 | 영문 및 국문 구현 기록 | + +## 부록 + +### 테스트 결과 + +- `cargo test --locked -p mlxcel -p mlxcel-surgery --lib --profile test-fast --features cuda -j4 text_wrap_tests -- --test-threads=1`: 8개 통과, 실패 0개이며 관련 없는 mlxcel 테스트 9,105개와 surgery 테스트 160개는 filter되었습니다. +- 저장소 contract gate를 통과했습니다: workspace version, kernel dtype key, kernel port dispatch, llama compatibility, ROCm overlay 검사, Python tooling 테스트 123개, WebUI contract fixture 49개. +- NVIDIA GB10/CUDA에서 `mlx-community/granite-vision-3.2-2b-4bit` revision `351831ab5218b1f66bbfdfd57bd8f20d81ce8baf`을 실행한 결과, CLI는 8 token으로 “The capital of France is Paris.”라고 답했습니다. 서버의 문자열 및 typed-text 요청도 모두 55 prompt token과 8 completion token으로 같은 답을 반환했습니다. +- 동결된 baseline은 두 서버 요청 형식 모두에서 47 prompt token과 질문에 무관한 20-token 인사말을 반환하여 결함을 재현했습니다. +- 구현 리뷰, 보안 재검토, 정적 최종화에서 남은 문제가 발견되지 않았습니다. + +### Provenance + +- 동결 source diff SHA-256: `1d088cbdc926b50a1af5fc8810435dd2cccb3b846bf5477a00b363ea6d9ceaa4`. +- Candidate binary SHA-256: CLI `706fcf0864c6adbd251e2725db1f623044b49e6a33436617925dc9b1da4d09f8`; server `496534a54f6ea92ceaea5e3cfaec20237b11f753bd5fef114927208607a2998c`. +- Baseline binary SHA-256: CLI `592c33a141ccd7a497732e132ac6da302661e22400e27fbcdb2cb3ca858ae8ff`; server `bc076e4d0b2fde769eb15ababe241e72828c5f9c0b3ec2142137cef688c5a656`. + +### 검증 한계 + +이 이슈 단위에서는 focused CUDA 테스트와 Linux/GB10의 실제 checkpoint 하나를 검증했습니다. 전체 workspace test 및 Clippy suite, ROCm inference, macOS inference, 추가 모델 family는 실행하지 않았으며 broad check는 상위 integration gate와 hosted CI가 담당합니다. diff --git a/src/server/chat_request.rs b/src/server/chat_request.rs index 0921396be..f8f039c93 100644 --- a/src/server/chat_request.rs +++ b/src/server/chat_request.rs @@ -2175,7 +2175,3 @@ mod tests; #[cfg(test)] #[path = "chat_request_reasoning_content_tests.rs"] mod reasoning_content_tests; - -#[cfg(test)] -#[path = "chat_request_content_tests.rs"] -mod content_tests; diff --git a/src/server/chat_request_content_tests.rs b/src/server/chat_request_content_tests.rs deleted file mode 100644 index 9ba4bd12a..000000000 --- a/src/server/chat_request_content_tests.rs +++ /dev/null @@ -1,249 +0,0 @@ -// Copyright 2025-2026 Lablup Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -//! Model-free request routing regressions for typed-only checkpoint templates. -//! -//! Embedded data URI resolves local bytes only, as in the existing request tests. - -use std::path::PathBuf; - -use serde_json::{Value, json}; - -use super::prepare_chat_request_with_cache; -use crate::server::chat_template::ChatTemplateProcessor; -use crate::server::types::ChatCompletionRequest; -use crate::tokenizer::ThinkingMarkers; - -const JSON_OBSERVER: &str = concat!( - "{% for m in messages %}", - "{% if m.content is not string %}{{ m | tojson }}{{ '\\n' }}{% endif %}", - "{% endfor %}", - "{% if add_generation_prompt %}GENERATE{% endif %}", -); - -fn checkpoint(template: &str) -> ChatTemplateProcessor { - let dir = tempfile::tempdir().unwrap(); - std::fs::write( - dir.path().join("tokenizer_config.json"), - json!({"chat_template": template}).to_string(), - ) - .unwrap(); - ChatTemplateProcessor::from_model_path(dir.path()) - .unwrap() - .unwrap() -} - -fn observed_messages(rendered: &str) -> Vec { - rendered - .strip_suffix("GENERATE") - .unwrap_or(rendered) - .lines() - .map(|line| serde_json::from_str(line).expect("template, not fallback, must render")) - .collect() -} - -fn request(messages: Value) -> ChatCompletionRequest { - serde_json::from_value(json!({"model": "test", "messages": messages})).unwrap() -} - -#[tokio::test] -async fn image_routing_keeps_system_assistant_and_plain_followup_text() { - let processor = ChatTemplateProcessor::from_model_path( - &PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("tests/fixtures/granite_vision_3_2"), - ) - .unwrap() - .unwrap(); - let req = request(json!([ - {"role": "system", "content": "SYSTEM"}, - {"role": "user", "content": [ - {"type": "text", "text": "IMAGE_QUESTION"}, - {"type": "image_url", "image_url": {"url": "data:image/png;base64,aGVsbG8="}}, - ]}, - {"role": "assistant", "content": "ANSWER"}, - {"role": "user", "content": "FOLLOWUP"}, - ])); - let prepared = prepare_chat_request_with_cache( - &processor, - &req, - None, - true, - true, - false, - &ThinkingMarkers::default(), - ) - .await - .unwrap(); - assert_eq!( - prepared.prompt, - concat!( - "<|system|>\nSYSTEM\n", - "<|user|>\n\nIMAGE_QUESTION\n", - "<|assistant|>\nANSWER<|end_of_text|>", - "<|user|>\nFOLLOWUP\n<|assistant|>\n", - ) - ); - assert_eq!(prepared.image_data, vec![b"hello".to_vec()]); - assert!( - prepared.history_prompt.is_none(), - "media history snapshot stays disabled" - ); -} - -#[tokio::test] -async fn reasoning_routing_preserves_strings_and_both_reasoning_spellings() { - let processor = checkpoint(JSON_OBSERVER); - let req = request(json!([ - {"role": "user", "content": "QUESTION"}, - {"role": "assistant", "content": "ANSWER", "reasoning_content": "TRACE"}, - {"role": "user", "content": "FOLLOWUP"}, - ])); - let prepared = prepare_chat_request_with_cache( - &processor, - &req, - None, - true, - true, - false, - &ThinkingMarkers::default(), - ) - .await - .unwrap(); - let history = prepared - .history_prompt - .as_ref() - .expect("raw text history boundary"); - assert_eq!(prepared.prompt, format!("{history}GENERATE")); - let rows = observed_messages(&prepared.prompt); - assert_eq!( - rows, - vec![ - json!({"role": "user", "content": [{"type": "text", "text": "QUESTION"}]}), - json!({ - "role": "assistant", "content": [{"type": "text", "text": "ANSWER"}], - "reasoning": "TRACE", "reasoning_content": "TRACE", - }), - json!({"role": "user", "content": [{"type": "text", "text": "FOLLOWUP"}]}), - ] - ); -} - -#[tokio::test] -async fn raw_normalization_does_not_reintroduce_stripped_old_thinking() { - let processor = checkpoint(JSON_OBSERVER); - let mut req = request(json!([ - {"role": "user", "content": "QUESTION"}, - { - "role": "assistant", - "content": [{"type": "text", "text": "PRIVATE_INLINEANSWER"}], - "reasoning_content": "PRIVATE_PARALLEL", - }, - {"role": "user", "content": "FOLLOWUP"}, - ])); - req.chat_template_kwargs = - Some(serde_json::from_value(json!({"preserve_thinking": false})).unwrap()); - let prepared = prepare_chat_request_with_cache( - &processor, - &req, - None, - true, - true, - false, - &ThinkingMarkers::default(), - ) - .await - .unwrap(); - let rows = observed_messages(&prepared.prompt); - assert_eq!(rows.len(), 3); - assert_eq!( - rows[1], - json!({ - "role": "assistant", - "content": [{"type": "text", "text": "ANSWER"}], - }) - ); - assert!(!prepared.prompt.contains("PRIVATE_")); - assert!(!prepared.history_prompt.unwrap().contains("PRIVATE_")); -} - -#[tokio::test] -async fn tool_routing_preserves_call_ids_names_arguments_and_empty_content() { - let processor = checkpoint(JSON_OBSERVER); - let req = request(json!([ - {"role": "user", "content": "QUESTION"}, - { - "role": "assistant", - "content": null, - "name": "planner", - "tool_calls": [{ - "id": "call-1", - "type": "function", - "function": {"name": "weather", "arguments": "{\"city\":\"Seoul\"}"}, - }], - }, - { - "role": "tool", - "content": "SUNNY", - "tool_call_id": "call-1", - "name": "weather", - }, - {"role": "user", "content": "FOLLOWUP"}, - ])); - let prepared = prepare_chat_request_with_cache( - &processor, - &req, - None, - true, - true, - false, - &ThinkingMarkers::default(), - ) - .await - .unwrap(); - let rows = observed_messages(&prepared.prompt); - assert_eq!(rows.len(), 4); - assert_eq!( - rows[0]["content"], - json!([{"type": "text", "text": "QUESTION"}]) - ); - assert_eq!( - rows[1], - json!({ - "role": "assistant", - "content": [{"type": "text", "text": ""}], - "name": "planner", - "tool_calls": [{ - "id": "call-1", - "type": "function", - "function": {"name": "weather", "arguments": {"city": "Seoul"}}, - }], - }) - ); - assert_eq!( - rows[2], - json!({ - "role": "tool", - "content": [{"type": "text", "text": "SUNNY"}], - "tool_call_id": "call-1", - "name": "weather", - }) - ); - assert_eq!( - rows[3]["content"], - json!([{"type": "text", "text": "FOLLOWUP"}]) - ); - let history = prepared - .history_prompt - .expect("tool-history boundary retained"); - assert_eq!(prepared.prompt, format!("{history}GENERATE")); -} diff --git a/src/server/chat_template.rs b/src/server/chat_template.rs index 31b140b50..64c2f76e0 100644 --- a/src/server/chat_template.rs +++ b/src/server/chat_template.rs @@ -52,9 +52,6 @@ use super::chat_template_kwargs::ChatTemplateKwargs; use super::tool_calls::{self, ToolCallFormat}; use super::types::request::Tool; -#[path = "chat_template_content.rs"] -mod content; - /// A message in the conversation #[derive(Debug, Clone, Serialize, Deserialize)] pub struct ChatMessage { @@ -62,6 +59,37 @@ pub struct ChatMessage { pub content: String, } +/// Borrowed serialization shape for templates that require typed text parts. +/// +/// The explicit, exhaustive destructuring in [`From`] makes adding a field to +/// [`ChatMessage`] a compile error here until that field is preserved in this +/// rendering shape as well. +#[derive(Serialize)] +struct TypedTextChatMessage<'a> { + role: &'a str, + content: [TypedTextContentPart<'a>; 1], +} + +#[derive(Serialize)] +struct TypedTextContentPart<'a> { + #[serde(rename = "type")] + kind: &'static str, + text: &'a str, +} + +impl<'a> From<&'a ChatMessage> for TypedTextChatMessage<'a> { + fn from(message: &'a ChatMessage) -> Self { + let ChatMessage { role, content } = message; + Self { + role, + content: [TypedTextContentPart { + kind: "text", + text: content, + }], + } + } +} + /// Sentinel attached as the [`std::error::Error::source`] of every error a /// chat template raises through the `raise_exception(...)` callable. /// @@ -181,10 +209,12 @@ pub struct ChatTemplateProcessor { bos_token: String, eos_token: String, add_generation_prompt: bool, - /// Checkpoint templates may need typed text parts. Decisions use the - /// effective render context and string-content roles, with a bounded cache. - /// Custom template strings keep their caller-controlled content contract. - content_normalizer: Option, + /// Whether typed [`ChatMessage`] renders must express text as content parts. + /// + /// Some multimodal checkpoint templates silently drop a plain string even + /// though they render an equivalent typed text part. This is detected once + /// while loading the checkpoint and never changes the raw JSON contract. + wrap_string_content: bool, /// Cached result of `supports_tools()` introspection. /// `None` means not yet computed. supports_tools_cached: Option, @@ -325,10 +355,7 @@ impl std::fmt::Display for ChatTemplateCompileError { impl std::error::Error for ChatTemplateCompileError {} impl ChatTemplateProcessor { - /// Create a new processor by loading template from tokenizer_config.json or chat_template.jinja. - /// - /// Used by: CLI and server model loading across families, including - /// Granite Vision, Llama and Qwen. + /// Create a new processor by loading template from tokenizer_config.json or chat_template.jinja pub fn from_model_path(model_path: &Path) -> Result> { let config_path = model_path.join("tokenizer_config.json"); let config: Option = if config_path.exists() { @@ -416,13 +443,13 @@ impl ChatTemplateProcessor { let template = preprocess_template(template); let forced_tool_call_format = detect_forced_tool_call_format(&template); - Ok(Some(Self { + let mut processor = Self { template, compiled_template: Arc::new(OnceLock::new()), bos_token, eos_token, add_generation_prompt: true, - content_normalizer: Some(content::ContentNormalizer::default()), + wrap_string_content: false, supports_tools_cached: None, forced_tool_call_format, default_enable_thinking: false, @@ -431,7 +458,9 @@ impl ChatTemplateProcessor { kimi_k3_family_unrenderable: false, #[cfg(test)] template_compile_count: Arc::new(std::sync::atomic::AtomicUsize::new(0)), - })) + }; + processor.wrap_string_content = template_requires_typed_text_content(&processor); + Ok(Some(processor)) } /// Create a processor with a custom template string @@ -444,7 +473,7 @@ impl ChatTemplateProcessor { bos_token: String::new(), eos_token: String::new(), add_generation_prompt: true, - content_normalizer: None, + wrap_string_content: false, supports_tools_cached: None, forced_tool_call_format, default_enable_thinking: false, @@ -927,11 +956,6 @@ impl ChatTemplateProcessor { /// `[{"type": "image"}, {"type": "text", "text": "..."}]`) that Jinja2 /// templates like Gemma3 VLM can iterate over. /// - /// Checkpoint-loaded processors conservatively adapt eligible string - /// content to typed text parts in the effective render context. Existing - /// arrays, null content and message metadata are untouched. Processors - /// created with with_template retain the caller's original raw contract. - /// /// When `tools` is `Some`, the tool definitions are passed to the Jinja2 /// template context, enabling tool-calling prompt formatting. /// @@ -956,12 +980,11 @@ impl ChatTemplateProcessor { /// plumbing path used's `preserve_thinking` feature and /// generalizes to any future kwarg a HuggingFace chat template expects. /// - /// Canonical server keys (messages, bos_token, eos_token, - /// add_generation_prompt and tools) are reserved; attempts to override - /// them through kwargs are ignored. enable_thinking remains overridable. - /// Other kwargs enter the effective context used for both adaptation and - /// rendering. - // Used by: chat_request, routes/chat, checkpoint-backed prompt inspection + /// When a kwarg key duplicates an already-provided context entry + /// (`messages`, `bos_token`, `eos_token`, `add_generation_prompt`, + /// `tools`), the kwarg value wins — this lets operators override defaults + /// if a template ships with unusual expectations. + // Used by: chat_request, routes/chat pub fn apply_raw_with_kwargs( &self, messages: &serde_json::Value, @@ -999,6 +1022,9 @@ impl ChatTemplateProcessor { kwargs: &ChatTemplateKwargs, add_generation_prompt: bool, ) -> Result { + // Convert serde_json::Value to minijinja::Value + let messages_val = minijinja::Value::from_serialize(messages); + // Leave `tools` UNDEFINED when the request carries no tools, and count // an explicit empty list as no tools (issue #1597). Three guard shapes // ship in the local template corpus and only an absent key satisfies @@ -1029,7 +1055,7 @@ impl ChatTemplateProcessor { .map(minijinja::Value::from_serialize); let context = build_template_context( - Value::UNDEFINED, + messages_val, &self.bos_token, &self.eos_token, add_generation_prompt, @@ -1040,16 +1066,11 @@ impl ChatTemplateProcessor { self.thinking_mode_sentinel().as_deref(), ); - // Both ordinary and raw messages use this one adaptation boundary. - // Probes call the unchecked renderer, so they cannot normalize again. - // Existing arrays, null content and message metadata remain untouched. - let messages = match &self.content_normalizer { - Some(normalizer) => { - normalizer.normalize(messages, &context, |probe| self.render_template(probe)) - } - None => std::borrow::Cow::Borrowed(messages), - }; - let rendered = self.render_template(content::with_messages(&context, messages.as_ref()))?; + // Render directly: minijinja reproduces the template's own + // enable_thinking branches faithfully (issue #686), so no post-render + // Gemma-4 patching is applied. The `enable_thinking` value reaches the + // template through `build_template_context` above. + let rendered = self.render_template(context)?; Ok(self.complete_generation_prompt(rendered, add_generation_prompt)) } @@ -1095,8 +1116,6 @@ impl ChatTemplateProcessor { self.apply_inner(messages, tools, kwargs, false) } - // Used by: text-only CLI and server generation/history renders across - // model families. Share adaptation, kwargs and rendering with the raw path. fn apply_inner( &self, messages: &[ChatMessage], @@ -1104,9 +1123,43 @@ impl ChatTemplateProcessor { kwargs: &ChatTemplateKwargs, add_generation_prompt: bool, ) -> Result { - let messages = - serde_json::to_value(messages).context("Failed to serialize chat messages")?; - self.apply_raw_inner(&messages, tools, kwargs, add_generation_prompt) + // Same `tools` rule as `apply_raw_inner`: the key is absent when the + // request carries no tools, and an explicit empty list counts as no + // tools (issue #1597). See that function for why undefined rather than + // `none` or an empty list is the value that every shipped guard shape + // agrees on. + let tools_val = tools + .filter(|t| !t.is_empty()) + .map(minijinja::Value::from_serialize); + + let wrapped_messages = self.wrap_string_content.then(|| { + messages + .iter() + .map(TypedTextChatMessage::from) + .collect::>() + }); + let messages_val = match wrapped_messages.as_ref() { + Some(messages) => minijinja::Value::from_serialize(messages), + None => minijinja::Value::from_serialize(messages), + }; + let context = build_template_context( + messages_val, + &self.bos_token, + &self.eos_token, + add_generation_prompt, + tools_val, + kwargs, + self.default_enable_thinking, + self.wants_bare_thinking_alias(), + self.thinking_mode_sentinel().as_deref(), + ); + + // Render directly: minijinja reproduces the template's own + // enable_thinking branches faithfully (issue #686), so no post-render + // Gemma-4 patching is applied. The `enable_thinking` value reaches the + // template through `build_template_context` above. + let rendered = self.render_template(context)?; + Ok(self.complete_generation_prompt(rendered, add_generation_prompt)) } } @@ -1473,6 +1526,34 @@ fn template_renders_typed_image_content(template: &str) -> bool { } } +const TEXT_CONTENT_PROBE_MARKER: &str = "__mlxcel_text_probe__"; + +/// Detect a checkpoint template that accepts typed text parts but drops strings. +/// +/// Both probes go through the raw renderer so this decision cannot observe or +/// recursively trigger the typed-message adaptation it controls. Any render +/// error is treated conservatively as no adaptation. +fn template_requires_typed_text_content(processor: &ChatTemplateProcessor) -> bool { + let string_content = serde_json::json!([{ + "role": "user", + "content": TEXT_CONTENT_PROBE_MARKER, + }]); + let typed_content = serde_json::json!([{ + "role": "user", + "content": [{"type": "text", "text": TEXT_CONTENT_PROBE_MARKER}], + }]); + match ( + processor.apply_raw(&string_content, None), + processor.apply_raw(&typed_content, None), + ) { + (Ok(string_render), Ok(typed_render)) => { + !string_render.contains(TEXT_CONTENT_PROBE_MARKER) + && typed_render.contains(TEXT_CONTENT_PROBE_MARKER) + } + _ => false, + } +} + fn preprocess_template(template: String) -> String { // Handle all whitespace-control variants of the two delimiters. Using // explicit replaces keeps this allocation-light and regex-free. @@ -5035,13 +5116,5 @@ TOOL } #[cfg(test)] -#[path = "chat_template_content_tests.rs"] -mod content_tests; - -#[cfg(test)] -#[path = "chat_template_raw_content_tests.rs"] -mod raw_content_tests; - -#[cfg(test)] -#[path = "chat_template_policy_tests.rs"] -mod policy_tests; +#[path = "chat_template_text_wrap_tests.rs"] +mod text_wrap_tests; diff --git a/src/server/chat_template_content.rs b/src/server/chat_template_content.rs deleted file mode 100644 index e2e56f7e6..000000000 --- a/src/server/chat_template_content.rs +++ /dev/null @@ -1,325 +0,0 @@ -// Copyright 2025-2026 Lablup Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -//! Conservative, context-sensitive adaptation for typed-content-only templates. -//! -//! Probes establish a limited compatibility heuristic, not a proof about every -//! possible conversation. A successful user-only probe never licenses changing -//! an already string-compatible system or assistant branch. Decisions are keyed -//! by the effective render context and the roles being converted, not messages. - -use std::borrow::Cow; -use std::collections::{BTreeSet, VecDeque}; -use std::io::Write; -use std::sync::{Arc, Mutex}; - -use anyhow::Result; -use minijinja::Value; -use serde_json::{Value as JsonValue, json}; - -const CACHE_ENTRIES: usize = 16; -const MAX_CACHE_KEY_BYTES: usize = 16 * 1024; -// Different lengths, initial/final characters and case catch more transformations -// than equal-length markers sharing a prefix (uppercasing, slicing, length, …). -const PROBE_A: &str = "a7!mlx_probe_Z"; -const PROBE_B: &str = "Q92_DIFFERENT_text_payload?end6"; -const USER_BEFORE: &str = "u3_CONTEXT_before!M"; -const USER_AFTER: &str = "v84_CONTEXT_after?N"; - -type DecisionCache = VecDeque<(Vec, bool)>; - -#[derive(Clone, Default)] -pub(super) struct ContentNormalizer { - // Clones share bounded decisions keyed by effective context and roles. - // Keys may retain kwargs/tools, but not message objects or rendered output. - // No template code runs while this mutex is held. - cache: Arc>, -} - -impl ContentNormalizer { - /// Normalize string content only, preserving every other message field. - /// - /// The context must be the effective context with messages undefined. - /// The render callback is unchecked, never another normalization entry. - /// Used by: checkpoint-backed CLI/server generation and history renders, - /// including their raw multimodal/tool-message counterparts. - pub(super) fn normalize<'a>( - &self, - messages: &'a JsonValue, - context: &Value, - render: impl Fn(Value) -> Result, - ) -> Cow<'a, JsonValue> { - let Some(roles) = string_roles(messages) else { - return Cow::Borrowed(messages); - }; - if roles.is_empty() { - return Cow::Borrowed(messages); - } - - // Metadata can select a different content schema even for the same - // role. Never retain it in keys or reuse a plain-message decision. - let has_metadata = has_message_metadata(messages); - let key = if has_metadata { - None - } else { - cache_key(context, &roles) - }; - let cached = key.as_ref().and_then(|key| { - self.cache.lock().ok().and_then(|cache| { - cache - .iter() - .find_map(|(entry, decision)| (entry == key).then_some(*decision)) - }) - }); - let should_wrap = cached.unwrap_or_else(|| { - let decision = infer(context, &roles, messages, has_metadata, &render); - if let Some(key) = key - && let Ok(mut cache) = self.cache.lock() - { - // Another clone may have populated this key while we rendered. - if !cache.iter().any(|(entry, _)| entry == &key) { - if cache.len() == CACHE_ENTRIES { - cache.pop_front(); - } - cache.push_back((key, decision)); - } - } - decision - }); - if !should_wrap { - return Cow::Borrowed(messages); - } - - let mut normalized = messages.clone(); - // The array and role/content shape were checked by string_roles. - if let Some(messages) = normalized.as_array_mut() { - for message in messages { - if let Some(content) = message.get_mut("content") - && let JsonValue::String(text) = content - { - *content = typed_text(std::mem::take(text)); - } - } - } - Cow::Owned(normalized) - } -} - -fn string_roles(messages: &JsonValue) -> Option> { - let mut roles = BTreeSet::new(); - for message in messages.as_array()? { - if message.get("content").is_some_and(JsonValue::is_string) { - roles.insert(message.get("role")?.as_str()?.to_owned()); - } - } - Some(roles) -} - -fn has_message_metadata(messages: &JsonValue) -> bool { - messages.as_array().is_some_and(|messages| { - messages.iter().any(|message| { - message - .as_object() - .is_some_and(|fields| fields.keys().any(|key| key != "role" && key != "content")) - }) - }) -} - -/// Replace only the messages slot without rebuilding or logging kwargs. -pub(super) fn with_messages(context: &Value, messages: &JsonValue) -> Value { - minijinja::context! { - messages => Value::from_serialize(messages), - ..context.clone() - } -} - -fn infer( - context: &Value, - roles: &BTreeSet, - messages: &JsonValue, - has_metadata: bool, - render: &impl Fn(Value) -> Result, -) -> bool { - let render_messages = |messages: &JsonValue| render(with_messages(context, messages)); - - // A transformation is not a dropped message: require two dissimilar - // string payloads to produce the same successful output. In particular, - // rejecting strings must not itself enable normalization. - let string_a = render_messages(&JsonValue::Array(vec![message("user", PROBE_A, false)])); - let string_b = render_messages(&JsonValue::Array(vec![message("user", PROBE_B, false)])); - let (Ok(string_a), Ok(string_b)) = (string_a, string_b) else { - return false; - }; - if string_a != string_b { - return false; - } - for (marker, other) in [(PROBE_A, PROBE_B), (PROBE_B, PROBE_A)] { - let Ok(typed) = render_messages(&JsonValue::Array(vec![message("user", marker, true)])) - else { - return false; - }; - if !typed.contains(marker) || typed.contains(other) { - return false; - } - } - - if has_metadata { - // Inspect every string-message prototype, including user messages: - // name, tool calls, reasoning and other values can select a schema. - // Only the content changes in these probes; actual metadata survives. - for prototype in messages.as_array().into_iter().flatten() { - if prototype.get("content").is_some_and(JsonValue::is_string) - && !prototype_can_wrap(prototype, &render_messages) - { - return false; - } - } - } else { - // Absent roles do not block an otherwise valid user-only template. - for role in roles.iter().filter(|role| role.as_str() != "user") { - if !prototype_can_wrap(&message(role, "", false), &render_messages) { - return false; - } - } - } - true -} - -fn prototype_can_wrap( - prototype: &JsonValue, - render: &impl Fn(&JsonValue) -> Result, -) -> bool { - for (marker, other) in [(PROBE_A, PROBE_B), (PROBE_B, PROBE_A)] { - let (messages, required) = prototype_scaffold(prototype, marker, true); - let Ok(typed) = render(&messages) else { - return false; - }; - if typed.contains(other) || required.iter().any(|text| !typed.contains(text.as_str())) { - return false; - } - } - - let string_a = render(&prototype_scaffold(prototype, PROBE_A, false).0); - let string_b = render(&prototype_scaffold(prototype, PROBE_B, false).0); - match (string_a, string_b) { - (Ok(a), Ok(b)) => a == b, - // Granite Vision 3.2 indexes system/assistant content[0].text, so a - // string in those roles is an engine error. Accept this only AFTER - // the successful user silent-drop gate and both typed-role renders - // preserved all markers. Explicit rejections still veto adaptation, - // as do inconsistent success/error outcomes. - (Err(a), Err(b)) => { - super::template_rejection_message(&a).is_none() - && super::template_rejection_message(&b).is_none() - } - _ => false, - } -} - -// Used by: checkpoint-backed string normalization and compatibility probes. -fn typed_text(text: String) -> JsonValue { - // Build the part from owned values: json! would serialize and clone text. - let part = serde_json::Map::from_iter([ - ("type".to_owned(), JsonValue::String("text".to_owned())), - ("text".to_owned(), JsonValue::String(text)), - ]); - JsonValue::Array(vec![JsonValue::Object(part)]) -} - -fn message(role: &str, text: &str, typed: bool) -> JsonValue { - let content = if typed { - typed_text(text.to_owned()) - } else { - json!(text) - }; - json!({"role": role, "content": content}) -} - -fn prototype_scaffold( - prototype: &JsonValue, - marker: &str, - typed: bool, -) -> (JsonValue, Vec) { - // Used by: checkpoint-backed text/raw generation and history probes. - // string_roles has already validated each converted message's role. - // Preserve metadata without cloning the potentially large, discarded body. - let target = JsonValue::Object( - prototype - .as_object() - .expect("string-content messages have validated roles") - .iter() - .map(|(key, value)| { - let value = if key == "content" { - if typed { - typed_text(marker.to_owned()) - } else { - JsonValue::String(marker.to_owned()) - } - } else { - value.clone() - }; - (key.clone(), value) - }) - .collect(), - ); - match prototype.get("role").and_then(JsonValue::as_str) { - Some("user") => (JsonValue::Array(vec![target]), vec![marker.to_owned()]), - Some("system") => ( - JsonValue::Array(vec![target, message("user", USER_AFTER, true)]), - vec![marker.to_owned(), USER_AFTER.to_owned()], - ), - _ => ( - JsonValue::Array(vec![ - message("user", USER_BEFORE, true), - target, - message("user", USER_AFTER, true), - ]), - vec![ - USER_BEFORE.to_owned(), - marker.to_owned(), - USER_AFTER.to_owned(), - ], - ), - } -} - -fn cache_key(context: &Value, roles: &BTreeSet) -> Option> { - // Bound serialized key bytes, not allocator/container overhead. Oversized - // contexts are still checked; their decisions are simply not cached. - let mut writer = KeyWriter(Vec::new()); - serde_json::to_writer(&mut writer, &(context, roles)).ok()?; - Some(writer.0) -} - -struct KeyWriter(Vec); - -impl Write for KeyWriter { - fn write(&mut self, bytes: &[u8]) -> std::io::Result { - if bytes.len() > MAX_CACHE_KEY_BYTES.saturating_sub(self.0.len()) { - return Err(std::io::Error::other( - "content-adaptation cache key too large", - )); - } - self.0.extend_from_slice(bytes); - Ok(bytes.len()) - } - - fn flush(&mut self) -> std::io::Result<()> { - Ok(()) - } -} - -#[cfg(test)] -#[path = "chat_template_content_cache_tests.rs"] -mod cache_tests; diff --git a/src/server/chat_template_content_cache_tests.rs b/src/server/chat_template_content_cache_tests.rs deleted file mode 100644 index b95fb2f9e..000000000 --- a/src/server/chat_template_content_cache_tests.rs +++ /dev/null @@ -1,136 +0,0 @@ -// Copyright 2025-2026 Lablup Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -use std::cell::Cell; - -use super::*; - -fn typed_only(context: Value, calls: &Cell) -> Result { - calls.set(calls.get() + 1); - let mut output = String::new(); - for message in context.get_attr("messages")?.try_iter()? { - let content = message.get_attr("content")?; - if content.as_str().is_none() { - for part in content.try_iter()? { - if let Some(text) = part.get_attr("text")?.as_str() { - output.push_str(text); - } - } - } - } - Ok(output) -} - -#[test] -fn cloned_cache_reuses_decisions_without_retaining_messages_or_metadata() { - let normalizer = ContentNormalizer::default(); - let context = minijinja::context! { messages => Value::UNDEFINED }; - let calls = Cell::new(0); - let first = json!([{"role": "user", "content": "first"}]); - assert!( - normalizer.normalize(&first, &context, |ctx| typed_only(ctx, &calls))[0]["content"] - .is_array() - ); - assert_eq!(calls.get(), 4); - let second = json!([{"role": "user", "content": "different private text"}]); - assert!( - normalizer - .clone() - .normalize(&second, &context, |ctx| typed_only(ctx, &calls))[0]["content"] - .is_array() - ); - assert_eq!(calls.get(), 4, "a warm clone must run no probes"); - - let metadata = json!([{ - "role": "user", "content": "question", "reasoning_content": "private metadata" - }]); - for expected in [12, 20] { - assert!( - normalizer.normalize(&metadata, &context, |ctx| typed_only(ctx, &calls))[0]["content"] - .is_array() - ); - assert_eq!(calls.get(), expected, "metadata is checked without caching"); - } - let cache = normalizer.cache.lock().unwrap(); - assert_eq!(cache.len(), 1); - let key = String::from_utf8_lossy(&cache[0].0); - assert!(!key.contains("different private text")); - assert!(!key.contains("private metadata")); -} - -#[test] -fn poisoned_cache_is_bypassed_without_changing_normalization() { - let normalizer = ContentNormalizer::default(); - let cache = normalizer.cache.clone(); - let worker = std::thread::spawn(move || { - let _guard = cache.lock().unwrap(); - panic!("deliberately poison the decision cache"); - }); - assert!(worker.join().is_err()); - assert!(normalizer.cache.is_poisoned()); - - let messages = json!([{"role": "user", "content": "still normalize"}]); - let calls = Cell::new(0); - let normalized = normalizer.normalize( - &messages, - &minijinja::context! { messages => Value::UNDEFINED }, - |ctx| typed_only(ctx, &calls), - ); - assert_eq!(normalized[0]["content"][0]["text"], "still normalize"); - assert_eq!(calls.get(), 4); -} - -#[test] -fn cache_bounds_entries_and_key_allocations_but_still_checks_large_contexts() { - let normalizer = ContentNormalizer::default(); - let messages = json!([{"role": "user", "content": "bounded"}]); - let calls = Cell::new(0); - for mode in 0..CACHE_ENTRIES + 4 { - let context = minijinja::context! { messages => Value::UNDEFINED, mode }; - assert!( - normalizer.normalize(&messages, &context, |ctx| typed_only(ctx, &calls))[0]["content"] - .is_array() - ); - } - let cache = normalizer.cache.lock().unwrap(); - assert_eq!(cache.len(), CACHE_ENTRIES); - assert!( - cache - .iter() - .all(|(key, _)| key.len() <= MAX_CACHE_KEY_BYTES) - ); - drop(cache); - - let large = minijinja::context! { - messages => Value::UNDEFINED, - hint => "x".repeat(MAX_CACHE_KEY_BYTES + 1) - }; - let roles = string_roles(&messages).unwrap(); - assert!(cache_key(&large, &roles).is_none()); - let before = calls.get(); - for _ in 0..2 { - assert!( - normalizer.normalize(&messages, &large, |ctx| typed_only(ctx, &calls))[0]["content"] - .is_array() - ); - } - assert_eq!(calls.get(), before + 8, "oversized contexts run uncached"); - assert_eq!(normalizer.cache.lock().unwrap().len(), CACHE_ENTRIES); - let mut writer = KeyWriter(Vec::new()); - assert!(writer.write(&[0; MAX_CACHE_KEY_BYTES + 1]).is_err()); - assert!( - writer.0.is_empty(), - "oversized write must not allocate its payload" - ); -} diff --git a/src/server/chat_template_content_tests.rs b/src/server/chat_template_content_tests.rs deleted file mode 100644 index 86c0dbefb..000000000 --- a/src/server/chat_template_content_tests.rs +++ /dev/null @@ -1,334 +0,0 @@ -// Copyright 2025-2026 Lablup Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -//! Checkpoint-selected templates that require typed text content (#2135). - -use std::path::PathBuf; - -use super::{ChatMessage, ChatTemplateProcessor}; -use serde_json::{Value, json}; - -const QUESTION: &str = "What is the capital of France?"; -const GRANITE_SYSTEM: &str = "<|system|>\nA chat between a curious user and an artificial intelligence assistant. The assistant gives helpful, detailed, and polite answers to the user's questions.\n"; -const GRANITE_3: &str = include_str!("../../tests/fixtures/granite_vision_3_2/chat_template.jinja"); -const GRANITE_3_CONFIG: &str = - include_str!("../../tests/fixtures/granite_vision_3_2/tokenizer_config.json"); - -fn message(role: &str, content: &str) -> ChatMessage { - ChatMessage { - role: role.into(), - content: content.into(), - } -} - -fn granite_3() -> ChatTemplateProcessor { - let path = PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("tests/fixtures/granite_vision_3_2"); - ChatTemplateProcessor::from_model_path(&path) - .unwrap() - .unwrap() -} - -fn checkpoint(template: &str) -> ChatTemplateProcessor { - let dir = tempfile::tempdir().unwrap(); - std::fs::write( - dir.path().join("tokenizer_config.json"), - json!({"chat_template": template}).to_string(), - ) - .unwrap(); - ChatTemplateProcessor::from_model_path(dir.path()) - .unwrap() - .unwrap() -} - -#[test] -fn granite_3_from_model_path_preserves_the_exact_user_prompt() { - // The standalone typed template must win over the competing string one, - // then receive typed text. This assertion fails against unmodified main. - let processor = granite_3(); - let actual = processor.apply(&[message("user", QUESTION)], None).unwrap(); - assert_eq!( - actual, - format!("{GRANITE_SYSTEM}<|user|>\n{QUESTION}\n<|assistant|>\n") - ); -} - -#[test] -fn granite_3_wraps_system_assistant_and_history_content() { - let processor = granite_3(); - let messages = [ - message("system", "Answer concisely."), - message("user", QUESTION), - message("assistant", "Paris."), - message("user", "And Germany?"), - ]; - let expected = concat!( - "<|system|>\nAnswer concisely.\n", - "<|user|>\nWhat is the capital of France?\n", - "<|assistant|>\nParis.<|end_of_text|>", - "<|user|>\nAnd Germany?\n" - ); - assert_eq!( - processor.apply(&messages, None).unwrap(), - format!("{expected}<|assistant|>\n") - ); - assert_eq!( - processor - .apply_history_with_kwargs(&messages, None, &Default::default()) - .unwrap(), - expected - ); - let mut no_generation_prompt = processor.clone(); - no_generation_prompt.set_add_generation_prompt(false); - assert_eq!( - no_generation_prompt.apply(&messages, None).unwrap(), - expected - ); -} - -#[test] -fn empty_text_remains_one_empty_text_part_in_every_role() { - let processor = granite_3(); - let messages = [ - message("system", ""), - message("user", ""), - message("assistant", ""), - ]; - assert_eq!( - processor - .apply_history_with_kwargs(&messages, None, &Default::default()) - .unwrap(), - "<|system|>\n\n<|user|>\n\n<|assistant|>\n<|end_of_text|>" - ); - - let count_parts = checkpoint( - "{% for m in messages %}{% if m.content is not string %}{{ m.content | length }}:{% for p in m.content %}{{ p.type }}={{ p.text }}{% endfor %}{% endif %}{% endfor %}", - ); - assert_eq!( - count_parts.apply(&[message("user", "")], None).unwrap(), - "1:text=" - ); -} - -#[test] -fn granite_3_raw_image_is_unchanged_and_strings_are_normalized() { - let processor = granite_3(); - let raw = json!([{ - "role": "user", - "content": [{"type": "image"}, {"type": "text", "text": QUESTION}], - }]); - assert_eq!( - processor.apply_raw(&raw, None).unwrap(), - format!("{GRANITE_SYSTEM}<|user|>\n\n{QUESTION}\n<|assistant|>\n") - ); - assert_eq!( - processor - .apply_raw_history_with_kwargs(&raw, None, &Default::default()) - .unwrap(), - format!("{GRANITE_SYSTEM}<|user|>\n\n{QUESTION}\n") - ); - // Raw strings use the same normalization as ordinary messages. - let raw_string = json!([{"role": "user", "content": QUESTION}]); - assert_eq!( - processor.apply_raw(&raw_string, None).unwrap(), - format!("{GRANITE_SYSTEM}<|user|>\n{QUESTION}\n<|assistant|>\n") - ); -} - -#[test] -fn raw_message_metadata_survives_on_a_wrapping_processor() { - let processor = checkpoint( - "{% for m in messages %}{{ m.role }}|{{ m.name | default('') }}|{{ m.tool_call_id | default('') }}|{{ m.reasoning | default('') }}|{% if m.tool_calls is defined %}{{ m.tool_calls[0].function.name }}{% endif %}|{% for p in m.content | selectattr('type', 'equalto', 'text') %}{{ p.text }}{% endfor %}{% endfor %}", - ); - let raw = json!([{ - "role": "assistant", - "content": [{"type": "text", "text": "answer"}], - "name": "speaker", - "tool_call_id": "call-1", - "reasoning": "reason", - "tool_calls": [{"function": {"name": "weather"}}], - }]); - assert_eq!( - processor.apply_raw(&raw, None).unwrap(), - "assistant|speaker|call-1|reason|weather|answer" - ); -} - -#[test] -fn granite_3_config_only_string_template_is_not_wrapped() { - let dir = tempfile::tempdir().unwrap(); - std::fs::write(dir.path().join("tokenizer_config.json"), GRANITE_3_CONFIG).unwrap(); - let processor = ChatTemplateProcessor::from_model_path(dir.path()) - .unwrap() - .unwrap(); - assert_eq!( - processor.apply(&[message("user", QUESTION)], None).unwrap(), - format!("{GRANITE_SYSTEM}<|user|>\n{QUESTION}\n<|assistant|>\n") - ); -} - -#[test] -fn granite_4_accepts_both_shapes_without_wrapping() { - let path = PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("tests/fixtures/granite_4_vision"); - let processor = ChatTemplateProcessor::from_model_path(&path) - .unwrap() - .unwrap(); - let messages = [ - message("system", "Answer concisely."), - message("user", QUESTION), - message("assistant", "Paris."), - ]; - let raw: Vec = messages - .iter() - .map(|m| json!({"role": m.role, "content": [{"type": "text", "text": m.content}]})) - .collect(); - // The checkpoint's render_content macro preserves eight indentation - // spaces for strings, but trims them for typed text parts. Both forms - // retain their original prompt contract; they need not render identically. - let expected_string = concat!( - "<|start_of_role|>system<|end_of_role|>Answer concisely.<|end_of_text|>\n", - "<|start_of_role|>user<|end_of_role|> What is the capital of France?<|end_of_text|>\n", - "<|start_of_role|>assistant<|end_of_role|> Paris.<|end_of_text|>\n", - "<|start_of_role|>assistant<|end_of_role|>" - ); - let expected_typed = concat!( - "<|start_of_role|>system<|end_of_role|>Answer concisely.<|end_of_text|>\n", - "<|start_of_role|>user<|end_of_role|>What is the capital of France?<|end_of_text|>\n", - "<|start_of_role|>assistant<|end_of_role|>Paris.<|end_of_text|>\n", - "<|start_of_role|>assistant<|end_of_role|>" - ); - let original = ChatTemplateProcessor::with_template( - std::fs::read_to_string(path.join("chat_template.jinja")).unwrap(), - ); - // Custom-template and raw rendering keep the pre-fix content contract. - assert_eq!(original.apply(&messages, None).unwrap(), expected_string); - assert_eq!( - original.apply_raw(&json!(raw), None).unwrap(), - expected_typed - ); - assert_eq!(processor.apply(&messages, None).unwrap(), expected_string); - assert_eq!( - processor.apply_raw(&json!(messages), None).unwrap(), - expected_string - ); - assert_eq!( - processor.apply_raw(&json!(raw), None).unwrap(), - expected_typed - ); -} - -#[test] -fn default_and_template_override_do_not_opt_into_wrapping() { - let default = ChatTemplateProcessor::default(); - assert!( - default - .apply(&[message("user", QUESTION)], None) - .unwrap() - .contains(QUESTION) - ); - - let explicit = ChatTemplateProcessor::with_template(GRANITE_3.into()); - assert!( - !explicit - .apply(&[message("user", QUESTION)], None) - .unwrap() - .contains(QUESTION) - ); -} - -#[test] -fn probe_uses_the_selected_processors_special_tokens_and_preprocessing() { - let dir = tempfile::tempdir().unwrap(); - let template = concat!( - "{% if bos_token != '' or eos_token != '' %}", - "{{ raise_exception('missing checkpoint tokens') }}{% endif %}", - "{% generation %}{% for p in messages[0].content | selectattr('type', 'equalto', 'text') %}", - "{{ bos_token }}{{ p.text }}{{ eos_token }}{% endfor %}{% endgeneration %}" - ); - std::fs::write( - dir.path().join("tokenizer_config.json"), - json!({ - "chat_template": template, - "bos_token": {"content": ""}, - "eos_token": "", - }) - .to_string(), - ) - .unwrap(); - let processor = ChatTemplateProcessor::from_model_path(dir.path()) - .unwrap() - .unwrap(); - assert_eq!( - processor.apply(&[message("user", QUESTION)], None).unwrap(), - format!("{QUESTION}") - ); - assert_eq!( - processor.template_compile_count(), - 1, - "probes reuse the compiled template" - ); -} - -#[test] -fn either_probe_render_error_leaves_wrapping_disabled() { - for (template, string_succeeds) in [ - ( - "{% if messages[0].content is string %}{{ raise_exception('string rejected') }}{% else %}{{ messages[0].content[0].text }}{% endif %}", - false, - ), - ( - "{% if messages[0].content is string %}{{ messages[0].content }}{% else %}{{ raise_exception('typed rejected') }}{% endif %}", - true, - ), - ( - "{% if messages[0].content is not string %}{{ raise_exception('typed rejected') }}{% endif %}", - true, - ), - ( - "{% if messages[0].content is string %}{{ unavailable_probe_function() }}{% else %}{{ messages[0].content[0].text }}{% endif %}", - false, - ), - ] { - let processor = checkpoint(template); - assert_eq!( - processor.apply(&[message("user", QUESTION)], None).is_ok(), - string_succeeds, - "{template}" - ); - } -} - -#[test] -fn parse_errors_and_missing_typed_markers_do_not_enable_wrapping() { - let malformed = checkpoint("{% for m in messages %}"); - assert!(malformed.apply(&[message("user", QUESTION)], None).is_err()); - - let drops_both = checkpoint("fixed prompt"); - assert_eq!( - drops_both - .apply(&[message("user", QUESTION)], None) - .unwrap(), - "fixed prompt" - ); -} - -#[test] -fn a_string_accepting_template_keeps_the_original_content_shape() { - let processor = checkpoint( - "{% for m in messages %}{% if m.content is string %}string:{{ m.content }}{% else %}typed:{{ m.content[0].text }}{% endif %}{% endfor %}", - ); - assert_eq!( - processor.apply(&[message("user", QUESTION)], None).unwrap(), - format!("string:{QUESTION}") - ); -} diff --git a/src/server/chat_template_policy_tests.rs b/src/server/chat_template_policy_tests.rs deleted file mode 100644 index ea2d34a62..000000000 --- a/src/server/chat_template_policy_tests.rs +++ /dev/null @@ -1,543 +0,0 @@ -// Copyright 2025-2026 Lablup Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -//! Context-sensitive compatibility policy regressions through public APIs. - -use serde_json::json; - -use super::{ChatMessage, ChatTemplateProcessor}; -use crate::server::chat_template_kwargs::ChatTemplateKwargs; -use crate::server::types::request::Tool; - -fn checkpoint(template: &str) -> ChatTemplateProcessor { - let dir = tempfile::tempdir().unwrap(); - std::fs::write( - dir.path().join("tokenizer_config.json"), - json!({"chat_template": template}).to_string(), - ) - .unwrap(); - ChatTemplateProcessor::from_model_path(dir.path()) - .unwrap() - .unwrap() -} - -fn message(role: &str, content: &str) -> ChatMessage { - ChatMessage { - role: role.into(), - content: content.into(), - } -} - -fn kwargs(value: serde_json::Value) -> ChatTemplateKwargs { - ChatTemplateKwargs::from_json_object(value.as_object().unwrap().clone()) -} - -const THINKING_SCHEMA: &str = concat!( - "{% if enable_thinking %}STRING:{% else %}TYPED:{% endif %}", - "{% for m in messages %}", - "{% if enable_thinking %}{{ m.content }}", - "{% else %}{% for p in m.content | selectattr('type', 'equalto', 'text') %}", - "{{ p.text }}{% endfor %}{% endif %}{% endfor %}", -); - -#[test] -fn transformed_string_content_is_not_mistaken_for_a_dropped_question() { - let processor = checkpoint(concat!( - "{% for m in messages %}", - "{% if m.content is string %}{{ m.content | upper }}", - "{% else %}{% for p in m.content | selectattr('type', 'equalto', 'text') %}", - "{{ p.text }}{% endfor %}{% endif %}{% endfor %}", - )); - let messages = [message("user", "Mixed_CASE question")]; - assert_eq!( - processor.apply(&messages, None).unwrap(), - "MIXED_CASE QUESTION" - ); - assert_eq!( - processor.apply_raw(&json!(messages), None).unwrap(), - "MIXED_CASE QUESTION" - ); -} - -#[test] -fn hybrid_roles_preserve_original_strings_without_poisoning_user_only_policy() { - let processor = checkpoint(concat!( - "{% for m in messages %}{{ m.role }}:", - "{% if m.role == 'user' %}", - "{% for p in m.content | selectattr('type', 'equalto', 'text') %}{{ p.text }}{% endfor %}", - "{% else %}{{ m.content }}{% endif %}|{% endfor %}", - )); - let user = [message("user", "QUESTION")]; - let conversation = [ - message("system", "SYSTEM"), - message("user", "QUESTION"), - message("assistant", "ANSWER"), - message("user", "FOLLOWUP"), - ]; - // User-only input can be repaired. If a real string system/assistant role - // is incompatible with wrapping, preserve the whole original conversation - // instead of corrupting previously correct instructions/history. - assert_eq!(processor.apply(&user, None).unwrap(), "user:QUESTION|"); - let original = "system:SYSTEM|user:|assistant:ANSWER|user:|"; - assert_eq!(processor.apply(&conversation, None).unwrap(), original); - assert_eq!( - processor.apply_raw(&json!(conversation), None).unwrap(), - original - ); - assert_eq!(processor.apply(&user, None).unwrap(), "user:QUESTION|"); -} - -#[test] -fn changing_default_thinking_rechecks_the_effective_content_contract() { - let mut processor = checkpoint(THINKING_SCHEMA); - let messages = [message("user", "QUESTION")]; - assert_eq!(processor.apply(&messages, None).unwrap(), "TYPED:QUESTION"); - processor.set_default_enable_thinking(true); - assert_eq!(processor.apply(&messages, None).unwrap(), "STRING:QUESTION"); - processor.set_default_enable_thinking(false); - assert_eq!(processor.apply(&messages, None).unwrap(), "TYPED:QUESTION"); -} - -#[test] -fn explicit_thinking_kwargs_override_the_default_without_cache_cross_contamination() { - let mut processor = checkpoint(THINKING_SCHEMA); - let messages = [message("user", "QUESTION")]; - let enabled = kwargs(json!({"enable_thinking": true})); - let disabled = kwargs(json!({"enable_thinking": false})); - assert_eq!(processor.apply(&messages, None).unwrap(), "TYPED:QUESTION"); - assert_eq!( - processor - .apply_with_kwargs(&messages, None, &enabled) - .unwrap(), - "STRING:QUESTION" - ); - processor.set_default_enable_thinking(true); - assert_eq!( - processor - .apply_with_kwargs(&messages, None, &disabled) - .unwrap(), - "TYPED:QUESTION" - ); - assert_eq!(processor.apply(&messages, None).unwrap(), "STRING:QUESTION"); - assert_eq!( - processor - .apply_with_kwargs(&messages, None, &enabled) - .unwrap(), - "STRING:QUESTION" - ); -} - -#[test] -fn generation_and_history_use_their_own_content_contract() { - let mut processor = checkpoint(concat!( - "{% for m in messages %}", - "{% if add_generation_prompt %}", - "{% for p in m.content | selectattr('type', 'equalto', 'text') %}{{ p.text }}{% endfor %}", - "{% else %}{{ m.content }}{% endif %}{% endfor %}", - "{% if add_generation_prompt %}|GENERATE{% endif %}", - )); - let messages = [message("user", "QUESTION")]; - let raw = json!(messages); - assert_eq!( - processor.apply(&messages, None).unwrap(), - "QUESTION|GENERATE" - ); - assert_eq!( - processor - .apply_history_with_kwargs(&messages, None, &Default::default()) - .unwrap(), - "QUESTION" - ); - assert_eq!( - processor - .apply_raw_history_with_kwargs(&raw, None, &Default::default()) - .unwrap(), - "QUESTION" - ); - processor.set_add_generation_prompt(false); - assert_eq!(processor.apply(&messages, None).unwrap(), "QUESTION"); - processor.set_add_generation_prompt(true); - assert_eq!( - processor.apply_raw(&raw, None).unwrap(), - "QUESTION|GENERATE" - ); -} - -#[test] -fn preserve_thinking_kwargs_participate_in_content_policy() { - let processor = checkpoint(concat!( - "{% for m in messages %}{% if preserve_thinking | default(false) %}", - "KEPT:{{ m.content }}{% else %}STRIPPED:", - "{% for p in m.content | selectattr('type', 'equalto', 'text') %}{{ p.text }}{% endfor %}", - "{% endif %}{% endfor %}", - )); - let messages = [message("user", "QUESTION")]; - let kept = kwargs(json!({"preserve_thinking": true})); - let stripped = kwargs(json!({"preserve_thinking": false})); - assert_eq!( - processor.apply(&messages, None).unwrap(), - "STRIPPED:QUESTION" - ); - assert_eq!( - processor.apply_with_kwargs(&messages, None, &kept).unwrap(), - "KEPT:QUESTION" - ); - assert_eq!( - processor - .apply_with_kwargs(&messages, None, &stripped) - .unwrap(), - "STRIPPED:QUESTION" - ); -} - -#[test] -fn arbitrary_nested_kwargs_participate_in_content_policy() { - let processor = checkpoint(concat!( - "{% set mode = render_mode | default({'schema': 'typed'}) %}", - "{% for m in messages %}{% if mode.schema == 'string' %}", - "STRING:{{ m.content }}{% else %}TYPED:", - "{% for p in m.content | selectattr('type', 'equalto', 'text') %}{{ p.text }}{% endfor %}", - "{% endif %}{% endfor %}", - )); - let messages = [message("user", "QUESTION")]; - let string = kwargs(json!({"render_mode": {"schema": "string", "revision": 1}})); - let typed = kwargs(json!({"render_mode": {"schema": "typed", "revision": 1}})); - assert_eq!(processor.apply(&messages, None).unwrap(), "TYPED:QUESTION"); - assert_eq!( - processor - .apply_with_kwargs(&messages, None, &string) - .unwrap(), - "STRING:QUESTION" - ); - assert_eq!( - processor - .apply_with_kwargs(&messages, None, &typed) - .unwrap(), - "TYPED:QUESTION" - ); - assert_eq!( - processor - .apply_raw_with_kwargs(&json!(messages), None, &string) - .unwrap(), - "STRING:QUESTION" - ); -} - -#[test] -fn tool_definitions_not_just_tool_presence_participate_in_content_policy() { - let processor = checkpoint(concat!( - "{% for m in messages %}", - "{% if tools is defined and tools[0].function.name == 'strings' %}", - "STRING:{{ m.content }}{% else %}TYPED:", - "{% for p in m.content | selectattr('type', 'equalto', 'text') %}{{ p.text }}{% endfor %}", - "{% endif %}{% endfor %}", - )); - let make_tool = |name: &str| -> Tool { - serde_json::from_value(json!({ - "type": "function", - "function": {"name": name, "parameters": {"type": "object"}}, - })) - .unwrap() - }; - let strings = [make_tool("strings")]; - let typed = [make_tool("typed")]; - let messages = [message("user", "QUESTION")]; - assert_eq!(processor.apply(&messages, None).unwrap(), "TYPED:QUESTION"); - assert_eq!( - processor.apply(&messages, Some(&strings)).unwrap(), - "STRING:QUESTION" - ); - assert_eq!( - processor.apply(&messages, Some(&typed)).unwrap(), - "TYPED:QUESTION" - ); - assert_eq!( - processor.apply(&messages, Some(&[])).unwrap(), - "TYPED:QUESTION" - ); - assert_eq!( - processor.apply(&messages, Some(&strings)).unwrap(), - "STRING:QUESTION" - ); -} - -#[test] -fn cloned_processors_share_cache_without_sharing_effective_thinking_defaults() { - let processor = checkpoint(THINKING_SCHEMA); - let mut thinking = processor.clone(); - thinking.set_default_enable_thinking(true); - let messages = [message("user", "QUESTION")]; - for _ in 0..3 { - assert_eq!(processor.apply(&messages, None).unwrap(), "TYPED:QUESTION"); - assert_eq!(thinking.apply(&messages, None).unwrap(), "STRING:QUESTION"); - } -} - -#[test] -fn many_distinct_contexts_and_revisited_entries_keep_their_content_contract() { - let processor = checkpoint(THINKING_SCHEMA); - let messages = [message("user", "QUESTION")]; - // More than the bounded decision cache can retain: evicting a decision - // must only cause another probe, never change rendering semantics. - for nonce in 0..48 { - let enabled = nonce % 2 == 0; - let context = kwargs(json!({ - "enable_thinking": enabled, - "cache_test_nonce": nonce, - })); - let expected = if enabled { - "STRING:QUESTION" - } else { - "TYPED:QUESTION" - }; - assert_eq!( - processor - .apply_with_kwargs(&messages, None, &context) - .unwrap(), - expected - ); - } - for nonce in [0, 1, 47, 0, 46, 1] { - let enabled = nonce % 2 == 0; - let context = kwargs(json!({ - "enable_thinking": enabled, - "cache_test_nonce": nonce, - })); - let expected = if enabled { - "STRING:QUESTION" - } else { - "TYPED:QUESTION" - }; - assert_eq!( - processor - .apply_with_kwargs(&messages, None, &context) - .unwrap(), - expected - ); - } -} - -#[test] -fn oversized_context_keys_still_render_correctly_between_cached_contexts() { - let processor = checkpoint(THINKING_SCHEMA); - let messages = [message("user", "QUESTION")]; - let padding = "context-only-padding-".repeat(2048); - for (oversized, enabled) in [ - (false, false), - (true, true), - (true, false), - (false, true), - (true, true), - (false, false), - ] { - let context = kwargs(json!({ - "enable_thinking": enabled, - "cache_test_padding": if oversized { padding.as_str() } else { "" }, - })); - let expected = if enabled { - "STRING:QUESTION" - } else { - "TYPED:QUESTION" - }; - assert_eq!( - processor - .apply_with_kwargs(&messages, None, &context) - .unwrap(), - expected - ); - } -} - -#[test] -fn concurrent_clones_render_different_contexts_without_cache_cross_contamination() { - let processor = checkpoint(THINKING_SCHEMA); - let barrier = std::sync::Arc::new(std::sync::Barrier::new(6)); - std::thread::scope(|scope| { - let handles: Vec<_> = (0..6) - .map(|worker| { - let processor = processor.clone(); - let barrier = std::sync::Arc::clone(&barrier); - scope.spawn(move || { - let messages = [message("user", "QUESTION")]; - let raw = json!(messages); - barrier.wait(); - for turn in 0..32 { - let enabled = (worker + turn) % 2 == 0; - let context = kwargs(json!({ - "enable_thinking": enabled, - "cache_test_partition": worker % 3, - })); - let expected = if enabled { - "STRING:QUESTION" - } else { - "TYPED:QUESTION" - }; - let actual = if turn % 2 == 0 { - processor.apply_with_kwargs(&messages, None, &context) - } else { - processor.apply_raw_with_kwargs(&raw, None, &context) - } - .unwrap(); - assert_eq!(actual, expected, "worker={worker}, turn={turn}"); - } - }) - }) - .collect(); - for handle in handles { - handle.join().expect("concurrent renderer must not panic"); - } - }); -} - -#[test] -fn tool_call_metadata_schema_does_not_reuse_plain_role_decisions() { - let processor = checkpoint(concat!( - "{% for m in messages %}{{ m.role }}:", - "{% if m.role == 'assistant' and m.tool_calls is defined ", - "and m.tool_calls | length > 0 and m.tool_calls[0].function.name == 'strings' %}", - "{{ m.content }}{% else %}", - "{% for p in m.content | selectattr('type', 'equalto', 'text') %}{{ p.text }}{% endfor %}", - "{% endif %}|{% endfor %}{% if add_generation_prompt %}GENERATE{% endif %}", - )); - let plain = json!([ - {"role": "user", "content": "QUESTION"}, - {"role": "assistant", "content": "ANSWER"}, - {"role": "user", "content": "FOLLOWUP"}, - ]); - let adapted = "user:QUESTION|assistant:ANSWER|user:FOLLOWUP|"; - let original_mixed = "user:|assistant:ANSWER|user:|"; - let call = |name: &str| { - json!([{ - "id": "call-1", "type": "function", - "function": {"name": name, "arguments": {"city": "Seoul"}}, - }]) - }; - // Prime a plain-role decision, then alternate values of the same metadata - // field. Guarding presence alone, or caching only context+roles, is unsafe. - for calls in [ - None, - Some(json!([])), - Some(call("strings")), - Some(call("typed")), - None, - Some(call("strings")), - ] { - let mut raw = plain.clone(); - let expected = if let Some(calls) = calls { - let strings = calls - .get(0) - .and_then(|v| v.pointer("/function/name")) - .is_some_and(|name| name == "strings"); - raw[1]["tool_calls"] = calls; - if strings { original_mixed } else { adapted } - } else { - adapted - }; - assert_eq!( - processor.apply_raw(&raw, None).unwrap(), - format!("{expected}GENERATE") - ); - assert_eq!( - processor - .apply_raw_history_with_kwargs(&raw, None, &Default::default()) - .unwrap(), - expected - ); - } -} - -#[test] -fn reasoning_metadata_schema_is_checked_without_retaining_a_stale_role_policy() { - let processor = checkpoint(concat!( - "{% for m in messages %}{{ m.role }}:", - "{% if m.role == 'assistant' and m.reasoning_content | default('') == 'STRING_MODE' %}", - "{{ m.content }}{% else %}", - "{% for p in m.content | selectattr('type', 'equalto', 'text') %}{{ p.text }}{% endfor %}", - "{% endif %}|{% endfor %}", - )); - for reasoning in ["TYPED_MODE", "STRING_MODE", "TYPED_MODE", "STRING_MODE"] { - let raw = json!([ - {"role": "user", "content": "QUESTION"}, - {"role": "assistant", "content": "ANSWER", "reasoning_content": reasoning}, - {"role": "user", "content": "FOLLOWUP"}, - ]); - let expected = if reasoning == "STRING_MODE" { - "user:|assistant:ANSWER|user:|" - } else { - "user:QUESTION|assistant:ANSWER|user:FOLLOWUP|" - }; - assert_eq!(processor.apply_raw(&raw, None).unwrap(), expected); - } -} - -#[test] -fn string_user_metadata_also_participates_in_the_safety_guard() { - let processor = checkpoint(concat!( - "{% for m in messages %}", - "{% if m.name | default('') == 'string-reader' %}{{ m.content }}", - "{% else %}{% for p in m.content | selectattr('type', 'equalto', 'text') %}", - "{{ p.text }}{% endfor %}{% endif %}{% endfor %}", - )); - for name in ["typed-reader", "string-reader", "typed-reader"] { - let raw = json!([{"role": "user", "content": "QUESTION", "name": name}]); - assert_eq!(processor.apply_raw(&raw, None).unwrap(), "QUESTION"); - } -} - -#[test] -fn explicit_assistant_rejection_is_not_bypassed_by_content_adaptation() { - let processor = checkpoint(concat!( - "{% for m in messages %}", - "{% if m.role == 'assistant' and m.content is string %}", - "{{ raise_exception('assistant string policy rejected') }}", - "{% endif %}", - "{% for p in m.content | selectattr('type', 'equalto', 'text') %}", - "{{ p.text }}{% endfor %}{% endfor %}", - )); - // A successful user-only decision must not override a later role's - // explicit policy rejection, even when typed assistant content would render. - assert_eq!( - processor - .apply(&[message("user", "QUESTION")], None) - .unwrap(), - "QUESTION" - ); - let conversation = [ - message("user", "QUESTION"), - message("assistant", "ANSWER"), - message("user", "FOLLOWUP"), - ]; - let raw = json!(conversation); - let typed = json!([ - {"role": "user", "content": [{"type": "text", "text": "QUESTION"}]}, - {"role": "assistant", "content": [{"type": "text", "text": "ANSWER"}]}, - {"role": "user", "content": [{"type": "text", "text": "FOLLOWUP"}]}, - ]); - assert_eq!( - processor.apply_raw(&typed, None).unwrap(), - "QUESTIONANSWERFOLLOWUP" - ); - for result in [ - processor.apply(&conversation, None), - processor.apply_history_with_kwargs(&conversation, None, &Default::default()), - processor.apply_raw(&raw, None), - processor.apply_raw_history_with_kwargs(&raw, None, &Default::default()), - ] { - let error = result.expect_err("explicit template policy must remain an error"); - assert_eq!( - super::template_rejection_message(&error), - Some("assistant string policy rejected") - ); - } -} diff --git a/src/server/chat_template_raw_content_tests.rs b/src/server/chat_template_raw_content_tests.rs deleted file mode 100644 index 5a8ba2ee8..000000000 --- a/src/server/chat_template_raw_content_tests.rs +++ /dev/null @@ -1,241 +0,0 @@ -// Copyright 2025-2026 Lablup Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -//! Raw-message content adaptation without losing non-content fields. -//! - -use std::path::PathBuf; - -use serde_json::{Value, json}; - -use super::{ChatMessage, ChatTemplateProcessor}; - -const JSON_OBSERVER: &str = concat!( - "{% for m in messages %}", - "{% if m.content is not string %}{{ m | tojson }}{{ '\\n' }}{% endif %}", - "{% endfor %}", - "{% if add_generation_prompt %}GENERATE{% endif %}", -); - -fn checkpoint(template: &str) -> ChatTemplateProcessor { - let dir = tempfile::tempdir().unwrap(); - std::fs::write( - dir.path().join("tokenizer_config.json"), - json!({"chat_template": template}).to_string(), - ) - .unwrap(); - ChatTemplateProcessor::from_model_path(dir.path()) - .unwrap() - .unwrap() -} - -fn granite() -> ChatTemplateProcessor { - ChatTemplateProcessor::from_model_path( - &PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("tests/fixtures/granite_vision_3_2"), - ) - .unwrap() - .unwrap() -} - -fn observed_messages(rendered: &str) -> Vec { - rendered - .strip_suffix("GENERATE") - .unwrap_or(rendered) - .lines() - .map(|line| serde_json::from_str(line).expect("template emits one JSON message per line")) - .collect() -} - -#[test] -fn granite_raw_mixed_image_history_preserves_all_string_roles() { - let processor = granite(); - let raw = json!([ - {"role": "system", "content": "SYSTEM"}, - {"role": "user", "content": [ - {"type": "text", "text": "IMAGE_QUESTION"}, - {"type": "image"}, - ]}, - {"role": "assistant", "content": "ANSWER"}, - {"role": "user", "content": "FOLLOWUP"}, - ]); - let original = raw.clone(); - let history = concat!( - "<|system|>\nSYSTEM\n", - "<|user|>\n\nIMAGE_QUESTION\n", - "<|assistant|>\nANSWER<|end_of_text|>", - "<|user|>\nFOLLOWUP\n", - ); - assert_eq!( - processor.apply_raw(&raw, None).unwrap(), - format!("{history}<|assistant|>\n") - ); - assert_eq!( - processor - .apply_raw_history_with_kwargs(&raw, None, &Default::default()) - .unwrap(), - history - ); - assert_eq!( - raw, original, - "normalization must not mutate caller-owned JSON" - ); -} - -#[test] -fn raw_and_ordinary_text_share_generation_and_history_representation() { - let processor = granite(); - let messages = [ - ChatMessage { - role: "system".into(), - content: "SYSTEM".into(), - }, - ChatMessage { - role: "user".into(), - content: "QUESTION".into(), - }, - ChatMessage { - role: "assistant".into(), - content: "ANSWER".into(), - }, - ChatMessage { - role: "user".into(), - content: "FOLLOWUP".into(), - }, - ]; - let raw = serde_json::to_value(&messages).unwrap(); - assert_eq!( - processor.apply_raw(&raw, None).unwrap(), - processor.apply(&messages, None).unwrap() - ); - assert_eq!( - processor - .apply_raw_history_with_kwargs(&raw, None, &Default::default()) - .unwrap(), - processor - .apply_history_with_kwargs(&messages, None, &Default::default()) - .unwrap() - ); -} - -#[test] -fn raw_string_conversion_preserves_tool_reasoning_and_extension_metadata() { - let processor = checkpoint(JSON_OBSERVER); - let raw = json!([ - { - "role": "assistant", - "content": "ANSWER", - "name": "speaker", - "reasoning": "TRACE", - "reasoning_content": "TRACE", - "tool_calls": [{ - "id": "call-1", - "type": "function", - "function": {"name": "weather", "arguments": {"city": "Seoul"}}, - }], - "extension": {"nested": [null, false, 17, "kept"]}, - }, - { - "role": "tool", - "content": "", - "tool_call_id": "call-1", - "name": "weather", - }, - ]); - let original = raw.clone(); - let rows = observed_messages(&processor.apply_raw(&raw, None).unwrap()); - assert_eq!(rows.len(), 2); - let mut expected_assistant = raw[0].clone(); - expected_assistant["content"] = json!([{"type": "text", "text": "ANSWER"}]); - let mut expected_tool = raw[1].clone(); - expected_tool["content"] = json!([{"type": "text", "text": ""}]); - assert_eq!(rows, [expected_assistant, expected_tool]); - assert_eq!(raw, original); -} - -#[test] -fn raw_nonstring_content_is_not_flattened_coerced_or_reordered() { - let processor = checkpoint(JSON_OBSERVER); - let raw = json!([ - {"role": "assistant", "content": null, "tool_calls": []}, - {"role": "assistant", "name": "missing-content"}, - {"role": "user", "content": [ - {"type": "text", "text": "before"}, - {"type": "image", "extension": {"position": 1}}, - {"type": "text", "text": "after"}, - ]}, - {"role": "assistant", "content": []}, - {"role": "assistant", "content": false}, - {"role": "assistant", "content": 17}, - ]); - let generation = processor.apply_raw(&raw, None).unwrap(); - let history = processor - .apply_raw_history_with_kwargs(&raw, None, &Default::default()) - .unwrap(); - assert_eq!(observed_messages(&generation), *raw.as_array().unwrap()); - assert_eq!(observed_messages(&history), *raw.as_array().unwrap()); - assert_eq!(generation, format!("{history}GENERATE")); -} - -#[test] -fn raw_normalization_remains_disabled_for_string_templates_and_explicit_overrides() { - let raw = json!([{"role": "user", "content": "QUESTION", "name": "speaker"}]); - let string_compatible = checkpoint("{{ messages | tojson }}"); - assert_eq!( - serde_json::from_str::(&string_compatible.apply_raw(&raw, None).unwrap()).unwrap(), - raw - ); - - // Operator-supplied templates keep their explicit content contract. - let explicit = ChatTemplateProcessor::with_template(JSON_OBSERVER.into()); - assert_eq!(explicit.apply_raw(&raw, None).unwrap(), "GENERATE"); - assert_eq!( - explicit - .apply_raw_history_with_kwargs(&raw, None, &Default::default()) - .unwrap(), - "" - ); -} - -#[test] -fn mixed_raw_normalization_wraps_only_strings_and_preserves_other_values() { - let processor = checkpoint(JSON_OBSERVER); - let raw = json!([ - {"role": "assistant", "content": null, "tool_calls": []}, - {"role": "assistant", "name": "missing-content"}, - {"role": "user", "content": [ - {"type": "text", "text": "before"}, - {"type": "image", "extension": {"position": 1}}, - {"type": "text", "text": "after"}, - ]}, - {"role": "assistant", "content": []}, - {"role": "assistant", "content": false}, - {"role": "assistant", "content": 17}, - {"role": "user", "content": "FOLLOWUP"}, - ]); - let original = raw.clone(); - let mut expected = raw.clone(); - expected[6]["content"] = json!([{"type": "text", "text": "FOLLOWUP"}]); - - let generation = processor.apply_raw(&raw, None).unwrap(); - let history = processor - .apply_raw_history_with_kwargs(&raw, None, &Default::default()) - .unwrap(); - assert_eq!( - observed_messages(&generation), - *expected.as_array().unwrap() - ); - assert_eq!(observed_messages(&history), *expected.as_array().unwrap()); - assert_eq!(generation, format!("{history}GENERATE")); - assert_eq!(raw, original, "caller-owned JSON must remain unchanged"); -} diff --git a/src/server/chat_template_text_wrap_tests.rs b/src/server/chat_template_text_wrap_tests.rs new file mode 100644 index 000000000..ec1435924 --- /dev/null +++ b/src/server/chat_template_text_wrap_tests.rs @@ -0,0 +1,196 @@ +// Copyright 2025-2026 Lablup Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! Checkpoint-selected templates that require typed text content (#2135). + +use std::path::PathBuf; + +use serde_json::json; + +use super::{ChatMessage, ChatTemplateProcessor}; + +const QUESTION: &str = "What is the capital of France?"; +const GRANITE_SYSTEM: &str = "<|system|>\nA chat between a curious user and an artificial intelligence assistant. The assistant gives helpful, detailed, and polite answers to the user's questions.\n"; +const GRANITE_3_CONFIG: &str = + include_str!("../../tests/fixtures/granite_vision_3_2/tokenizer_config.json"); + +fn message(role: &str, content: &str) -> ChatMessage { + ChatMessage { + role: role.into(), + content: content.into(), + } +} + +fn granite_3() -> ChatTemplateProcessor { + let path = PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("tests/fixtures/granite_vision_3_2"); + ChatTemplateProcessor::from_model_path(&path) + .expect("fixture must load") + .expect("fixture must contain a template") +} + +fn checkpoint(template: &str) -> ChatTemplateProcessor { + let directory = tempfile::tempdir().expect("tempdir"); + std::fs::write( + directory.path().join("tokenizer_config.json"), + json!({"chat_template": template}).to_string(), + ) + .expect("write tokenizer config"); + ChatTemplateProcessor::from_model_path(directory.path()) + .expect("checkpoint must load") + .expect("checkpoint must contain a template") +} + +#[test] +fn granite_3_from_model_path_preserves_the_exact_user_prompt() { + let processor = granite_3(); + assert!(processor.wrap_string_content); + assert_eq!( + processor + .apply(&[message("user", QUESTION)], None) + .expect("render"), + format!("{GRANITE_SYSTEM}<|user|>\n{QUESTION}\n<|assistant|>\n") + ); +} + +#[test] +fn granite_3_wraps_system_assistant_and_empty_content() { + let processor = granite_3(); + let history = [ + message("system", "Answer concisely."), + message("user", QUESTION), + message("assistant", "Paris."), + message("user", ""), + ]; + let expected = concat!( + "<|system|>\nAnswer concisely.\n", + "<|user|>\nWhat is the capital of France?\n", + "<|assistant|>\nParis.<|end_of_text|>", + "<|user|>\n\n" + ); + assert_eq!( + processor + .apply_history_with_kwargs(&history, None, &Default::default()) + .expect("history render"), + expected + ); + assert_eq!( + processor.apply(&history, None).expect("generation render"), + format!("{expected}<|assistant|>\n") + ); +} + +#[test] +fn granite_3_raw_rendering_keeps_its_original_contract() { + let processor = granite_3(); + let typed = json!([{ + "role": "user", + "content": [{"type": "image"}, {"type": "text", "text": QUESTION}], + }]); + assert_eq!( + processor.apply_raw(&typed, None).expect("typed render"), + format!("{GRANITE_SYSTEM}<|user|>\n\n{QUESTION}\n<|assistant|>\n") + ); + + let raw_string = json!([{"role": "user", "content": QUESTION}]); + let rendered = processor + .apply_raw(&raw_string, None) + .expect("raw string render"); + assert!(!rendered.contains(QUESTION)); +} + +#[test] +fn string_compatible_templates_keep_string_content() { + let processor = checkpoint( + "{% for m in messages %}{% if m.content is string %}string:{{ m.content }}{% else %}typed:{{ m.content[0].text }}{% endif %}{% endfor %}", + ); + assert!(!processor.wrap_string_content); + assert_eq!( + processor + .apply(&[message("user", QUESTION)], None) + .expect("render"), + format!("string:{QUESTION}") + ); +} + +#[test] +fn granite_3_config_only_string_template_is_not_wrapped() { + let directory = tempfile::tempdir().expect("tempdir"); + std::fs::write( + directory.path().join("tokenizer_config.json"), + GRANITE_3_CONFIG, + ) + .expect("write tokenizer config"); + let processor = ChatTemplateProcessor::from_model_path(directory.path()) + .expect("checkpoint must load") + .expect("checkpoint must contain a template"); + assert!(!processor.wrap_string_content); + assert!( + processor + .apply(&[message("user", QUESTION)], None) + .expect("render") + .contains(QUESTION) + ); +} + +#[test] +fn granite_4_accepts_both_shapes_without_wrapping() { + let path = PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("tests/fixtures/granite_4_vision"); + let processor = ChatTemplateProcessor::from_model_path(&path) + .expect("fixture must load") + .expect("fixture must contain a template"); + assert!(!processor.wrap_string_content); + assert!( + processor + .apply(&[message("user", QUESTION)], None) + .expect("string render") + .contains(QUESTION) + ); + assert!( + processor + .apply_raw( + &json!([{ + "role": "user", + "content": [{"type": "text", "text": QUESTION}], + }]), + None, + ) + .expect("typed render") + .contains(QUESTION) + ); +} + +#[test] +fn explicit_template_constructors_do_not_enable_wrapping() { + let template = include_str!("../../tests/fixtures/granite_vision_3_2/chat_template.jinja"); + let processor = ChatTemplateProcessor::with_template(template.into()); + assert!(!processor.wrap_string_content); + assert!( + !processor + .apply(&[message("user", QUESTION)], None) + .expect("render") + .contains(QUESTION) + ); + assert!(!ChatTemplateProcessor::default().wrap_string_content); +} + +#[test] +fn probe_render_errors_leave_wrapping_disabled() { + let processor = checkpoint(concat!( + "{% if messages[0].content is string %}", + "{{ raise_exception('string rejected') }}", + "{% else %}{{ messages[0].content[0].text }}{% endif %}" + )); + assert!(!processor.wrap_string_content); + assert!(processor.apply(&[message("user", QUESTION)], None).is_err()); +}