From ae24752257ac1d3108e0793d99cf62d5b2875761 Mon Sep 17 00:00:00 2001 From: hallerite Date: Sun, 11 Oct 2026 13:19:04 +0200 Subject: [PATCH 1/2] fix: align tagged reasoning parsing with model templates and vLLM --- renderers/deepseek_v4.py | 30 ++--- renderers/glm45.py | 23 +--- renderers/glm5.py | 21 +-- renderers/parsing.py | 146 ++++++++++---------- renderers/reasoning.py | 83 +++++++++++- tests/test_deepseek_v4.py | 38 +++++- tests/test_parse_response.py | 5 +- tests/test_parse_response_robustness.py | 9 +- tests/test_reasoning_boundaries.py | 170 ++++++++++++++++++++++-- 9 files changed, 368 insertions(+), 157 deletions(-) diff --git a/renderers/deepseek_v4.py b/renderers/deepseek_v4.py index ae9315702..0743a2fa5 100644 --- a/renderers/deepseek_v4.py +++ b/renderers/deepseek_v4.py @@ -22,8 +22,6 @@ from dataclasses import dataclass, field from typing import Any, Mapping -from renderers.reasoning import scan_reasoning, prompt_ends_in_reasoning - from renderers.base import ( Message, ParsedResponse, @@ -41,7 +39,7 @@ ) from renderers.configs import DeepSeekV4RendererConfig from renderers.parsing import parse_deepseek_v4 - +from renderers.reasoning import prompt_ends_in_reasoning _BOS = "<｜begin▁of▁sentence｜>" _EOS = "<｜end▁of▁sentence｜>" @@ -798,11 +796,15 @@ def parse_response( self._tokenizer, token_ids, stop_ids={self._eos}, - thinking_enabled=prompt_ends_in_reasoning( - self._tokenizer, - prompt_ids, - stop_ids=set(self.get_stop_token_ids()), - assistant_prefix="<｜Assistant｜>", + thinking_enabled=( + prompt_ends_in_reasoning( + self._tokenizer, + prompt_ids, + stop_ids=set(self.get_stop_token_ids()), + assistant_prefix="<｜Assistant｜>", + ) + if prompt_ids + else self.config.enable_thinking ), think_end_id=self._think_end, think_start_id=self._think_start, @@ -827,16 +829,10 @@ def bridge_to_next_turn( ): return None - boundary = scan_reasoning( - self._tokenizer, - previous_completion_ids, - prompt_ids=previous_prompt_ids, - stop_ids=set(self.get_stop_token_ids()), - tool_start_id=self._dsml, - tool_start_closes_reasoning=True, - assistant_prefix="<｜Assistant｜>", + boundary = self.parse_response( + previous_completion_ids, prompt_ids=previous_prompt_ids, tools=tools ) - if boundary.is_open: + if boundary.reasoning_complete is False: if any(t in self.get_stop_token_ids() for t in previous_completion_ids): return None previous_completion_ids = [*previous_completion_ids, self._think_end] diff --git a/renderers/glm45.py b/renderers/glm45.py index 8dd33555e..cb967552d 100644 --- a/renderers/glm45.py +++ b/renderers/glm45.py @@ -13,14 +13,12 @@ import json from typing import Any -from renderers.reasoning import scan_reasoning, prompt_ends_in_reasoning - from renderers.base import ( Message, ParsedResponse, RenderedTokens, - ToolSpec, Tokenizer, + ToolSpec, _content_mask_or_empty, attribute_text_segments, extract_message_tool_names, @@ -296,12 +294,7 @@ def parse_response( arg_value_id=self._arg_value, arg_value_end_id=self._arg_value_end, tools=tools, - prefilled_thinking=prompt_ends_in_reasoning( - self._tokenizer, - prompt_ids, - stop_ids=set(self.get_stop_token_ids()), - assistant_prefix="<|assistant|>", - ), + thinking_enabled=self.config.enable_thinking, ) def get_stop_token_ids(self) -> list[int]: @@ -322,16 +315,10 @@ def bridge_to_next_turn( ): return None - boundary = scan_reasoning( - self._tokenizer, - previous_completion_ids, - prompt_ids=previous_prompt_ids, - stop_ids=set(self.get_stop_token_ids()), - tool_start_id=self._tool_call_tok, - tool_start_closes_reasoning=True, - assistant_prefix="<|assistant|>", + boundary = self.parse_response( + previous_completion_ids, prompt_ids=previous_prompt_ids, tools=tools ) - if boundary.is_open: + if boundary.reasoning_complete is False: if any(t in self.get_stop_token_ids() for t in previous_completion_ids): return None previous_completion_ids = [*previous_completion_ids, self._think_end] diff --git a/renderers/glm5.py b/renderers/glm5.py index 5a7eac50c..b8d61cdc4 100644 --- a/renderers/glm5.py +++ b/renderers/glm5.py @@ -14,8 +14,6 @@ import json from typing import Any -from renderers.reasoning import scan_reasoning, prompt_ends_in_reasoning - from renderers.base import ( Message, ParsedResponse, @@ -351,12 +349,7 @@ def parse_response( arg_value_id=self._arg_value, arg_value_end_id=self._arg_value_end, tools=tools, - prefilled_thinking=prompt_ends_in_reasoning( - self._tokenizer, - prompt_ids, - stop_ids=set(self.get_stop_token_ids()), - assistant_prefix="<|assistant|>", - ), + thinking_enabled=getattr(self.config, "enable_thinking", True), ) def get_stop_token_ids(self) -> list[int]: @@ -377,16 +370,10 @@ def bridge_to_next_turn( ): return None - boundary = scan_reasoning( - self._tokenizer, - previous_completion_ids, - prompt_ids=previous_prompt_ids, - stop_ids=set(self.get_stop_token_ids()), - tool_start_id=self._tool_call_tok, - tool_start_closes_reasoning=True, - assistant_prefix="<|assistant|>", + boundary = self.parse_response( + previous_completion_ids, prompt_ids=previous_prompt_ids, tools=tools ) - if boundary.is_open: + if boundary.reasoning_complete is False: if any(t in self.get_stop_token_ids() for t in previous_completion_ids): return None previous_completion_ids = [*previous_completion_ids, self._think_end] diff --git a/renderers/parsing.py b/renderers/parsing.py index 6bc27624f..0ad0967ed 100644 --- a/renderers/parsing.py +++ b/renderers/parsing.py @@ -17,8 +17,6 @@ from __future__ import annotations import json - -from renderers.reasoning import scan_reasoning from typing import Any from renderers.base import ( @@ -27,7 +25,7 @@ ToolCallParseStatus, ToolSpec, ) - +from renderers.reasoning import scan_reasoning, split_tagged_reasoning # ── Schema-aware argument coercion ────────────────────────────────── # @@ -486,6 +484,7 @@ def parse_glm( arg_value_end_id: int, tools: list[ToolSpec] | None = None, prefilled_thinking: bool = False, + thinking_enabled: bool | None = None, ) -> ParsedResponse: """Parse GLM completion tokens. Token-level thinking + arg_key/arg_value tool calls. @@ -505,36 +504,23 @@ def parse_glm( behaves the same when the request carries no tools). """ ids = _strip_stop_tokens(token_ids, stop_ids) - boundary = scan_reasoning( - tokenizer, - ids, - prefilled=prefilled_thinking, - open_id=think_id, - close_id=think_end_id, - tool_start_id=tool_call_id, - tool_start_closes_reasoning=True, - assistant_prefix="<|assistant|>", + split = split_tagged_reasoning( + _decode(tokenizer, ids), + thinking=prefilled_thinking if thinking_enabled is None else thinking_enabled, + enabled=thinking_enabled is not False, + tool_markers=("",), ) + boundary = split.boundary + reasoning = boundary.text.rstrip() if boundary.text is not None else None if boundary.is_open: return ParsedResponse( - content="", reasoning_content=boundary.text, reasoning_complete=False + content=split.content, + reasoning_content=reasoning, + reasoning_complete=False, ) - - reasoning = None - parse_offset = 0 - think_end = _find(ids, think_end_id) - if boundary.closed_by_tool: - reasoning = boundary.text.strip() - parse_offset = _find(ids, tool_call_id) - elif boundary.text is not None and think_end != -1: - reasoning = boundary.text.strip() - parse_offset = think_end + 1 - ids = ids[parse_offset:] - - tc_start = _find(ids, tool_call_id) + tc_start = _find(ids, tool_call_id) if split.tool_start is not None else -1 tool_calls: list[ParsedToolCall] = [] if tc_start != -1: - content_text = _decode(tokenizer, ids[:tc_start]).strip() tool_calls = _parse_glm_tool_calls( tokenizer, ids[tc_start:], @@ -544,12 +530,48 @@ def parse_glm( arg_key_end_id, arg_value_id, arg_value_end_id, - section_offset=parse_offset + tc_start, + section_offset=tc_start, param_index=_build_param_type_index(tools), known_names=_extract_tool_names(tools), ) + # Continue the reasoning/content grammar between complete tool calls, + # without interpreting delimiter-looking text inside their arguments. + content_parts = [split.content] + reasoning_parts = [boundary.text or ""] + cursor = tc_start + for call in tool_calls: + assert call.token_span is not None + start, end = call.token_span + between = split_tagged_reasoning( + _decode(tokenizer, ids[cursor:start]), + thinking=False, + enabled=thinking_enabled is not False, + ) + content_parts.append(between.content) + reasoning_parts.append(between.boundary.text or "") + cursor = end + # A truncated tool body still owns the remaining bytes. + tail = ( + "" + if tool_calls + and tool_calls[-1].status == ToolCallParseStatus.UNCLOSED_BLOCK + else _decode(tokenizer, ids[cursor:]) + ) + after = split_tagged_reasoning( + tail, + thinking=False, + enabled=thinking_enabled is not False, + ) + content_parts.append(after.content) + reasoning_parts.append(after.boundary.text or "") + return ParsedResponse( + content="".join(content_parts).strip(), + reasoning_content="".join(reasoning_parts).rstrip() or None, + tool_calls=tool_calls, + reasoning_complete=not after.boundary.is_open, + ) else: - content_text = _decode(tokenizer, ids).strip() + content_text = split.content return ParsedResponse( content=content_text, @@ -1244,57 +1266,33 @@ def parse_deepseek_v4( reasoning opened in the completion is recognized independently. """ ids = _strip_stop_tokens(token_ids, stop_ids) - boundary = scan_reasoning( - tokenizer, - ids, - prefilled=thinking_enabled, - open_id=think_start_id, - close_id=think_end_id, - tool_start_id=dsml_id, - tool_start_closes_reasoning=True, - assistant_prefix="<｜Assistant｜>", + split = split_tagged_reasoning( + _decode(tokenizer, ids), + thinking=thinking_enabled, + tool_markers=("<｜DSML｜tool_calls>", "<｜DSML｜toolcalls>", "<｜DSML｜tool>"), ) + boundary = split.boundary + reasoning = boundary.text.rstrip() if boundary.text is not None else None if boundary.is_open: return ParsedResponse( - content="", reasoning_content=boundary.text, reasoning_complete=False + content=split.content, + reasoning_content=reasoning, + reasoning_complete=False, ) - - reasoning: str | None = None - content_offset = 0 - think_end = _find(ids, think_end_id) - if boundary.closed_by_tool: - content_offset = int(bool(ids) and ids[0] == think_start_id) - elif boundary.text is not None and think_end != -1: - reasoning = boundary.text - content_offset = think_end + 1 - - content_ids = ids[content_offset:] - decoded = _decode(tokenizer, content_ids) - if boundary.closed_by_tool: - # Reasoning ran into DSML markup without . The section opener - # is text around the atomic ｜DSML｜ token, so split before its tag - # and the canonical blank line. - prefix = _decode(tokenizer, content_ids[: _find(content_ids, dsml_id)]) - section_pos = len(prefix.removesuffix("<")) - if decoded[:section_pos].endswith("\n\n"): - section_pos -= 2 - reasoning = decoded[:section_pos] - else: - section_pos = decoded.find("\n\n<｜DSML｜tool_calls>") - if section_pos == -1 or dsml_id not in content_ids: + if split.tool_start is None: return ParsedResponse( - content=decoded, + content=split.content, reasoning_content=reasoning or None, tool_calls=[], ) - - content = "" if boundary.closed_by_tool else decoded[:section_pos] - section_text = decoded[section_pos:] - section_token_offset = content_offset + _decoded_char_to_token_index( - tokenizer, - content_ids, - section_pos, + # Include a BPE token straddling the opener's character boundary so tool + # spans remain offsets into the original sampled completion. + section_token_offset = _decoded_char_to_token_index( + tokenizer, ids, split.tool_start ) + if len(_decode(tokenizer, ids[:section_token_offset])) > split.tool_start: + section_token_offset -= 1 + section_text = _decode(tokenizer, ids[section_token_offset:]) tool_calls = _parse_deepseek_v4_tool_calls( tokenizer, section_text, @@ -1302,7 +1300,7 @@ def parse_deepseek_v4( section_offset=section_token_offset, ) return ParsedResponse( - content=content, + content=split.content if split.content.strip() else "", reasoning_content=reasoning or None, tool_calls=tool_calls, ) @@ -1338,11 +1336,9 @@ def _parse_deepseek_v4_tool_calls( ) tool_calls: list[ParsedToolCall] = [] - outer_end = section_text.find(section_end) for invoke_match in re.finditer(r"<｜DSML｜invoke", section_text): start = invoke_match.start() - if outer_end != -1 and outer_end < start: - break + outer_end = section_text.find(section_end, start) close = section_text.find(invoke_end, start + len(invoke_start)) unclosed = close == -1 or (outer_end != -1 and outer_end < close) diff --git a/renderers/reasoning.py b/renderers/reasoning.py index 6f753ec2e..c4589ebe3 100644 --- a/renderers/reasoning.py +++ b/renderers/reasoning.py @@ -1,5 +1,6 @@ """Reasoning boundaries, independent of response parsing and renderer classes.""" +import re from dataclasses import dataclass @@ -16,6 +17,63 @@ class ReasoningBoundary: closed_by_tool: bool = False +@dataclass(frozen=True) +class TaggedReasoning: + """A decoded view of tagged reasoning; sampled tokens remain untouched.""" + + boundary: ReasoningBoundary + content: str + tool_start: int | None = None + + +def split_tagged_reasoning( + text: str, + *, + thinking: bool, + enabled: bool = True, + tool_markers: tuple[str, ...] = (), +) -> TaggedReasoning: + """Reasoning transitions from vLLM 0.31's glm47_moe/deepseek_v4 engines. + + Both grammars absorb duplicate delimiters, can reopen reasoning from + content, and transition to tools at their opener even before a later + ````. Tool bodies are handed to the tool parser unchanged. + GLM disables reasoning terminals when thinking is disabled; V4 retains + them and changes only the initial state. + """ + markers = (*(("", "") if enabled else ()), *tool_markers) + if not markers: + return TaggedReasoning(ReasoningBoundary(False, None), text) + pattern = "|".join(re.escape(marker) for marker in markers) + active = thinking and enabled + opened = active + reasoning: list[str] = [] + content: list[str] = [] + start = 0 + for match in re.finditer(pattern, text): + (reasoning if active else content).append(text[start : match.start()]) + marker = match.group() + if marker in tool_markers: + return TaggedReasoning( + ReasoningBoundary( + False, "".join(reasoning) if opened else None, active + ), + "".join(content), + match.start(), + ) + if marker == "": + active = True + opened = True + else: + active = False + start = match.end() + (reasoning if active else content).append(text[start:]) + return TaggedReasoning( + ReasoningBoundary(active, "".join(reasoning) if opened else None), + "".join(content), + ) + + def _decode(tokenizer, ids: list[int]) -> str: return tokenizer.decode(ids, skip_special_tokens=False) if ids else "" @@ -100,7 +158,7 @@ def scan_reasoning( ) -> ReasoningBoundary: """Read the initial reasoning region without interpreting final content. - Tagged formats recognize an opener only at the beginning of the assistant + Tagged formats recognize an opener after leading whitespace in the assistant body, or continue reasoning opened in the prompt. A first close permanently switches to content. Atomic delimiters match token IDs, not lookalike text. Explicit channel formats can opt into channel transitions instead. @@ -133,10 +191,11 @@ def scan_reasoning( # Reasoning-first grammars have one initial reasoning region. Once it # closes, later markers are content, not a new reasoning channel. if open_id is not None and close_id is not None: - explicit = bool(ids) and ids[0] == open_id + opening = next((i for i, token in enumerate(ids) if token == open_id), -1) + explicit = opening != -1 and not _decode(tokenizer, ids[:opening]).strip() if not prefilled and not explicit: return ReasoningBoundary(False, None) - start = int(explicit) + start = opening + 1 if explicit else 0 close = next((i for i in range(start, len(ids)) if ids[i] == close_id), -1) by_tool = False if close == -1 and tool_start_closes_reasoning: @@ -146,14 +205,22 @@ def scan_reasoning( by_tool = close != -1 return ReasoningBoundary( close == -1, - _decode(tokenizer, ids[start : close if close != -1 else len(ids)]), + _decode( + tokenizer, + [ + token + for token in ids[start : close if close != -1 else len(ids)] + if token != open_id + ], + ), by_tool, ) text = _decode(tokenizer, ids) - explicit = text.startswith(open_marker) + opening = text.find(open_marker) + explicit = opening != -1 and not text[:opening].strip() if not prefilled and not explicit: return ReasoningBoundary(False, None) - start = len(open_marker) if explicit else 0 + start = opening + len(open_marker) if explicit else 0 close = text.find(close_marker, start) by_tool = False if close == -1 and tool_start_closes_reasoning: @@ -162,7 +229,9 @@ def scan_reasoning( if tool_pos >= start: close, by_tool = tool_pos, True return ReasoningBoundary( - close == -1, text[start : close if close != -1 else len(text)], by_tool + close == -1, + text[start : close if close != -1 else len(text)].replace(open_marker, ""), + by_tool, ) events: list[tuple[int, int, bool | None]] diff --git a/tests/test_deepseek_v4.py b/tests/test_deepseek_v4.py index a74c77ef3..7abdde74b 100644 --- a/tests/test_deepseek_v4.py +++ b/tests/test_deepseek_v4.py @@ -6,7 +6,6 @@ import pytest from pydantic import TypeAdapter, ValidationError - from renderers import ( DeepSeekV4Renderer, DeepSeekV4RendererConfig, @@ -17,7 +16,6 @@ from renderers.base import MODEL_RENDERER_MAP, load_tokenizer from tests.reference_rendering import render_reference - MODEL = "deepseek-ai/DeepSeek-V4-Flash-0731" BOS = "<｜begin▁of▁sentence｜>" EOS = "<｜end▁of▁sentence｜>" @@ -62,6 +60,30 @@ def _decode(renderer, messages, **kwargs): ) +def test_repeated_tool_sections_keep_both_calls_and_drop_trailing_text(): + renderer = _renderer(enable_thinking=True) + section = ( + '<｜DSML｜tool_calls>\n<｜DSML｜invoke name="weather">\n\n' + "\n" + ) + sampled = _tokenizer().encode( + "Reasoning" + section + section + "Ignored", + add_special_tokens=False, + ) + parsed = renderer.parse_response(sampled) + assert parsed.reasoning_content == "Reasoning" + assert parsed.content == "" + assert parsed.reasoning_complete is True + assert [call.name for call in parsed.tool_calls] == ["weather", "weather"] + for call in parsed.tool_calls: + assert call.status == ToolCallParseStatus.OK + # The closing tag and newline can share a BPE token. + assert ( + _tokenizer().decode(sampled[slice(*call.token_span)]).rstrip("\n") + == call.raw + ) + + def test_registration_and_native_defaults(): tokenizer = _tokenizer() renderer = create_renderer(tokenizer) @@ -224,7 +246,7 @@ def test_dsml_roundtrip_preserves_string_and_json_argument_types(prefilled, enab ) assert parsed.reasoning_content == "Use weather." - assert parsed.content == "checking" + assert parsed.content == "checking\n\n" assert len(parsed.tool_calls) == 1 call = parsed.tool_calls[0] assert call.status == ToolCallParseStatus.OK @@ -440,8 +462,8 @@ def test_merged_followup_task_is_dropped_like_reference_encoder(): assert text.endswith(f"{ASSISTANT}") -@pytest.mark.parametrize("prefix", ["", "\n"]) -def test_dsml_parser_requires_reference_encoders_two_newlines(prefix): +@pytest.mark.parametrize("prefix", ["", "\n", "\n\n"]) +def test_dsml_parser_accepts_vllm_tool_opener_without_template_newlines(prefix): text = ( f"{prefix}<｜DSML｜tool_calls>\n" '<｜DSML｜invoke name="weather">\n\n\n' @@ -452,8 +474,10 @@ def test_dsml_parser_requires_reference_encoders_two_newlines(prefix): _tokenizer().encode(text, add_special_tokens=False) ) - assert parsed.content == text - assert parsed.tool_calls == [] + assert parsed.content == "" + assert [(call.name, call.arguments, call.status) for call in parsed.tool_calls] == [ + ("weather", {}, ToolCallParseStatus.OK) + ] def test_rendered_masks_keep_dsml_sampled_and_tool_wrappers_scaffolded(): diff --git a/tests/test_parse_response.py b/tests/test_parse_response.py index 63bd62c45..84e4adc27 100644 --- a/tests/test_parse_response.py +++ b/tests/test_parse_response.py @@ -7,7 +7,6 @@ from functools import lru_cache import pytest - from renderers import create_renderer from renderers.base import ToolCallParseStatus, load_tokenizer @@ -20,11 +19,11 @@ def _qwen3_vl(): def test_parse_simple_content(model_name, tokenizer, renderer): - """Plain content, no thinking.""" + """Bare text survives in the format's initial channel.""" text = "Hello there!" ids = tokenizer.encode(text, add_special_tokens=False) parsed = renderer.parse_response(ids, prompt_ids=[]) - assert "Hello" in parsed.content + assert "Hello" in parsed.content + (parsed.reasoning_content or "") def test_parse_thinking_and_content(model_name, tokenizer, renderer): diff --git a/tests/test_parse_response_robustness.py b/tests/test_parse_response_robustness.py index ff921ab0b..fd1b333dc 100644 --- a/tests/test_parse_response_robustness.py +++ b/tests/test_parse_response_robustness.py @@ -6,7 +6,6 @@ from renderers.base import ParsedResponse, ParsedToolCall - # ── Truncation ─────────────────────────────────────────────────────── @@ -73,11 +72,11 @@ def test_thinking_with_newlines(model_name, tokenizer, renderer): def test_content_only_no_thinking(model_name, tokenizer, renderer): - """Plain content with no thinking markers.""" + """Bare text follows the parser's initial state without losing text.""" text = "Hello! How can I help you today?" ids = tokenizer.encode(text, add_special_tokens=False) parsed = renderer.parse_response(ids, prompt_ids=[]) - assert "Hello" in parsed.content + assert "Hello" in parsed.content + (parsed.reasoning_content or "") assert parsed.tool_calls == [] @@ -98,7 +97,7 @@ def test_content_with_special_chars(model_name, tokenizer, renderer): text = 'The formula is x < y and a > b. Use "quotes" freely.' ids = tokenizer.encode(text, add_special_tokens=False) parsed = renderer.parse_response(ids, prompt_ids=[]) - assert parsed.content # Should not crash on angle brackets + assert parsed.content or parsed.reasoning_content def test_very_long_content(model_name, tokenizer, renderer): @@ -106,7 +105,7 @@ def test_very_long_content(model_name, tokenizer, renderer): text = "word " * 500 ids = tokenizer.encode(text, add_special_tokens=False) parsed = renderer.parse_response(ids, prompt_ids=[]) - assert len(parsed.content) > 100 + assert len(parsed.content + (parsed.reasoning_content or "")) > 100 # ── Return type guarantees ─────────────────────────────────────────── diff --git a/tests/test_reasoning_boundaries.py b/tests/test_reasoning_boundaries.py index 8302b4a2b..943be2a9b 100644 --- a/tests/test_reasoning_boundaries.py +++ b/tests/test_reasoning_boundaries.py @@ -4,17 +4,18 @@ import pytest from parity import MODEL_CATALOG - from renderers import create_renderer from renderers.base import ToolCallParseStatus, load_tokenizer from renderers.configs import config_from_name - # Exercise every registered renderer, including inherited family variants. _CASES = {} for _case in MODEL_CATALOG: _CASES.setdefault(_case.resolved_renderer, _case.model) +_GLM = {"glm-4.5", "glm-5", "glm-5.1", "glm-5.3"} +_REOPENING_TAGS = _GLM | {"deepseek-v4"} + @lru_cache(None) def _renderer(name): @@ -236,7 +237,9 @@ def test_tool_opener_ends_unclosed_reasoning(name, stop): assert "danger" in tok.decode(sampled[start:end], skip_special_tokens=False) -@pytest.mark.parametrize("name", sorted(set(_TOOL_CALLS) - {"inkling"})) +@pytest.mark.parametrize( + "name", sorted(set(_TOOL_CALLS) - {"inkling"} - _REOPENING_TAGS) +) def test_explicit_reasoning_close_outranks_earlier_tool_opener(name): # Inkling closes thinking and tool segments with the same <|end_message|>. tok, renderer = _renderer(name) @@ -332,6 +335,120 @@ def test_empty_final_channel_is_not_unfinished_reasoning(): assert result.reasoning_complete is True +@pytest.mark.parametrize("name", sorted(_REOPENING_TAGS)) +def test_upstream_template_continuation_opens_reasoning(name): + from tests.reference_rendering import render_reference + + tok, renderer = _renderer(name) + messages = [{"role": "user", "content": "Calculate carefully."}] + kwargs = { + key: getattr(renderer.config, key) + for key in renderer.config.template_field_names() + if getattr(renderer.config, key) is not None + } + prompt = render_reference(tok, messages, add_generation_prompt=True, **kwargs) + completed = render_reference( + tok, + messages + + [ + {"role": "assistant", "reasoning_content": "Reasoning", "content": "Answer"} + ], + **kwargs, + ) + assert renderer.render_ids(messages, add_generation_prompt=True) == prompt + assert completed[: len(prompt)] == prompt + sampled = completed[len(prompt) :] + original = list(sampled) + parsed = renderer.parse_response(sampled, prompt_ids=prompt) + assert parsed.reasoning_content == ( + "\nReasoning" if name == "glm-4.5" else "Reasoning" + ) + assert parsed.content == ("\nAnswer" if name == "glm-4.5" else "Answer") + assert parsed.reasoning_complete is True + assert not parsed.tool_calls + assert sampled == original + + +@pytest.mark.parametrize( + "name", sorted(set(_CASES) - {"llama-3", "default", "gpt-oss", "gemma4", "inkling"}) +) +@pytest.mark.parametrize("prefix", ["\n", " \n"]) +def test_initial_reasoning_accepts_whitespace_and_duplicate_openers(name, prefix): + tok, renderer = _renderer(name) + prompt, _, _ = _thinking_stream(name, tok, renderer) + opening, closing = ( + ("", "") + if name == "hy3" + else ("", "") + ) + sampled = _encode( + tok, prefix + opening + "\n" + opening + "Reasoning" + closing + "Answer" + ) + original = list(sampled) + parsed = renderer.parse_response(sampled, prompt_ids=prompt) + assert parsed.reasoning_content.strip() == "Reasoning" + assert parsed.content == "Answer" + assert parsed.reasoning_complete is True + assert sampled == original + + +@pytest.mark.parametrize("name", sorted(_REOPENING_TAGS)) +@pytest.mark.parametrize("closed", [False, True]) +def test_tagged_grammar_reopens_reasoning_after_content(name, closed): + tok, renderer = _renderer(name) + prompt, _, _ = _thinking_stream(name, tok, renderer) + sampled = _encode( + tok, + "FirstAnswerSecond" + ("After" if closed else ""), + ) + parsed = renderer.parse_response(sampled, prompt_ids=prompt) + assert parsed.reasoning_content == "FirstSecond" + assert parsed.content == "Answer" + ("After" if closed else "") + assert parsed.reasoning_complete is closed + + +@pytest.mark.parametrize("name", sorted(_REOPENING_TAGS)) +def test_tagged_tool_transition_precedes_later_reasoning_close(name): + tok, renderer = _renderer(name) + prompt, incomplete, _ = _thinking_stream(name, tok, renderer) + sampled = incomplete + _encode(tok, _TOOL_CALLS[name] + "") + original = list(sampled) + parsed = renderer.parse_response(sampled, prompt_ids=prompt) + assert parsed.reasoning_content == "unfinished reasoning" + assert [(c.name, c.arguments, c.status) for c in parsed.tool_calls] == [ + ("danger", {}, ToolCallParseStatus.OK) + ] + assert parsed.reasoning_complete is True + start, end = parsed.tool_calls[0].token_span + assert 0 <= start < end <= len(sampled) + assert "danger" in tok.decode(sampled[start:end], skip_special_tokens=False) + assert sampled == original + + +@pytest.mark.parametrize("name", ["glm-4.5", "glm-5", "glm-5.1", "deepseek-v4"]) +@pytest.mark.parametrize("enabled", [False, True]) +def test_tagged_grammar_generation_mode_matches_vllm(name, enabled): + tok, original = _renderer(name) + renderer = create_renderer( + tok, original.config.model_copy(update={"enable_thinking": enabled}) + ) + prompt = renderer.render_ids( + [{"role": "user", "content": "Hello"}], add_generation_prompt=True + ) + sampled = _encode(tok, "\nReasoning\nAnswer") + parsed = renderer.parse_response(sampled, prompt_ids=prompt) + if enabled: + assert parsed.reasoning_content == "\nReasoning" + assert parsed.content == "\nAnswer" + elif name == "deepseek-v4": + assert parsed.reasoning_content == "Reasoning" + assert parsed.content == "\n\nAnswer" + else: + assert parsed.reasoning_content is None + assert parsed.content == "\nReasoning\nAnswer" + assert parsed.reasoning_complete is True + + def test_no_reasoning_format_keeps_literal_think_text(): tok, renderer = _renderer("llama-3") result = renderer.parse_response(_encode(tok, "Literal text")) @@ -376,7 +493,7 @@ def test_inkling_partial_reasoning_prefix_is_not_returned_as_content(): @pytest.mark.parametrize( - "name", sorted((set(_CASES) - {"llama-3"}) | {"default-think"}) + "name", sorted((set(_CASES) - {"llama-3"} - _GLM) | {"default-think"}) ) @pytest.mark.parametrize("enabled", [False, True]) @pytest.mark.parametrize("origin", ["prompt", "completion", "closed_prompt", "both"]) @@ -442,7 +559,12 @@ def test_reasoning_markers_override_generation_mode(name, enabled, origin, endin @pytest.mark.parametrize( - "name", sorted(set(_CASES) - {"llama-3", "gpt-oss", "gemma4", "inkling", "default"}) + "name", + sorted( + set(_CASES) + - {"llama-3", "gpt-oss", "gemma4", "inkling", "default"} + - _REOPENING_TAGS + ), ) @pytest.mark.parametrize("closed", [False, True]) def test_late_think_markers_stay_content(name, closed): @@ -520,9 +642,14 @@ def test_missing_and_empty_prompt_are_self_contained(name): tok, renderer = _renderer(name) bare = _encode(tok, "Answer") expected = renderer.parse_response(bare, prompt_ids=[]) - assert expected.content == "Answer" - assert expected.reasoning_content is None - assert expected.reasoning_complete is not False + if name in _REOPENING_TAGS: + assert expected.content == "" + assert expected.reasoning_content == "Answer" + assert expected.reasoning_complete is False + else: + assert expected.content == "Answer" + assert expected.reasoning_content is None + assert expected.reasoning_complete is not False assert renderer.parse_response(bare) == expected assert renderer.parse_response(bare, prompt_ids=None) == expected @@ -591,6 +718,33 @@ def test_harmony_later_channels_end_unterminated_analysis( assert not any(bridge.sampled_mask[len(prefix) :]) +@pytest.mark.parametrize("name", sorted(_GLM)) +@pytest.mark.parametrize("closed", [False, True]) +def test_glm_reopens_reasoning_after_tool_without_parsing_argument_tags(name, closed): + tok, renderer = _renderer(name) + prompt = renderer.render_ids( + [{"role": "user", "content": "Hello"}], add_generation_prompt=True + ) + tool = ( + "weathercity" + "literal" + ) + sampled = _encode( + tok, + "FirstBefore" + + tool + + "AfterSecond" + + ("Last" if closed else ""), + ) + parsed = renderer.parse_response(sampled, prompt_ids=prompt) + assert parsed.reasoning_content == "FirstSecond" + assert parsed.content == "BeforeAfter" + ("Last" if closed else "") + assert parsed.reasoning_complete is closed + assert len(parsed.tool_calls) == 1 + assert parsed.tool_calls[0].arguments == {"city": "literal"} + assert sampled[slice(*parsed.tool_calls[0].token_span)] == _encode(tok, tool) + + def test_harmony_later_unfinished_analysis_preserves_earlier_output(): tok, renderer = _renderer("gpt-oss") prompt, _, _ = _thinking_stream("gpt-oss", tok, renderer) From 3faca8296bf84ba8e9841c6288a31f292652586e Mon Sep 17 00:00:00 2001 From: hallerite Date: Sun, 11 Oct 2026 16:15:13 +0200 Subject: [PATCH 2/2] Add black-box renderer conformance and pinned vLLM audits --- .github/workflows/conformance.yml | 88 +++++ renderers/base.py | 10 +- renderers/laguna_xs2.py | 4 + renderers/parsing.py | 10 +- tests/conformance/README.md | 122 +++++++ tests/conformance/__init__.py | 1 + tests/conformance/audit_vllm.py | 406 +++++++++++++++++++++ tests/conformance/encoder.py | 88 +++++ tests/conformance/export_vllm.py | 157 ++++++++ tests/conformance/harness.py | 134 +++++++ tests/conformance/prepare_upstream.py | 43 +++ tests/conformance/report.py | 180 +++++++++ tests/conformance/run_upstream.py | 67 ++++ tests/conformance/test_rollouts.py | 262 +++++++++++++ tests/conformance/test_tool_cycles.py | 208 +++++++++++ tests/conformance/test_traces.py | 64 ++++ tests/conformance/tokenizers.json | 182 +++++++++ tests/conformance/upstream_tokenizers.json | 13 + tests/conformance/vllm_reference.py | 185 ++++++++++ 19 files changed, 2218 insertions(+), 6 deletions(-) create mode 100644 .github/workflows/conformance.yml create mode 100644 tests/conformance/README.md create mode 100644 tests/conformance/__init__.py create mode 100644 tests/conformance/audit_vllm.py create mode 100644 tests/conformance/encoder.py create mode 100644 tests/conformance/export_vllm.py create mode 100644 tests/conformance/harness.py create mode 100644 tests/conformance/prepare_upstream.py create mode 100644 tests/conformance/report.py create mode 100644 tests/conformance/run_upstream.py create mode 100644 tests/conformance/test_rollouts.py create mode 100644 tests/conformance/test_tool_cycles.py create mode 100644 tests/conformance/test_traces.py create mode 100644 tests/conformance/tokenizers.json create mode 100644 tests/conformance/upstream_tokenizers.json create mode 100644 tests/conformance/vllm_reference.py diff --git a/.github/workflows/conformance.yml b/.github/workflows/conformance.yml new file mode 100644 index 000000000..c4ddfffdc --- /dev/null +++ b/.github/workflows/conformance.yml @@ -0,0 +1,88 @@ +name: Renderer conformance + +on: + pull_request: + push: + branches: [main] + workflow_dispatch: + +jobs: + templates-and-rollouts: + runs-on: ubuntu-latest + env: + RENDERERS_UPSTREAM_ENCODER: "1" + steps: + - uses: actions/checkout@v4 + - uses: astral-sh/setup-uv@v7 + - run: uv sync --python 3.12 + - name: Pinned templates and production-client rollouts + run: uv run pytest tests/conformance -v --junitxml=conformance.xml + - uses: actions/upload-artifact@v4 + if: always() + with: + name: template-conformance + path: conformance.xml + + vllm-parsers: + runs-on: ubuntu-latest + timeout-minutes: 60 + env: + VLLM_SOURCE_COMMIT: db9527a46873454610df6dbedf79a36d6bf1a7f6 + RENDERERS_UPSTREAM_ENCODER: "1" + steps: + - uses: actions/checkout@v4 + - uses: actions/checkout@v4 + with: + repository: vllm-project/vllm + ref: db9527a46873454610df6dbedf79a36d6bf1a7f6 + path: upstream-vllm + - uses: astral-sh/setup-uv@v7 + - name: Install actual pinned parser runtime + id: runtime + run: | + uv sync --python 3.12 --no-dev --extra transformers + uv pip install --python .venv/bin/python vllm==0.31.0 transformers==5.17.0 openai-harmony==0.0.8 pytest pytest-asyncio + - name: Pin tokenizer assets for unchanged upstream fixtures + id: assets + env: + HF_HUB_CACHE: ${{ runner.temp }}/vllm-tokenizers + run: uv run --no-sync tests/conformance/prepare_upstream.py + - name: Run upstream engine tests without changing their expectations + if: ${{ !cancelled() && steps.runtime.outcome == 'success' }} + run: uv run --no-sync tests/conformance/run_upstream.py upstream-vllm --junitxml=vllm-engine.xml + - name: Unmodified upstream reasoning tests + if: ${{ !cancelled() && steps.assets.outcome == 'success' }} + env: + HF_HUB_CACHE: ${{ runner.temp }}/vllm-tokenizers + HF_HUB_OFFLINE: "1" + TRANSFORMERS_OFFLINE: "1" + run: uv run --no-sync tests/conformance/run_upstream.py upstream-vllm --suite reasoning --junitxml=vllm-reasoning.xml + - name: Unmodified upstream tool parser tests + if: ${{ !cancelled() && steps.assets.outcome == 'success' }} + env: + HF_HUB_CACHE: ${{ runner.temp }}/vllm-tokenizers + HF_HUB_OFFLINE: "1" + TRANSFORMERS_OFFLINE: "1" + run: uv run --no-sync tests/conformance/run_upstream.py upstream-vllm --suite tools --junitxml=vllm-tools.xml + - name: Export upstream self-validated scenarios + if: ${{ !cancelled() && steps.runtime.outcome == 'success' }} + run: uv run --no-sync tests/conformance/export_vllm.py upstream-vllm vllm-corpus.json + - name: Real BPE, complete and streaming parser fidelity + id: audit + if: ${{ !cancelled() && steps.runtime.outcome == 'success' }} + run: uv run --no-sync -m tests.conformance.audit_vllm vllm-corpus.json renderer-fidelity.json + - name: Summarize disagreements and authority decisions + if: ${{ !cancelled() && (steps.audit.outcome == 'success' || steps.audit.outcome == 'failure') }} + run: uv run --no-sync -m tests.conformance.report renderer-fidelity.json renderer-fidelity.md + - uses: actions/upload-artifact@v4 + if: always() + with: + name: vllm-fidelity + path: | + vllm-engine.xml + vllm-reasoning.xml + vllm-tools.xml + vllm-corpus.json + renderer-fidelity.json + renderer-fidelity.jsonl + renderer-fidelity.md diff --git a/renderers/base.py b/renderers/base.py index 5477c470a..fda9a7322 100644 --- a/renderers/base.py +++ b/renderers/base.py @@ -1308,7 +1308,7 @@ def _load_tokenizer_via_auto(model_name_or_path: str, **kwargs) -> Any: return tok -def load_tokenizer(model_name_or_path: str): +def load_tokenizer(model_name_or_path: str, *, revision: str | None = None): """Load a tokenizer with the renderers-package security policy. Default ``trust_remote_code=False``. Models listed in @@ -1328,11 +1328,19 @@ def load_tokenizer(model_name_or_path: str): ``unsloth`` mirrors instead, then restore ``tokenizer.name_or_path`` to the requested Meta ID so auto-resolution still selects ``Llama3Renderer``. Requires the ``renderers[transformers]`` extra. + + ``revision`` pins ordinary tokenizer assets for reproducible reference + comparisons. It cannot override a trusted remote-code revision. """ try: _require_transformers("Loading a tokenizer") load_name_or_path = _tokenizer_source_for(model_name_or_path) kwargs = _tokenizer_load_kwargs(load_name_or_path) + if revision is not None: + trusted = kwargs.get("revision") + if trusted is not None and revision != trusted: + raise ValueError("Cannot override the reviewed remote-code revision") + kwargs["revision"] = revision tok = _load_tokenizer_via_auto(load_name_or_path, **kwargs) return _preserve_requested_tokenizer_name( tok, diff --git a/renderers/laguna_xs2.py b/renderers/laguna_xs2.py index 123d7b8b0..4f6c587f0 100644 --- a/renderers/laguna_xs2.py +++ b/renderers/laguna_xs2.py @@ -421,6 +421,7 @@ def bridge_to_next_turn( # ``\n`` — the same scaffold the template emits. previous_ids = list(previous_prompt_ids) + list(previous_completion_ids) stop_ids = {self._assistant_end, self._eos} + needs_separator = previous_ids[-1] in stop_ids if ( not previous_ids[len(previous_prompt_ids) :] or previous_ids[-1] not in stop_ids @@ -478,6 +479,9 @@ def emit_text_segments( ext_sampled.append(is_sampled) ext_content.append(is_content) + if needs_separator: + emit_text("\n", -1) + for i, msg in enumerate(new_messages): role = msg.get("role") content = self._visible_text(msg.get("content")) diff --git a/renderers/parsing.py b/renderers/parsing.py index 0ad0967ed..ed1fba949 100644 --- a/renderers/parsing.py +++ b/renderers/parsing.py @@ -441,7 +441,7 @@ def _parse_xml_tool_calls( r"]+)>\n?(.*?)\n?", block_text, re.DOTALL ): arg_name = pm.group(1) - arg_value = pm.group(2).strip() + arg_value = pm.group(2) value, used_fallback = _coerce_arg_value( arg_value, params.get(arg_name) ) @@ -639,7 +639,7 @@ def _parse_glm_tool_calls( if ave == -1: structure_broke = True break - val_text = _decode(tokenizer, block[av + 1 : ave]).strip() + val_text = _decode(tokenizer, block[av + 1 : ave]) value, used_fallback = _coerce_arg_value( val_text, params.get(key) ) @@ -849,7 +849,7 @@ def _parse_hy3_tool_calls( if ave == -1: structure_broke = True break - val_text = _decode(tokenizer, arg_ids[av + 1 : ave]).strip() + val_text = _decode(tokenizer, arg_ids[av + 1 : ave]) value, used_fallback = _coerce_arg_value(val_text, params.get(key)) arguments[key] = value any_json_fallback = any_json_fallback or used_fallback @@ -1028,7 +1028,7 @@ def _parse_laguna_xs2_tool_calls( re.DOTALL, ): k = m.group(1).strip() - v = m.group(2).strip() + v = m.group(2) value, used_fallback = _coerce_arg_value(v, params.get(k)) arguments[k] = value any_json_fallback = any_json_fallback or used_fallback @@ -1528,7 +1528,7 @@ def parse_minimax( re.DOTALL, ): pname = pm.group(1) - pval = pm.group(2).strip() + pval = pm.group(2) value, used_fallback = _coerce_arg_value( pval, params.get(pname) ) diff --git a/tests/conformance/README.md b/tests/conformance/README.md new file mode 100644 index 000000000..d5612cc61 --- /dev/null +++ b/tests/conformance/README.md @@ -0,0 +1,122 @@ +# Renderer conformance + +The renderer is a black box. Tests use `create_renderer`, `render`, `render_ids`, +`parse_response`, `bridge_to_next_turn`, and the production `generate` client. +Only sampling is scripted. No native parser internals generate expectations. + +The shared model/scenario catalog drives all 29 concrete renderer formats and +45 checkpoints. `tokenizers.json` pins the reviewed assets; a new renderer must +have a catalog checkpoint. Complete transcripts exercise every valid scenario +and finite template setting, including raw assistant serialization. Structured +inference settings also run three user turns, token-budget stops, and two tool +cycles with ordinary, empty, and whitespace/Unicode string arguments. Parsed +public-client messages are reused in subsequent turns. Raw assistant +passthrough and training serialization belong to the transcript matrix, rather +than the structured-message client loop. + +Generation boundaries must come from the official generation prompt. Historical +assistant headers are never inserted into completion tokens. R1 and Harmony +have explicit live fixtures because their history format removes reasoning or +changes channels. Closed thinking prefills use the official finished body and +turn close. No `.strip()` is allowed on parsed nonempty channel contents. + +A bridge must preserve every sampled token and append the exact template suffix. +The suffix is obtained by perturbing only the previous assistant body in two +reference renders and locating their common turn-close suffix. This still works +when a new query changes historical reasoning. A declined bridge takes a full +reference-checked render. Empty/absent wire reasoning and null/string content +are mapped to the renderer API; nonempty whitespace is never normalized. + +## Independent references + +Most encoders are the checkpoint's unmodified Jinja template. The isolated CI +jobs set `RENDERERS_UPSTREAM_ENCODER=1` to execute DeepSeek V4's actual Python +encoder at its pinned revision and SHA256. The host suite uses the existing +portable DeepSeek adapter. That adapter is additionally checked against the +actual encoder in CI. Harmony's existing reference profile uses the official +encoder with an explicit forced-analysis prefill. An additional normative +generation-prefix test calls the unmodified official encoder and exposes that +profile's divergence instead of silently assuming it is template fidelity. + +vLLM is **0.31.0**, source commit +`db9527a46873454610df6dbedf79a36d6bf1a7f6`, matching the production lockfile. +The reference job pins transformers 5.17.0 and openai-harmony 0.0.8 and records +the installed reference dependencies in its exported corpus. +`run_upstream.py` runs upstream engine tests plus selected unchanged reasoning +and tool parser suites without editing their expectations. `prepare_upstream.py` +pins their public tokenizer snapshots in an isolated HF cache; the unchanged +fixtures resolve `main` to those snapshots and run offline. The legacy gated +Llama fixture is not in that selection; Llama's actual public parser is covered +by native-versus-vLLM tool cycles using the reviewed unrestricted tokenizer. +`export_vllm.py` imports the upstream trace builder normally and exports its +self-validated scenarios. It also invokes the original DeepSeek V3 test-config +fixture function to export the unchanged legacy tool cases, including malformed +calls, for V3 and R1's completion parsers. Those checkpoints do not have a +structured tool-history template. Every checkpoint must have both reasoning +and tool-wire comparisons; missing coverage fails the audit. There are no import stubs, +AST extraction, copied parser implementations, or fake tokenizer IDs in the +native-versus-vLLM comparison. +The installed parser, reasoning, tool-parser, and tokenizer Python files must +hash-identically to the pinned source checkout before upstream tests run. + +`audit_vllm.py` retokenizes that corpus with the real pinned checkpoint tokenizer, +adds truncations, and includes template-derived live completions for every +renderer. `vllm_reference.py` uses `ParserManager` and the serving parser names, +plus vLLM's own incremental detokenizer. Complete and streaming outputs are +recorded separately, with chunks of 1, 2, 5, and the whole completion. + +Run the portable tests: + +```sh +uv run pytest tests/conformance +``` + +Run the isolated reference job, with the pinned source checkout: + +```sh +uv pip install --python .venv/bin/python vllm==0.31.0 transformers==5.17.0 openai-harmony==0.0.8 +HF_HUB_CACHE=/tmp/vllm-tokenizers uv run --no-sync tests/conformance/prepare_upstream.py +uv run --no-sync tests/conformance/run_upstream.py /path/to/vllm +HF_HUB_CACHE=/tmp/vllm-tokenizers HF_HUB_OFFLINE=1 TRANSFORMERS_OFFLINE=1 \ + uv run --no-sync tests/conformance/run_upstream.py /path/to/vllm --suite reasoning +HF_HUB_CACHE=/tmp/vllm-tokenizers HF_HUB_OFFLINE=1 TRANSFORMERS_OFFLINE=1 \ + uv run --no-sync tests/conformance/run_upstream.py /path/to/vllm --suite tools +uv run --no-sync tests/conformance/export_vllm.py /path/to/vllm /tmp/vllm-corpus.json +RENDERERS_UPSTREAM_ENCODER=1 uv run --no-sync -m tests.conformance.audit_vllm \ + /tmp/vllm-corpus.json /tmp/renderer-fidelity.json +uv run --no-sync -m tests.conformance.report \ + /tmp/renderer-fidelity.json /tmp/renderer-fidelity.md +``` + +## Disagreement policy + +Serialization follows the official template. Live channels follow the exact +generation prefix and vLLM's streaming projection when it is chunk invariant. +Chunk-dependent reference output is an upstream defect to resolve against the +template and fixture, not a reason to bless one arbitrary chunk size. +Complete-versus-streaming disagreement is reported; the prompt-aware live +stream owns the live contract. Each report includes inputs, settings, +raw text, both reference outputs, native output, an authority decision, and a +fingerprint. Any native-versus-authoritative or chunk-size disagreement fails +CI after writing the full report; there is no blanket skip/xfail allowlist. +`report.py` groups differences and recommends an authority per contract without +changing any comparison result. These recommendations do not turn known +disagreements green. The broader fidelity gate can remain red while the +profile/template/client tests pass; that is evidence of outstanding parser +work. The unmodified-prefill guard separately rejects a renderer that injects +an unconfigured channel prefill. It is not skipped or marked xfail. + +Tool arguments compare as JSON values, preserving every string byte from the +official encoding. Serving-inserted spaces and withheld incomplete Unicode are +reported as detokenization differences, independently of parsing. Random +serving-generated call IDs are excluded from that projection. Kimi's native +qualified IDs are retained for history/tool result association. Successful +calls must match the grammar and schema; a serving parser that trims payload +strings loses to the template's original payload. Malformed attempts remain +extra renderer metadata and never become executable calls merely to match a +salvage parser's partial intent. Raw sampled +IDs remain intact irrespective of any structured projection. + +The test suite checks text/token protocols. Existing processor parity tests +cover image/audio/video expansion; the new text suite does not claim to validate +GPU sampling, media processing, or the dashboard UI. diff --git a/tests/conformance/__init__.py b/tests/conformance/__init__.py new file mode 100644 index 000000000..8169ee34e --- /dev/null +++ b/tests/conformance/__init__.py @@ -0,0 +1 @@ +"""Independent reference tests through the public renderer interface.""" diff --git a/tests/conformance/audit_vllm.py b/tests/conformance/audit_vllm.py new file mode 100644 index 000000000..fe376d5d0 --- /dev/null +++ b/tests/conformance/audit_vllm.py @@ -0,0 +1,406 @@ +"""Compare public renderers to real vLLM; emit every disagreement before failing. + +uv run --with vllm==0.31.0 -m tests.conformance.audit_vllm \ + /tmp/vllm-corpus.json /tmp/renderer-fidelity.json +""" + +from __future__ import annotations + +import argparse +import hashlib +import json +from pathlib import Path + +from renderers.base import ToolCallParseStatus +from tests.conformance.harness import ( + reference, + renderer, + tokenizer, + unmodified_generation_reference, +) +from tests.conformance.report import POLICY, differences +from tests.conformance.test_rollouts import canonical_turn, matrix +from tests.conformance.test_tool_cycles import ( + cases as tool_cases, +) +from tests.conformance.test_tool_cycles import ( + reference_turn, + tool_renderer, + tool_turn, +) +from tests.conformance.vllm_reference import ( + PARSERS, + parse_reference, + projection, + same_projection, +) +from tests.parity import MODEL_CATALOG, TOOLS + +CORPUS_ROUTES = { + "prime-qwen3": "qwen3", + "qwen3.5": "qwen3", + "qwen3.6": "qwen3", + "qwen3.8": "qwen3", + "glm-4.5": "glm47_moe", + "glm-5": "glm47_moe", + "glm-5.1": "glm47_moe", + "glm-5.3": "glm47_moe", + "gemma4": "gemma4", + "deepseek-v4": "deepseek_v4", + "minimax-m2": "minimax_m2", + "nemotron-3": "nemotron_v3", + "nemotron-3-ultra": "nemotron_v3", + "nemotron-3.5": "nemotron_v3", + "kimi-k2": "kimi_k2", + "kimi-k2.5": "kimi_k2", + "inkling": "inkling", +} + + +def native_projection(parsed): + return projection( + parsed.reasoning_content, + parsed.content, + [ + {"name": call.name, "arguments": call.arguments} + for call in parsed.tool_calls + if call.status == ToolCallParseStatus.OK + ], + ) + + +def generation_records(models=None): + records = [] + for parameter in matrix(): + case, kwargs = parameter.values + if models and case.model not in models: + continue + messages = [{"role": "user", "content": "Test."}] + actual = renderer(case, kwargs).render_ids(messages, add_generation_prompt=True) + expected = unmodified_generation_reference(case, messages, kwargs) + tok = tokenizer(case.model) + records.append( + { + "model": case.model, + "settings": kwargs, + "agrees": actual == expected, + "native_ids": actual, + "reference_ids": expected, + "native_tail": tok.decode(actual[-96:], skip_special_tokens=False), + "reference_tail": tok.decode(expected[-96:], skip_special_tokens=False), + } + ) + return records + + +def inputs(corpus): + # Every structured-inference configuration, including actual bridged + # prompts and structured parser output reused as conversation history. + for parameter in matrix(): + case, kwargs = parameter.values + history = [{"role": "user", "content": "What is two plus two?"}] + next_prompt = None + for turn in range(3): + try: + native = renderer(case, kwargs) + prompt, sampled, _, _, _ = canonical_turn(case, kwargs, native, history) + if next_prompt is not None: + prompt = next_prompt.token_ids + completion = ( + sampled[:-1] + if sampled and sampled[-1] in native.get_stop_token_ids() + else sampled + ) + yield ( + case, + kwargs, + f"template-continuation/turn-{turn}", + prompt, + completion, + [], + ) + parsed = native.parse_response(sampled, prompt_ids=prompt) + assistant = {"role": "assistant", "content": parsed.content} + if parsed.reasoning_content is not None: + assistant["reasoning_content"] = parsed.reasoning_content + new = [{"role": "user", "content": f"Next query {turn}."}] + bridge = native.bridge_to_next_turn(prompt, sampled, new) + history += [assistant] + new + next_prompt = ( + bridge + if bridge is not None + else native.render(history, add_generation_prompt=True) + ) + except Exception as exc: + yield ( + case, + kwargs, + f"fixture-error/{type(exc).__name__}: {exc}", + [], + [], + [], + ) + break + for parameter in tool_cases(): + (case,) = parameter.values + kwargs = {"is_training": False} if case.resolved_renderer == "hy3" else {} + native = tool_renderer(case, kwargs) + for city in ("Paris", "", " Paris\nMünchen ☁ "): + history = [ + {"role": "system", "content": "Use the weather tool."}, + {"role": "user", "content": "Weather?"}, + ] + next_prompt = None + for turn in range(2): + assistant, new = tool_turn(case, turn, city) + prompt, sampled, _ = reference_turn( + case, native, history, assistant, new, kwargs + ) + if next_prompt is not None: + prompt = next_prompt.token_ids + completion = ( + sampled[:-1] + if sampled and sampled[-1] in native.get_stop_token_ids() + else sampled + ) + yield ( + case, + kwargs, + f"template-tool-cycle/{city!r}/turn-{turn}", + prompt, + completion, + TOOLS, + ) + parsed = native.parse_response(sampled, prompt_ids=prompt, tools=TOOLS) + assistant = { + "role": "assistant", + "content": parsed.content, + "reasoning_content": parsed.reasoning_content, + "tool_calls": [ + { + "id": call.id or new[0]["tool_call_id"], + "type": "function", + "function": { + "name": call.name, + "arguments": call.arguments, + }, + } + for call in parsed.tool_calls + if call.status == ToolCallParseStatus.OK + ], + } + new[0]["tool_call_id"] = assistant["tool_calls"][0]["id"] + bridge = native.bridge_to_next_turn(prompt, sampled, new, tools=TOOLS) + history += [assistant] + new + next_prompt = ( + bridge + if bridge is not None + else native.render(history, tools=TOOLS, add_generation_prompt=True) + ) + # Legacy V3/R1 checkpoints have no structured tool-history template, but + # their public completion parsers support the upstream wire protocol. + for case in MODEL_CATALOG: + if case.resolved_renderer not in {"deepseek-v3", "deepseek-r1"}: + continue + tok = tokenizer(case.model) + prompt = reference( + case, [{"role": "user", "content": "Test."}], {}, add_generation_prompt=True + ) + for record in corpus["legacy_tool_traces"]["deepseek_v3"]: + # R1's exact live prefix opens reasoning. Close that empty channel + # before sampling the unmodified upstream tool-parser trace. + text = ( + "" if case.resolved_renderer == "deepseek-r1" else "" + ) + record["text"] + ids = tok.encode(text, add_special_tokens=False) + yield case, {}, record["id"], prompt, ids, [] + if len(ids) > 1: + yield ( + case, + {}, + record["id"] + "/truncated", + prompt, + ids[: len(ids) // 2], + [], + ) + + # Upstream's independently self-validated multi-format scenario corpus. + for case in MODEL_CATALOG: + family = CORPUS_ROUTES.get(case.resolved_renderer) + if family is None: + continue + kwargs = {} + fields = renderer(case, {}).config.template_field_names() + for field in ("enable_thinking", "thinking"): + if field in fields: + kwargs[field] = True + for record in corpus["families"][family]: + settings = dict(kwargs) + upstream_settings = dict(record["chat_template_kwargs"]) + if family == "deepseek_v4" and "thinking" in upstream_settings: + settings["enable_thinking"] = upstream_settings.pop("thinking") + settings.update(upstream_settings) + tok = tokenizer(case.model) + tools = record["tools"] + prompt = ( + tok.encode(record["prompt_text"], add_special_tokens=False) + if record["prompt_text"] + else reference( + case, + [{"role": "user", "content": "Test."}], + settings, + tools=tools or None, + add_generation_prompt=True, + ) + ) + ids = tok.encode(record["text"], add_special_tokens=False) + yield case, settings, record["id"], prompt, ids, tools + if record["text"].startswith(""): + for whitespace in ("\n", "\r\n"): + spaced = tok.encode( + whitespace + record["text"], add_special_tokens=False + ) + yield ( + case, + settings, + record["id"] + f"/leading-{whitespace!r}", + prompt, + spaced, + tools, + ) + # A token-budget stop halfway through the very SAME upstream output. + # Compare partial serving intents too; executable validity remains + # a separate renderer contract, not implied by streamed deltas. + if len(ids) > 1: + yield ( + case, + settings, + record["id"] + "/truncated", + prompt, + ids[: len(ids) // 2], + tools, + ) + + +def audit(corpus, on_record, models=None): + assert set(PARSERS) == {c.resolved_renderer for c in MODEL_CATALOG} + selected = set(models) if models else {case.model for case in MODEL_CATALOG} + assert selected <= {case.model for case in MODEL_CATALOG}, selected + records = [] + for case, kwargs, scenario, prompt, completion, tools in inputs(corpus): + if models and case.model not in models: + continue + tok = tokenizer(case.model) + key = f"{case.model}/{scenario}/{json.dumps(kwargs, sort_keys=True)}" + record = { + "case": key, + "model": case.model, + "format": case.resolved_renderer, + "prompt_profile": "forced-analysis" + if case.oracle == "harmony" + else "official-generation-prefill", + "settings": kwargs, + "scenario": scenario, + "prompt_ids": prompt, + "prompt_tail": tok.decode(prompt[-96:], skip_special_tokens=False), + "completion_ids": completion, + "text": tok.decode(completion, skip_special_tokens=False), + "tools": tools, + } + try: + if scenario.startswith("fixture-error/"): + raise RuntimeError(scenario) + native = tool_renderer(case, kwargs) if tools else renderer(case, kwargs) + parsed = native.parse_response( + completion, prompt_ids=prompt, tools=tools or None + ) + record["native"] = native_projection(parsed) + record["reasoning_complete"] = parsed.reasoning_complete + record["attempts"] = [ + {"status": c.status.value, "raw": c.raw, "span": c.token_span} + for c in parsed.tool_calls + ] + record["vllm"] = parse_reference( + case, tok, prompt, completion, tools, kwargs + ) + streams = list(record["vllm"]["stream"].values()) + expected = streams[0] if streams else record["vllm"]["complete"] + record["agrees"] = ( + not record["vllm"].get("stream_errors") + and same_projection(record["native"], expected) + and all(same_projection(value, expected) for value in streams) + ) + record["vllm_complete_agrees_with_stream"] = same_projection( + record["vllm"]["complete"], expected + ) + record["vllm_chunk_invariant"] = not record["vllm"].get( + "stream_errors" + ) and all(same_projection(value, expected) for value in streams) + record["decision"] = { + "authority": "vllm-stream-with-exact-prompt" + if streams + else ( + "reference-phase-failure" + if expected is None + else "configured-identity-parser" + ), + "expected": expected, + "reason": "The real generation prompt determines the initial channel. Streaming chunk size must not change the result. Raw attempts are retained separately.", + } + except Exception as exc: + record["agrees"] = False + record["error"] = f"{type(exc).__name__}: {exc}" + record["differences"] = differences(record) + record["authority_decisions"] = { + category: POLICY[category] for category in record["differences"] + } + signature = json.dumps(record, sort_keys=True).encode() + record["fingerprint"] = hashlib.sha256(signature).hexdigest() + records.append(record) + on_record(record) + print(f"{'PASS' if record['agrees'] else 'DIFF'} {key}", flush=True) + assert {row["model"] for row in records} == selected + assert { + row["model"] + for row in records + if row["scenario"].startswith(("template-tool-cycle/", "deepseek_v3-")) + } == selected + prefills = generation_records(models) + return { + "reference": corpus["reference"], + "tokenizers": json.loads( + Path(__file__).with_name("tokenizers.json").read_text() + ), + "parser_profiles": PARSERS, + "authority_policy": POLICY, + "generation_prefills": prefills, + "cases": records, + "summary": { + "total": len(records), + "disagreements": sum(not r["agrees"] for r in records), + "generation_prefill_disagreements": sum(not r["agrees"] for r in prefills), + }, + } + + +if __name__ == "__main__": + options = argparse.ArgumentParser() + options.add_argument("corpus", type=Path) + options.add_argument("report", type=Path) + options.add_argument("--model", action="append") + args = options.parse_args() + with args.report.with_suffix(".jsonl").open("w") as progress: + + def save(record): + progress.write(json.dumps(record) + "\n") + progress.flush() + + result = audit(json.loads(args.corpus.read_text()), save, models=args.model) + args.report.write_text(json.dumps(result, indent=2) + "\n") + print(json.dumps(result["summary"])) + raise SystemExit( + bool( + result["summary"]["disagreements"] + or result["summary"]["generation_prefill_disagreements"] + ) + ) diff --git a/tests/conformance/encoder.py b/tests/conformance/encoder.py new file mode 100644 index 000000000..75713d292 --- /dev/null +++ b/tests/conformance/encoder.py @@ -0,0 +1,88 @@ +"""Execute the reviewed, pinned model-supplied Python encoder directly.""" + +import copy +import hashlib +import importlib.util +import json +import tempfile +from functools import cache +from pathlib import Path + +from huggingface_hub import hf_hub_download + +DEEPSEEK_ENCODER = { + "repo": "deepseek-ai/DeepSeek-V4-Flash-0731", + "revision": "7872f01b1d1fe23eabc4c98b48bffcef5a386062", + "file": "encoding/encoding_dsv4.py", + "sha256": "abc0d26120250dda0ae077dc64aa28836026e61e970854aaeb792445e6a0dde6", +} + + +@cache +def encoder(): + source = DEEPSEEK_ENCODER + path = hf_hub_download( + source["repo"], + source["file"], + revision=source["revision"], + cache_dir=Path(tempfile.gettempdir()) / "renderers-reference-encoders", + ) + assert hashlib.sha256(Path(path).read_bytes()).hexdigest() == source["sha256"] + spec = importlib.util.spec_from_file_location("pinned_deepseek_encoder", path) + assert spec is not None and spec.loader is not None + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +def deepseek_reference( + tok, + messages, + *, + tools=None, + add_generation_prompt=False, + enable_thinking=False, + drop_thinking=True, + reasoning_effort="low", +): + upstream = encoder() + messages = copy.deepcopy(messages) + if tools: + target = next((m for m in messages if m["role"] == "developer"), None) + if target is None: + target = next((m for m in messages if m["role"] == "system"), None) + if target is None: + target = {"role": "system", "content": ""} + messages.insert(0, target) + target["tools"] = tools + for message in messages: + for call in message.get("tool_calls") or []: + function = call["function"] + if not isinstance(function["arguments"], str): + function["arguments"] = json.dumps( + function["arguments"], ensure_ascii=False + ) + text = upstream.encode_messages( + messages, + thinking_mode="thinking" if enable_thinking else "chat", + drop_thinking=drop_thinking, + reasoning_effort=reasoning_effort, + ) + # Upstream's API always opens generation after a terminal query. The + # renderer API also exposes a completed-transcript view. Remove only that + # terminal generation prefill; all internal transitions remain upstream's. + if not add_generation_prompt: + merged = upstream.merge_tool_messages(messages) + if ( + merged + and merged[-1]["role"] in {"user", "developer"} + and not merged[-1].get("task") + ): + suffix = upstream.ASSISTANT_SP_TOKEN + ( + upstream.thinking_start_token + if enable_thinking + else upstream.thinking_end_token + ) + assert text.endswith(suffix) + text = text[: -len(suffix)] + return tok.encode(text, add_special_tokens=False) diff --git a/tests/conformance/export_vllm.py b/tests/conformance/export_vllm.py new file mode 100644 index 000000000..d54c88283 --- /dev/null +++ b/tests/conformance/export_vllm.py @@ -0,0 +1,157 @@ +"""Export upstream's SELF-VALIDATED corpus; run in the real pinned vLLM env. + +uv run tests/conformance/export_vllm.py /path/to/vllm /tmp/vllm-corpus.json +No copies of parser logic, import stubs, or hand-authored expected outputs. +""" + +from __future__ import annotations + +import hashlib +import importlib.metadata +import json +import subprocess +import sys +from dataclasses import asdict +from pathlib import Path + +VLLM_VERSION = "0.31.0" +VLLM_COMMIT = "db9527a46873454610df6dbedf79a36d6bf1a7f6" +FAMILIES = ( + "qwen3", + "minimax_m2", + "gemma4", + "nemotron_v3", + "deepseek_v4", + "glm47_moe", + "kimi_k2", + "inkling", +) + + +def verify_source(source: Path): + import vllm + + commit = subprocess.check_output( + [ + "git", + "-c", + f"safe.directory={source}", + "-C", + str(source), + "rev-parse", + "HEAD", + ], + text=True, + ).strip() + assert commit == VLLM_COMMIT, (commit, VLLM_COMMIT) + subprocess.run( + [ + "git", + "-c", + f"safe.directory={source}", + "-C", + str(source), + "diff", + "--exit-code", + "HEAD", + "--", + "tests", + "vllm", + ], + check=True, + ) + runtime = Path(vllm.__file__).parent + hashes = {} + for directory in ("parser", "reasoning", "tool_parsers", "tokenizers"): + for file in sorted((source / "vllm" / directory).rglob("*.py")): + relative = file.relative_to(source / "vllm") + digest = hashlib.sha256(file.read_bytes()).hexdigest() + assert ( + hashlib.sha256((runtime / relative).read_bytes()).hexdigest() == digest + ), relative + hashes[str(relative)] = digest + return hashes + + +def export(source: Path) -> dict: + assert importlib.metadata.version("vllm") == VLLM_VERSION + # Import the installed runtime BEFORE adding the upstream tests to sys.path. + import vllm.parser # noqa: F401 + + runtime_hashes = verify_source(source) + sys.path.insert(0, str(source)) + from tests.parser.engine.trace_builder import build_samples + from tests.tool_parsers.test_deepseekv3_tool_parser import TestDeepSeekV3ToolParser + + corpus = {} + for family in FAMILIES: + records = [] + # build_samples validates every sample using the actual upstream parser. + for sample in build_samples(family): + records.append( + { + "id": sample.id, + "description": sample.description, + "text": "".join(text for _, text in sample.tokens), + "reasoning": sample.expected_reasoning, + "content": sample.expected_content, + "tool_calls": sample.expected_tool_calls, + "tools": [ + t.model_dump(exclude_none=True) for t in sample.tools or [] + ], + "chat_template_kwargs": sample.chat_template_kwargs or {}, + "prompt_text": "".join( + next(text for text, tid in sample.vocab.items() if tid == token) + for token in sample.prompt_token_ids or [] + ), + } + ) + corpus[family] = records + # Invoke the ORIGINAL plain fixture function, without rewriting the test + # or its data. The unchanged legacy suite also exercises this fixture. + config = asdict( + TestDeepSeekV3ToolParser.test_config.__wrapped__(TestDeepSeekV3ToolParser()) + ) + legacy = [ + {"id": f"deepseek_v3-{name}", "text": value} + for name, value in config.items() + if name.endswith("_output") and isinstance(value, str) + ] + legacy += [ + {"id": f"deepseek_v3-malformed-{i}", "text": value} + for i, value in enumerate(config["malformed_input_outputs"]) + ] + files = [ + "tests/parser/engine/trace_builder.py", + "tests/parser/engine/replay_harness.py", + "tests/tool_parsers/test_deepseekv3_tool_parser.py", + ] + return { + "reference": { + "version": VLLM_VERSION, + "commit": VLLM_COMMIT, + "dependencies": { + name: importlib.metadata.version(name) + for name in ( + "transformers", + "openai-harmony", + "torch", + "huggingface-hub", + "xgrammar", + ) + }, + "runtime_parser_files": runtime_hashes, + "files": { + name: hashlib.sha256((source / name).read_bytes()).hexdigest() + for name in files + }, + }, + "families": corpus, + "legacy_tool_configs": {"deepseek_v3": config}, + "legacy_tool_traces": {"deepseek_v3": legacy}, + } + + +if __name__ == "__main__": + output = export(Path(sys.argv[1]).resolve()) + Path(sys.argv[2]).write_text(json.dumps(output, indent=2) + "\n") diff --git a/tests/conformance/harness.py b/tests/conformance/harness.py new file mode 100644 index 000000000..10cf06051 --- /dev/null +++ b/tests/conformance/harness.py @@ -0,0 +1,134 @@ +"""No renderer internals: official encoder → token backend → public client.""" + +from __future__ import annotations + +import json +import os +from functools import cache +from pathlib import Path +from typing import Any + +import httpx + +from renderers import config_from_name, create_renderer +from renderers.base import load_tokenizer +from tests.conformance.encoder import deepseek_reference +from tests.parity import ModelCase +from tests.reference_rendering import _harmony_messages, render_reference + + +@cache +def tokenizer(model: str): + pins = json.loads(Path(__file__).with_name("tokenizers.json").read_text()) + return load_tokenizer(model, revision=pins[model]["revision"]) + + +def renderer(case: ModelCase, kwargs: dict[str, Any]): + return create_renderer( + tokenizer(case.model), + config_from_name(case.resolved_renderer), + chat_template_kwargs=kwargs or None, + ) + + +def reference(case: ModelCase, messages: list, kwargs: dict, **render_kwargs): + options = dict(case.oracle_defaults) | kwargs + options = {key: value for key, value in options.items() if value is not None} + if ( + case.oracle == "deepseek-v4" + and os.environ.get("RENDERERS_UPSTREAM_ENCODER") == "1" + ): + return deepseek_reference( + tokenizer(case.model), messages, **options, **render_kwargs + ) + return render_reference( + tokenizer(case.model), messages, oracle=case.oracle, **options, **render_kwargs + ) + + +def unmodified_generation_reference(case: ModelCase, messages: list, kwargs: dict): + if case.oracle != "harmony": + return reference(case, messages, kwargs, add_generation_prompt=True) + # Check the official encoder's default separately from the existing + # forced-analysis profile; a shared oracle/renderer assumption can hide drift. + from openai_harmony import ( + Conversation, + HarmonyEncodingName, + Role, + load_harmony_encoding, + ) + + options = {key: value for key, value in kwargs.items() if value is not None} + conversation = Conversation.from_messages( + _harmony_messages( + messages, + tools=None, + reasoning_effort=options.get("reasoning_effort", "medium"), + conversation_start_date=options.get("conversation_start_date"), + ) + ) + return load_harmony_encoding( + HarmonyEncodingName.HARMONY_GPT_OSS + ).render_conversation_for_completion(conversation, next_turn_role=Role.ASSISTANT) + + +def assert_tokens(actual: list[int], expected: list[int], case: ModelCase): + if actual == expected: + return + first = next( + (i for i, (a, b) in enumerate(zip(actual, expected)) if a != b), + min(len(actual), len(expected)), + ) + tok = tokenizer(case.model) + window = slice(max(0, first - 12), first + 24) + raise AssertionError( + f"{case.model}: first differing token {first}; " + f"lengths native={len(actual)}, reference={len(expected)}\n" + f"native: {tok.decode(actual[window], skip_special_tokens=False)!r}\n" + f"reference: {tok.decode(expected[window], skip_special_tokens=False)!r}" + ) + + +def assert_attribution(rendered): + size = len(rendered.token_ids) + for field in ("message_indices",): + assert len(getattr(rendered, field)) == size, field + assert len(rendered.sampled_mask) in {0, size} + assert len(rendered.is_content) in {0, size} + + +class TokenBackend: + """Script sampling only. Rendering and parsing remain the real implementation.""" + + base_url = "http://conformance.invalid/v1" + + def __init__(self, prompt: list[int], completion: list[int], finish="stop"): + self.prompt = prompt + self.completion = completion + self.finish = finish + self.requests: list[dict] = [] + + async def post(self, endpoint, *, body, cast_to, **kwargs): + assert endpoint == "http://conformance.invalid/inference/v1/generate" + assert body["token_ids"] == self.prompt + assert body["sampling_params"]["skip_special_tokens"] is False + self.requests.append(body) + return httpx.Response( + 200, + content=json.dumps( + { + "choices": [ + { + "token_ids": self.completion, + "finish_reason": self.finish, + "logprobs": { + "content": [ + {"token": f"token_id:{t}", "logprob": -0.5} + for t in self.completion + ] + }, + } + ] + } + ).encode(), + ) diff --git a/tests/conformance/prepare_upstream.py b/tests/conformance/prepare_upstream.py new file mode 100644 index 000000000..205f1a6e9 --- /dev/null +++ b/tests/conformance/prepare_upstream.py @@ -0,0 +1,43 @@ +"""Pin assets used by unchanged upstream fixtures in an isolated HF cache. + +Fixtures request `main`. Populate that ref with reviewed snapshots, then run +upstream offline. This changes neither fixture code nor its expected results. +""" + +import json +import os +from pathlib import Path + +from huggingface_hub import snapshot_download + +root = Path(__file__).parent +pins = json.loads((root / "upstream_tokenizers.json").read_text()) +native = json.loads((root / "tokenizers.json").read_text()) +for model in ( + "Qwen/Qwen3.5-397B-A17B", + "moonshotai/Kimi-K2-Instruct", + "moonshotai/Kimi-K2.5", + "google/gemma-4-E2B-it", +): + pins[model] = native[model]["revision"] +cache = Path(os.environ["HF_HUB_CACHE"]) +for model, revision in pins.items(): + path = Path( + snapshot_download( + model, + revision=revision, + cache_dir=cache, + allow_patterns=[ + "*.json", + "*.jinja", + "*.py", + "*.model", + "*.txt", + "*.tiktoken", + ], + ) + ) + refs = path.parent.parent / "refs" + refs.mkdir(exist_ok=True) + (refs / "main").write_text(revision) + print(f"{model}@{revision}", flush=True) diff --git a/tests/conformance/report.py b/tests/conformance/report.py new file mode 100644 index 000000000..3d30cf0f7 --- /dev/null +++ b/tests/conformance/report.py @@ -0,0 +1,180 @@ +"""Summarize the raw comparison without changing any comparison result.""" + +from __future__ import annotations + +import json +import sys +from collections import Counter +from pathlib import Path + +from tests.conformance.vllm_reference import same_projection +from tests.parity import MODEL_CATALOG + + +def differences(record): + if "error" in record: + return ["reference-or-fixture-error"] + reference = record["vllm"] + streams = list(reference["stream"].values()) + expected = streams[0] if streams else reference["complete"] + categories = [] + if reference.get("complete_error") or reference.get("stream_errors"): + categories.append("parser-phase-error") + if reference.get("decode_agrees") is False: + categories.append("serving-detokenization") + if any(not same_projection(value, expected) for value in streams): + categories.append("stream-chunk-dependence") + if not same_projection(reference["complete"], expected): + categories.append("complete-versus-stream") + native = record["native"] + if ( + expected is not None + and record["scenario"].startswith("template-") + and native["reasoning"] is not None + and expected["reasoning"] is None + and any( + marker in expected["content"] + for marker in ("", "") + ) + ): + categories.append("template-channel-configuration") + if expected is not None and (native["reasoning"], native["content"]) != ( + expected["reasoning"], + expected["content"], + ): + categories.append("channel-bytes") + if expected is not None and not same_projection( + native["tool_calls"], expected["tool_calls"] + ): + categories.append("tool-projection") + if any(a["status"] != "ok" for a in record["attempts"]): + categories.append("malformed-attempt-metadata") + return categories + + +POLICY = { + "template-channel-configuration": { + "authority": "explicit-official-template-markers", + "decision": "Honor explicit reasoning markers emitted by the official template. A disabled or incompatible serving parser profile must be aligned with that grammar, rather than displaying valid reasoning as final text with think tags.", + }, + "generation-prefill": { + "authority": "unmodified-official-encoder", + "decision": "Use the encoder's actual generation prefix. Remove GPT-OSS's unconfigured forced-analysis suffix; do not change the oracle to imitate the renderer.", + }, + "parser-phase-error": { + "authority": "official-generation-prefill", + "decision": "Keep complete and streaming phase failures visible. The actual prompt must be supported. GPT-OSS should use the official bare assistant prefill rather than an implicit forced-analysis profile that the serving parser does not initialize.", + }, + "serialization": { + "authority": "pinned-official-chat-template", + "decision": "Match token IDs exactly, including generation prefills and turn closers. Bridges preserve sampled IDs and append only the encoder's boundary suffix.", + }, + "channel-bytes": { + "authority": "vllm-stream-with-exact-prompt", + "decision": "Preserve nonempty channel bytes; do not strip whitespace to make a comparison pass. An explicit channel marker and the actual generation prefix determine reasoning state.", + }, + "complete-versus-stream": { + "authority": "live-stream-with-exact-prompt", + "decision": "Prefer the prompt-aware streaming result for live completions. Keep the complete-parser result visible as a separate upstream discrepancy.", + }, + "stream-chunk-dependence": { + "authority": "template-protocol", + "decision": "Reject chunk-dependent behavior. No arbitrary chunk size is a trustworthy oracle; resolve the upstream bug against its fixture and the official template before accepting native equivalence.", + }, + "serving-detokenization": { + "authority": "sampled-token-bytes", + "decision": "Preserve sampled IDs. Report serving-inserted spaces or withheld incomplete Unicode separately; configure serving to avoid inserted spaces rather than teaching the native parser to invent bytes.", + }, + "tool-projection": { + "authority": "official-tool-format-and-schema", + "decision": "Preserve string argument bytes and compare valid arguments as JSON values. Follow vLLM's grammar for valid calls, but distinguish streamed partial intents from complete executable calls.", + }, + "malformed-attempt-metadata": { + "authority": "complete-call-validity", + "decision": "Retain malformed attempts and their raw text/status. Partial JSON, unclosed blocks, missing names, and unknown tools must not become executable merely because a serving parser emitted a delta.", + }, + "reference-or-fixture-error": { + "authority": "reproducible-reference", + "decision": "Fail the audit; an unavailable or misconfigured reference is not evidence of fidelity.", + }, +} + + +def summarize(result): + formats = {case.model: case.resolved_renderer for case in MODEL_CATALOG} + rows = result["cases"] + prefills = result.get("generation_prefills", []) + counts = Counter(category for row in rows for category in differences(row)) + counts["generation-prefill"] = sum(not r["agrees"] for r in prefills) + ref = result["reference"] + lines = [ + "# Renderer fidelity audit", + "", + f"Reference: vLLM {ref['version']}, source `{ref['commit']}`.", + "", + f"{len(rows)} comparisons; {sum(not r['agrees'] for r in rows)} raw projection disagreements.", + "A disagreement is reported even when the recommended contracts differ intentionally. No output is normalized or exempted from the strict comparison.", + f"Unmodified encoder prefills: {len(prefills)} comparisons, {sum(not r['agrees'] for r in prefills)} disagreements.", + "", + "| Format | Comparisons | Raw disagreements |", + "| --- | ---: | ---: |", + ] + for name in sorted(set(formats.values())): + group = [r for r in rows if formats[r["model"]] == name] + lines.append( + f"| {name} | {len(group)} | {sum(not r['agrees'] for r in group)} |" + ) + lines += ["", "## Authority decisions", ""] + for category, policy in POLICY.items(): + lines += [ + f"- **{category}** ({counts.get(category, 0)} cases): {policy['decision']} Authority: `{policy['authority']}`." + ] + lines += ["", "## Representative inputs and outputs", ""] + if any(not row["agrees"] for row in prefills): + row = next(row for row in prefills if not row["agrees"]) + shown = { + key: row[key] + for key in ("model", "settings", "native_tail", "reference_tail") + } + lines += [ + "### generation-prefill", + "", + "```json", + json.dumps(shown, indent=2), + "```", + "", + ] + for category in counts: + if category == "generation-prefill": + continue + row = next(r for r in rows if category in differences(r)) + shown = { + key: row[key] + for key in ( + "case", + "prompt_tail", + "prompt_profile", + "settings", + "text", + "native", + "reasoning_complete", + "attempts", + "vllm", + "error", + ) + if key in row + } + lines += [ + f"### {category}", + "", + "```json", + json.dumps(shown, indent=2, ensure_ascii=False), + "```", + "", + ] + return "\n".join(lines) + + +if __name__ == "__main__": + result = json.loads(Path(sys.argv[1]).read_text()) + Path(sys.argv[2]).write_text(summarize(result)) diff --git a/tests/conformance/run_upstream.py b/tests/conformance/run_upstream.py new file mode 100644 index 000000000..80407d7f1 --- /dev/null +++ b/tests/conformance/run_upstream.py @@ -0,0 +1,67 @@ +"""Run unmodified upstream tests against the installed, pinned runtime.""" + +import argparse +import importlib.metadata +import sys +from pathlib import Path + +import pytest +import vllm.parser # noqa: F401 -- fix the runtime path BEFORE adding upstream tests +from export_vllm import VLLM_VERSION, verify_source + +options = argparse.ArgumentParser() +options.add_argument("source", type=Path) +options.add_argument( + "--suite", choices=("engine", "reasoning", "tools"), default="engine" +) +args, extra = options.parse_known_args() +assert importlib.metadata.version("vllm") == VLLM_VERSION +source = args.source.resolve() +verify_source(source) +sys.path.insert(0, str(source)) +if args.suite == "engine": + directory = source / "tests/parser/engine" + paths = [str(directory)] +else: + directory = source / ( + "tests/reasoning" if args.suite == "reasoning" else "tests/tool_parsers" + ) + families = ( + ( + "qwen3", + "deepseekr1", + "deepseekv3", + "minimax_m2", + "kimi_k2", + "gptoss", + "hy_v3", + "nemotron_v3", + "gemma4", + "glm4_moe", + ) + if args.suite == "reasoning" + else ( + "qwen3coder", + "hermes", + "minimax_m2", + "glm47_moe", + "deepseekv3", + "deepseekv4", + "kimi_k2", + "hy_v3", + "gemma4", + "poolside_v1", + ) + ) + suffix = "reasoning_parser" if args.suite == "reasoning" else "tool_parser" + paths = [str(directory / f"test_{name}_{suffix}.py") for name in families] +raise SystemExit( + pytest.main( + [ + *paths, + f"--confcutdir={directory}", + "-q", + *extra, + ] + ) +) diff --git a/tests/conformance/test_rollouts.py b/tests/conformance/test_rollouts.py new file mode 100644 index 000000000..2bd2a621d --- /dev/null +++ b/tests/conformance/test_rollouts.py @@ -0,0 +1,262 @@ +"""Production client + actual generation boundary + three successive turns.""" + +import asyncio + +import pytest + +from renderers.client import generate +from tests.conformance.harness import ( + TokenBackend, + assert_attribution, + assert_tokens, + reference, + renderer, + tokenizer, +) +from tests.parity import MODEL_CATALOG, kwarg_combinations + + +def matrix(): + for case in MODEL_CATALOG: + for kwargs in kwarg_combinations(case): + # These knobs accept pre-serialized assistant bodies, rather than + # the structured assistant messages returned by generate(). Their + # complete matrix is exercised by test_finished_trace. + if ( + kwargs.get("is_training") + or kwargs.get("render_assistant_messages_raw") + or kwargs.get("raw_last_assistant") + ): + continue + yield pytest.param(case, kwargs, id=f"{case.model}/{kwargs!r}") + + +def canonical_turn(case, kwargs, native, history): + prompt = reference(case, history, kwargs, add_generation_prompt=True) + # Some templates require an open reasoning channel, others prefill a closed + # channel. Select a message representable at that exact reference boundary. + # The generation prompt is NEVER replaced with a historical-message prefix. + for assistant in ( + {"role": "assistant", "content": "Four.", "reasoning_content": "Compute."}, + {"role": "assistant", "content": "Four."}, + ): + finished = reference(case, history + [assistant], kwargs) + if finished[: len(prompt)] == prompt: + break + else: + if case.resolved_renderer == "deepseek-r1": + # R1 deliberately removes ALL historical reasoning, so historical + # serialization cannot produce a live completion. This is its + # official prefill + vLLM's deepseek_r1 output grammar. + completion = tokenizer(case.model).encode( + "Compute.Four.<｜end▁of▁sentence｜>", add_special_tokens=False + ) + return prompt, completion, assistant, "Compute.", "Four." + if case.resolved_renderer == "gpt-oss": + # Harmony's live prefix is analysis; a finished content-only + # history is final. Use the official Harmony channel transition. + completion = tokenizer(case.model).encode( + "Compute.<|end|><|start|>assistant<|channel|>final<|message|>Four.<|return|>", + add_special_tokens=False, + ) + return prompt, completion, assistant, "Compute.", "Four." + closed_channels = ("", "", "") + if ( + tokenizer(case.model) + .decode(prompt, skip_special_tokens=False) + .rstrip() + .endswith(closed_channels) + ): + assistant = {"role": "assistant", "content": "Four."} + finished = reference(case, history + [assistant], kwargs) + text = tokenizer(case.model).decode(finished, skip_special_tokens=False) + body = text[text.rindex("Four.") :] + completion = tokenizer(case.model).encode(body, add_special_tokens=False) + return finish_turn( + case, + kwargs, + native, + history, + prompt, + finished, + completion, + assistant, + None, + "Four.", + ) + assert_tokens(finished[: len(prompt)], prompt, case) + raise AssertionError( + "No canonical completion at the official generation boundary" + ) + + completion = finished[len(prompt) :] + reason = assistant.get("reasoning_content") + if ( + reference(case, history + [{"role": "assistant", "content": "Four."}], kwargs) + == finished + ): + reason = None # The reference template ignores this metadata field. + content = "Four." + if case.resolved_renderer == "glm-4.5": + reason = "\n" + reason if reason else None + content = "\n" + content + return finish_turn( + case, + kwargs, + native, + history, + prompt, + finished, + completion, + assistant, + reason, + content, + ) + + +def finish_turn( + case, + kwargs, + native, + history, + prompt, + finished, + completion, + assistant, + reason, + content, +): + stops = set(native.get_stop_token_ids()) + end = next( + (i + 1 for i in range(len(completion) - 1, -1, -1) if completion[i] in stops), + None, + ) + if end is not None: + completion = completion[:end] + else: + extended = reference( + case, history + [assistant, {"role": "user", "content": "Next."}], kwargs + ) + # A new query may drop historical reasoning, so its prefix need not + # equal the live turn. GLM samples the next user role as its stop. + stop = max(i for i, token in enumerate(extended) if token in stops) + completion += extended[stop : stop + 1] + return prompt, completion, assistant, reason, content + + +def expected_extension(case, kwargs, history, assistant, new, completion, native): + following = reference( + case, history + [assistant] + new, kwargs, add_generation_prompt=True + ) + # Perturb only the previous assistant body. The common suffix locates the + # actual turn-close boundary IN THE FOLLOWING REFERENCE RENDER, even when + # the template drops historical reasoning upon arrival of a new query. + probe = dict(assistant, content="BOUNDARY_PROBE_endXYZ") + other = reference(case, history + [probe] + new, kwargs, add_generation_prompt=True) + common = 0 + while ( + common < min(len(following), len(other)) + and following[-common - 1] == other[-common - 1] + ): + common += 1 + suffix = following[len(following) - common :] + stops = set(native.get_stop_token_ids()) + if case.resolved_renderer == "gpt-oss": + # Harmony history closes a message with end, whereas sampled final + # answers close the turn with return. The reference owns both forms. + stops.add(tokenizer(case.model).convert_tokens_to_ids("<|end|>")) + stop = next(i for i, token in enumerate(suffix) if token in stops) + return suffix[stop + 1 :] + + +@pytest.mark.parametrize("case,kwargs", tuple(matrix())) +def test_three_turn_client_rollout(case, kwargs): + native = renderer(case, kwargs) + history = [{"role": "user", "content": "What is two plus two?"}] + next_prompt = None + for turn in range(3): + prompt, completion, assistant, reasoning, content = canonical_turn( + case, kwargs, native, history + ) + if next_prompt is not None: + prompt = next_prompt.token_ids + backend = TokenBackend(prompt, completion) + result = asyncio.run( + generate( + client=backend, + renderer=native, + messages=history, + model=case.model, + prompt_ids=next_prompt.token_ids if next_prompt is not None else None, + prompt_attribution=next_prompt, + max_prompt_len=1_000_000, + ) + ) + assert result["prompt_ids"] == prompt + assert result["completion_ids"] == completion + assert result["completion_logprobs"] == [-0.5] * len(completion) + assert result["content"] == content + # Empty channel versus absent channel has the same OpenAI projection; + # do not normalize ANY nonempty reasoning or content whitespace. + assert (result["reasoning_content"] or None) == reasoning + assert result["reasoning_complete"] is ( + None if case.resolved_renderer == "default" else True + ) + assert not result["tool_calls"] + assert len(backend.requests) == 1 + assert_attribution(result["prompt_attribution"]) + # Reuse the PUBLIC parser result, just as a real conversation does. + # Reusing the fixture message here would hide parser/template drift. + assistant = {"role": "assistant", "content": result["content"]} + if result["reasoning_content"] is not None: + assistant["reasoning_content"] = result["reasoning_content"] + new = [{"role": "user", "content": f"Next query {turn}."}] + bridge = native.bridge_to_next_turn(prompt, completion, new) + extension = expected_extension( + case, kwargs, history, assistant, new, completion, native + ) + history += [assistant] + new + expected = reference(case, history, kwargs, add_generation_prompt=True) + if bridge is not None: + # For this canonical stream no serialization normalization is + # needed. Both prefix preservation AND complete template parity + # must hold, not just a plausible-looking decoded suffix. + assert ( + bridge.token_ids[: len(prompt) + len(completion)] == prompt + completion + ) + assert_tokens(bridge.token_ids, prompt + completion + extension, case) + assert_attribution(bridge) + assert not any(bridge.sampled_mask) + next_prompt = bridge + else: + next_prompt = native.render(history, add_generation_prompt=True) + assert_tokens(next_prompt.token_ids, expected, case) + + +@pytest.mark.parametrize("case,kwargs", tuple(matrix())) +def test_budget_stop_preserves_sampled_prefix(case, kwargs): + native = renderer(case, kwargs) + history = [{"role": "user", "content": "What is two plus two?"}] + prompt, completed, _, _, _ = canonical_turn(case, kwargs, native, history) + sampled = completed[: max(1, len(completed) // 2)] + backend = TokenBackend(prompt, sampled, finish="length") + result = asyncio.run( + generate( + client=backend, + renderer=native, + messages=history, + model=case.model, + max_prompt_len=1_000_000, + ) + ) + assert result["prompt_ids"] == prompt + assert result["completion_ids"] == sampled + assert result["completion_logprobs"] == [-0.5] * len(sampled) + assert result["finish_reason"] == "length" + bridge = native.bridge_to_next_turn( + prompt, sampled, [{"role": "user", "content": "Continue."}] + ) + if bridge is not None: + assert bridge.token_ids[: len(prompt) + len(sampled)] == prompt + sampled + assert_attribution(bridge) + assert not any(bridge.sampled_mask) diff --git a/tests/conformance/test_tool_cycles.py b/tests/conformance/test_tool_cycles.py new file mode 100644 index 000000000..192d71dcb --- /dev/null +++ b/tests/conformance/test_tool_cycles.py @@ -0,0 +1,208 @@ +"""Two tool cycles through the production client and exact reference bridges.""" + +import asyncio + +import pytest + +from renderers import config_from_name, create_renderer +from renderers.base import ToolCallParseStatus +from renderers.client import generate +from tests.conformance.harness import ( + TokenBackend, + assert_attribution, + assert_tokens, + reference, + renderer, + tokenizer, +) +from tests.parity import MODEL_CATALOG, TOOLS + + +def cases(): + for case in MODEL_CATALOG: + if "full-tool-cycle" not in case.excluded_scenarios: + yield pytest.param(case, id=case.model) + + +def tool_renderer(case, kwargs): + if case.resolved_renderer == "default": + return create_renderer( + tokenizer(case.model), + config_from_name("default").model_copy(update={"tool_parser": "qwen3"}), + ) + return renderer(case, kwargs) + + +def tool_turn(case, turn, city): + call_id = ( + f"functions.get_weather:{turn}" + if case.resolved_renderer in {"kimi-k2", "kimi-k2.5"} + else f"call_{turn}" + ) + assistant = { + "role": "assistant", + "content": "", + "reasoning_content": "Check weather.", + "tool_calls": [ + { + "id": call_id, + "type": "function", + "function": {"name": "get_weather", "arguments": {"city": city}}, + } + ], + } + new = [ + { + "role": "tool", + "name": "get_weather", + "tool_call_id": call_id, + "content": "Sunny.", + } + ] + return assistant, new + + +def reference_turn(case, native, history, assistant, new, kwargs): + prompt = reference(case, history, kwargs, tools=TOOLS, add_generation_prompt=True) + finished = reference(case, history + [assistant], kwargs, tools=TOOLS) + if finished[: len(prompt)] == prompt: + completion = finished[len(prompt) :] + else: + # A closed live prefill may be absent in history. Locate the start of + # the tool output by comparing two official completed serializations. + assistant = dict(assistant) + assistant.pop("reasoning_content", None) + finished = reference(case, history + [assistant], kwargs, tools=TOOLS) + empty = reference( + case, history + [{"role": "assistant", "content": ""}], kwargs, tools=TOOLS + ) + start = next( + (i for i, (a, b) in enumerate(zip(finished, empty)) if a != b), + min(len(finished), len(empty)), + ) + completion = finished[start:] + stops = set(native.get_stop_token_ids()) + end = next( + (i + 1 for i in range(len(completion) - 1, -1, -1) if completion[i] in stops), + None, + ) + if end is not None: + completion = completion[:end] + else: + extended = reference(case, history + [assistant] + new, kwargs, tools=TOOLS) + stop = max(i for i, token in enumerate(extended) if token in stops) + completion += extended[stop : stop + 1] + return prompt, completion, assistant + + +def tool_extension(case, native, history, assistant, new, kwargs): + following = reference( + case, + history + [assistant] + new, + kwargs, + tools=TOOLS, + add_generation_prompt=True, + ) + probe = dict( + assistant, + tool_calls=[ + { + "id": "call_probe", + "type": "function", + "function": { + "name": "get_weather", + "arguments": {"city": "BOUNDARY_PROBE_endXYZ"}, + }, + } + ], + ) + other = reference( + case, history + [probe] + new, kwargs, tools=TOOLS, add_generation_prompt=True + ) + common = 0 + while ( + common < min(len(following), len(other)) + and following[-common - 1] == other[-common - 1] + ): + common += 1 + suffix = following[len(following) - common :] + stops = set(native.get_stop_token_ids()) + stop = next(i for i, token in enumerate(suffix) if token in stops) + return suffix[stop + 1 :] + + +@pytest.mark.parametrize("case", tuple(cases())) +@pytest.mark.parametrize("city", ["Paris", "", " Paris\nMünchen ☁ "]) +def test_two_tool_cycles(case, city): + kwargs = {"is_training": False} if case.resolved_renderer == "hy3" else {} + native = tool_renderer(case, kwargs) + history = [ + {"role": "system", "content": "Use the weather tool."}, + {"role": "user", "content": "Weather?"}, + ] + next_prompt = None + for turn in range(2): + assistant, new = tool_turn(case, turn, city) + prompt, completion, assistant = reference_turn( + case, native, history, assistant, new, kwargs + ) + if next_prompt is not None: + prompt = next_prompt.token_ids + backend = TokenBackend(prompt, completion) + result = asyncio.run( + generate( + client=backend, + renderer=native, + messages=history, + model=case.model, + tools=TOOLS, + max_prompt_len=1_000_000, + prompt_ids=next_prompt.token_ids if next_prompt is not None else None, + prompt_attribution=next_prompt, + ) + ) + assert result["prompt_ids"] == prompt + assert result["completion_ids"] == completion + assert result["completion_logprobs"] == [-0.5] * len(completion) + assert len(result["tool_calls"]) == 1 + call = result["tool_calls"][0] + assert call.status == ToolCallParseStatus.OK + assert call.name == "get_weather" + assert call.arguments == {"city": city} + assert result["finish_reason"] == "tool_calls" + assistant = { + "role": "assistant", + "content": result["content"], + "reasoning_content": result["reasoning_content"], + "tool_calls": [ + { + "id": call.id or new[0]["tool_call_id"], + "type": "function", + "function": {"name": call.name, "arguments": call.arguments}, + } + ], + } + new[0]["tool_call_id"] = assistant["tool_calls"][0]["id"] + bridge = native.bridge_to_next_turn(prompt, completion, new, tools=TOOLS) + if bridge is not None: + extension = tool_extension(case, native, history, assistant, new, kwargs) + assert_tokens(bridge.token_ids, prompt + completion + extension, case) + assert_attribution(bridge) + assert not any(bridge.sampled_mask) + next_prompt = bridge + else: + next_prompt = native.render( + history + [assistant] + new, tools=TOOLS, add_generation_prompt=True + ) + assert_tokens( + next_prompt.token_ids, + reference( + case, + history + [assistant] + new, + kwargs, + tools=TOOLS, + add_generation_prompt=True, + ), + case, + ) + history += [assistant] + new diff --git a/tests/conformance/test_traces.py b/tests/conformance/test_traces.py new file mode 100644 index 000000000..5068d1bba --- /dev/null +++ b/tests/conformance/test_traces.py @@ -0,0 +1,64 @@ +"""Completed traces exercise render(), including attribution, against the encoder.""" + +import json +from pathlib import Path +from typing import get_args + +import pytest + +from renderers.configs import RendererConfig +from tests.conformance.harness import ( + assert_attribution, + assert_tokens, + reference, + renderer, + unmodified_generation_reference, +) +from tests.conformance.test_rollouts import matrix as inference_matrix +from tests.parity import MODEL_CATALOG, SCENARIOS, kwarg_combinations, scenario_is_valid + + +def matrix(): + for case in MODEL_CATALOG: + for kwargs in kwarg_combinations(case): + for scenario in SCENARIOS: + if scenario_is_valid(case, scenario, kwargs): + yield pytest.param( + case, + kwargs, + scenario, + id=f"{case.model}/{scenario.id}/{kwargs!r}", + ) + + +def test_every_renderer_has_a_reference_checkpoint(): + union = get_args(RendererConfig)[0] + declared = {cls.model_fields["name"].default for cls in get_args(union)} + # Factory choices are not wire formats. + assert declared - {"auto", "custom"} == { + case.resolved_renderer for case in MODEL_CATALOG + } + pins = json.loads(Path(__file__).with_name("tokenizers.json").read_text()) + assert set(pins) == {case.model for case in MODEL_CATALOG} + assert all(len(pin["revision"]) == 40 for pin in pins.values()) + + +@pytest.mark.parametrize("case,kwargs,scenario", tuple(matrix())) +def test_finished_trace(case, kwargs, scenario): + native = renderer(case, kwargs) + messages = [dict(message) for message in scenario.messages] + rendered = native.render(messages, **scenario.render_kwargs) + expected = reference(case, messages, kwargs, **scenario.render_kwargs) + assert_tokens(rendered.token_ids, expected, case) + assert_attribution(rendered) + assert_tokens(native.render_ids(messages, **scenario.render_kwargs), expected, case) + + +@pytest.mark.parametrize("case,kwargs", tuple(inference_matrix())) +def test_unmodified_generation_prefill(case, kwargs): + messages = [{"role": "user", "content": "Test."}] + assert_tokens( + renderer(case, kwargs).render_ids(messages, add_generation_prompt=True), + unmodified_generation_reference(case, messages, kwargs), + case, + ) diff --git a/tests/conformance/tokenizers.json b/tests/conformance/tokenizers.json new file mode 100644 index 000000000..822d0e1ac --- /dev/null +++ b/tests/conformance/tokenizers.json @@ -0,0 +1,182 @@ +{ + "Qwen/Qwen3-8B": { + "source": "Qwen/Qwen3-8B", + "revision": "b968826d9c46dd6066d109eabc6255188de91218" + }, + "PrimeIntellect/Qwen3-0.6B": { + "source": "PrimeIntellect/Qwen3-0.6B", + "revision": "15aba67c8ec68aeac96f57ba7e31d373564ee03c" + }, + "PrimeIntellect/Qwen3-1.7B": { + "source": "PrimeIntellect/Qwen3-1.7B", + "revision": "61a79cd101170642a653d1d44d18f69f88f54e08" + }, + "Qwen/Qwen3.5-0.8B": { + "source": "Qwen/Qwen3.5-0.8B", + "revision": "2fc06364715b967f1860aea9cf38778875588b17" + }, + "Qwen/Qwen3.5-2B": { + "source": "Qwen/Qwen3.5-2B", + "revision": "15852e8c16360a2fea060d615a32b45270f8a8fc" + }, + "Qwen/Qwen3.5-4B": { + "source": "Qwen/Qwen3.5-4B", + "revision": "851bf6e806efd8d0a36b00ddf55e13ccb7b8cd0a" + }, + "Qwen/Qwen3.5-9B": { + "source": "Qwen/Qwen3.5-9B", + "revision": "c202236235762e1c871ad0ccb60c8ee5ba337b9a" + }, + "Qwen/Qwen3.5-35B-A3B": { + "source": "Qwen/Qwen3.5-35B-A3B", + "revision": "59d61f3ce65a6d9863b86d2e96597125219dc754" + }, + "Qwen/Qwen3.5-122B-A10B": { + "source": "Qwen/Qwen3.5-122B-A10B", + "revision": "dc4d348443bc740c68e2d77492492c11606384d5" + }, + "Qwen/Qwen3.5-397B-A17B": { + "source": "Qwen/Qwen3.5-397B-A17B", + "revision": "8472618112abcbd45acbcdc58436aff4233c23f7" + }, + "Qwen/Qwen3.6-35B-A3B": { + "source": "Qwen/Qwen3.6-35B-A3B", + "revision": "995ad96eacd98c81ed38be0c5b274b04031597b0" + }, + "Qwen/Qwen3.8-27B": { + "source": "Qwen/Qwen3.8-27B", + "revision": "1d4bf0f2ff6012fd82039f2fa52739d0dd7c60c0" + }, + "Qwen/Qwen3.8-Flash-Next": { + "source": "Qwen/Qwen3.8-Flash-Next", + "revision": "de4b8e4d43b917e7706784d8bb445c9af86a3540" + }, + "Qwen/Qwen3-VL-4B-Instruct": { + "source": "Qwen/Qwen3-VL-4B-Instruct", + "revision": "ebb281ec70b05090aa6165b016eac8ec08e71b17" + }, + "google/gemma-4-E2B-it": { + "source": "google/gemma-4-E2B-it", + "revision": "3e22461f65e89153144f8adb70e3b8c2cc9845a7" + }, + "google/gemma-4-E4B-it": { + "source": "google/gemma-4-E4B-it", + "revision": "ee0ef6023621cff504d758262d4e04895a5af4a2" + }, + "google/gemma-4-26B-A4B-it": { + "source": "google/gemma-4-26B-A4B-it", + "revision": "4d7ae4984b7db7de8f8457170b3f1a419ee76d52" + }, + "google/gemma-4-31B-it": { + "source": "google/gemma-4-31B-it", + "revision": "842da3794eaa0b77d5f08bae87a17459d91ff475" + }, + "zai-org/GLM-5": { + "source": "zai-org/GLM-5", + "revision": "c183ef8c61faee82855eca1ed9bb3a9a7ce3b0b2" + }, + "zai-org/GLM-5.1": { + "source": "zai-org/GLM-5.1", + "revision": "26e1bd6e011feb778d25ae34b09b07074139d92d" + }, + "zai-org/GLM-5.3": { + "source": "zai-org/GLM-5.3", + "revision": "aca966e4e02791568aa6a4ced368624b3d897f42" + }, + "zai-org/GLM-4.7-Flash": { + "source": "zai-org/GLM-4.7-Flash", + "revision": "7dd20894a642a0aa287e9827cb1a1f7f91386b67" + }, + "THUDM/GLM-4.5-Air": { + "source": "THUDM/GLM-4.5-Air", + "revision": "a24ceef6ce4f3536971efe9b778bdaa1bab18daa" + }, + "MiniMaxAI/MiniMax-M2.5": { + "source": "MiniMaxAI/MiniMax-M2.5", + "revision": "f710177d938eff80b684d42c5aa84b382612f21f" + }, + "moonshotai/Kimi-K2-Instruct": { + "source": "moonshotai/Kimi-K2-Instruct", + "revision": "fd1984e2b7a3350dbf7305fe73a4ede25c14de50" + }, + "moonshotai/Kimi-K2.5": { + "source": "moonshotai/Kimi-K2.5", + "revision": "4d01dfe0332d63057c186e0b262165819efb6611" + }, + "moonshotai/Kimi-K2.6": { + "source": "moonshotai/Kimi-K2.6", + "revision": "2755962d07cb42aa2d988a35bcb65cd4a9c2de82" + }, + "deepseek-ai/DeepSeek-V3": { + "source": "deepseek-ai/DeepSeek-V3", + "revision": "e815299b0bcbac849fa540c768ef21845365c9eb" + }, + "deepseek-ai/DeepSeek-R1": { + "source": "deepseek-ai/DeepSeek-R1", + "revision": "56d4cbbb4d29f4355bab4b9a39ccb717a14ad5ad" + }, + "deepseek-ai/DeepSeek-V4-Flash-0731": { + "source": "deepseek-ai/DeepSeek-V4-Flash-0731", + "revision": "7872f01b1d1fe23eabc4c98b48bffcef5a386062" + }, + "nvidia/NVIDIA-Nemotron-3-Nano-30B-A3B-BF16": { + "source": "nvidia/NVIDIA-Nemotron-3-Nano-30B-A3B-BF16", + "revision": "bf77c3174f68ad409e1c2aa60daeb46e32d1c606" + }, + "nvidia/NVIDIA-Nemotron-3-Super-120B-A12B-BF16": { + "source": "nvidia/NVIDIA-Nemotron-3-Super-120B-A12B-BF16", + "revision": "2dc98e2afe4face0e4ce40972a915c45368bd34a" + }, + "nvidia/NVIDIA-Nemotron-3-Ultra-550B-A55B-BF16": { + "source": "nvidia/NVIDIA-Nemotron-3-Ultra-550B-A55B-BF16", + "revision": "77df655d5e9f8362164ed14dd8b48f8bce657498" + }, + "nvidia/NVIDIA-Nemotron-3.5-Lightning-30B-A3B-BF16": { + "source": "nvidia/NVIDIA-Nemotron-3.5-Lightning-30B-A3B-BF16", + "revision": "a9904d24bcc1d289a1950fa9d2b978c47cf903b9" + }, + "poolside/Laguna-XS.2": { + "source": "poolside/Laguna-XS.2", + "revision": "a397bde04501591d0ac3d74a5d9d0a4ec298ffb8" + }, + "poolside/Laguna-M.1": { + "source": "poolside/Laguna-M.1", + "revision": "2bf8a4ab6d2e8e1e0d92a41318532a0fee1640d6" + }, + "poolside/Laguna-XS-2.1": { + "source": "poolside/Laguna-XS-2.1", + "revision": "c5f36269bbdbd3f27fddc9a9f9dbae0cf2cf57db" + }, + "poolside/Laguna-S-2.1": { + "source": "poolside/Laguna-S-2.1", + "revision": "0f573140834b11cfac0c2af97a101a7a69a13e22" + }, + "meta-llama/Llama-3.2-1B-Instruct": { + "source": "unsloth/Llama-3.2-1B-Instruct", + "revision": "5a8abab4a5d6f164389b1079fb721cfab8d7126c" + }, + "meta-llama/Llama-3.2-3B-Instruct": { + "source": "unsloth/Llama-3.2-3B-Instruct", + "revision": "006f5dcd1393c3add266de40994ba96225e9689d" + }, + "openai/gpt-oss-20b": { + "source": "openai/gpt-oss-20b", + "revision": "6cee5e81ee83917806bbde320786a8fb61efebee" + }, + "tencent/Hy3": { + "source": "tencent/Hy3", + "revision": "a960ebc3da325ba167f069f76c41eb62c9280d22" + }, + "thinkingmachines/Inkling": { + "source": "thinkingmachines/Inkling", + "revision": "828496eeae4c243ff1a22f7f28ff83694f2f7bc9" + }, + "thinkingmachines/Inkling-Small": { + "source": "thinkingmachines/Inkling-Small", + "revision": "8cc5877b44d343f88b92086aa1fb72897950f06a" + }, + "Qwen/Qwen2.5-0.5B-Instruct": { + "source": "Qwen/Qwen2.5-0.5B-Instruct", + "revision": "7ae557604adf67be50417f59c2c2f167def9a775" + } +} diff --git a/tests/conformance/upstream_tokenizers.json b/tests/conformance/upstream_tokenizers.json new file mode 100644 index 000000000..72ba0979f --- /dev/null +++ b/tests/conformance/upstream_tokenizers.json @@ -0,0 +1,13 @@ +{ + "Qwen/Qwen3-0.6B": "c1899de289a04d12100db370d81485cdf75e47ca", + "Qwen/Qwen3-4B-Thinking-2507": "768f209d9ea81521153ed38c47d515654e938aea", + "deepseek-ai/DeepSeek-V3.1": "c0781d039fb7a1ba2abc4add0bdc293e92d2b8db", + "deepseek-ai/DeepSeek-R1-Distill-Qwen-1.5B": "ad9f0ae0864d7fbcd1cd905e3c6c5b069cc8b562", + "MiniMaxAI/MiniMax-M2": "757303d492a50514c312788b5247a4f696a4c6a3", + "zai-org/GLM-4.7": "602d01efcdd332c5238ca4bcede555defbe83eb7", + "tencent/Hy3-preview": "549c2b3a0fd5b9a6c6059a9935bf0d59ab69d75a", + "Qwen/Qwen3-Coder-30B-A3B-Instruct-FP8": "dcaee4d4dfc5ee71ad501f01f530e5652438fde0", + "openai-community/gpt2": "607a30d783dfa663caf39e06633721c8d4cfcd7e", + "Qwen/Qwen3-32B": "9216db5781bf21249d130ec9da846c4624c16137", + "deepseek-ai/DeepSeek-V3": "e815299b0bcbac849fa540c768ef21845365c9eb" +} diff --git a/tests/conformance/vllm_reference.py b/tests/conformance/vllm_reference.py new file mode 100644 index 000000000..24d924fd8 --- /dev/null +++ b/tests/conformance/vllm_reference.py @@ -0,0 +1,185 @@ +"""Actual vLLM public parsers, isolated from portable tests (Linux runtime). + +This module never substitutes parser classes, tokenizer IDs, or imports. +Streaming uses vLLM's own incremental detokenizer, including byte fragments. +""" + +from __future__ import annotations + +import importlib.metadata +import json + +from tests.conformance.export_vllm import VLLM_VERSION + +# (tool parser, reasoning parser), using serving CLI names, not private engines. +PARSERS = { + "default": (None, None), + "qwen3": ("hermes", "qwen3"), + "qwen3-vl": ("hermes", "qwen3"), + "prime-qwen3": ("qwen3_xml", "qwen3"), + "qwen3.5": ("qwen3_xml", "qwen3"), + "qwen3.6": ("qwen3_xml", "qwen3"), + "qwen3.8": ("qwen3_xml", "qwen3"), + "glm-4.5": ("glm47", "glm47"), + "glm-5": ("glm47", "glm47"), + "glm-5.1": ("glm47", "glm47"), + "glm-5.3": ("glm47", "glm47"), + "deepseek-v3": ("deepseek_v3", None), + "deepseek-r1": ("deepseek_v3", "deepseek_r1"), + "deepseek-v4": ("deepseek_v4", "deepseek_v4"), + "minimax-m2": ("minimax_m2", "minimax_m2"), + "kimi-k2": ("kimi_k2", "kimi_k2"), + "kimi-k2.5": ("kimi_k2", "kimi_k2"), + "nemotron-3": ("qwen3_xml", "nemotron_v3"), + "nemotron-3-ultra": ("qwen3_xml", "nemotron_v3"), + "nemotron-3.5": ("qwen3_xml", "nemotron_v3"), + "gemma4": ("gemma4", "gemma4"), + "hy3": ("hy_v3", "hy_v3"), + "laguna-xs.2": ("poolside_v1", "poolside_v1"), + "laguna-m.1": ("poolside_v1", "poolside_v1"), + "laguna-xs-2.1": ("poolside_v1", "poolside_v1"), + "laguna-s-2.1": ("poolside_v1", "poolside_v1"), + "llama-3": ("llama3_json", None), + "gpt-oss": ("openai", "openai_gptoss"), + "inkling": ("inkling", "inkling"), +} + + +def arguments(value): + if isinstance(value, str): + try: + return json.loads(value) + except json.JSONDecodeError: + return value # Keep unfinished argument bytes visible in the report. + return value + + +def projection(reasoning, content, calls): + """OpenAI projection; no whitespace stripping or argument coercion.""" + return { + "reasoning": reasoning or None, + "content": content or "", + "tool_calls": [ + {"name": call["name"], "arguments": arguments(call["arguments"])} + for call in calls or [] + ], + } + + +def same_projection(left, right): + # Python equality conflates True with 1 (and 1 with 1.0). JSON preserves + # argument types while ignoring object key order. + return json.dumps(left, sort_keys=True, allow_nan=False) == json.dumps( + right, sort_keys=True, allow_nan=False + ) + + +def parse_reference(case, tok, prompt, completion, tools, kwargs): + assert importlib.metadata.version("vllm") == VLLM_VERSION + from vllm.entrypoints.openai.chat_completion.protocol import ChatCompletionRequest + from vllm.parser import ParserManager + from vllm.tokenizers.detokenizer_utils import detokenize_incrementally + + # Renderer config None means "defer to the template default", exactly as + # reference() treats it. Passing a literal None to a vLLM parser instead + # would disable thinking without changing the actual generation prompt. + kwargs = {key: value for key, value in kwargs.items() if value is not None} + if case.model == "moonshotai/Kimi-K2-Instruct": + # The Instruct checkpoint's official prefill opens content, unlike + # the K2.5/K2.6 thinking checkpoint. This is a serving profile choice. + kwargs = dict(kwargs, thinking=False) + + tool_name, reasoning_name = PARSERS[case.resolved_renderer] + if case.resolved_renderer == "default" and tools: + tool_name = "hermes" # Explicit DefaultRenderer tool_parser="qwen3". + cls = ParserManager.get_parser( + tool_parser_name=tool_name, + reasoning_parser_name=reasoning_name, + enable_auto_tools=True, + model_name=case.model, + is_harmony=case.resolved_renderer == "gpt-oss", + ) + text = tok.decode(completion, skip_special_tokens=False) + if cls is None: + return {"complete": projection(None, text, []), "stream": {}} + request = ChatCompletionRequest( + model=case.model, + messages=[{"role": "user", "content": "test"}], + tools=tools or None, + chat_template_kwargs=kwargs or None, + skip_special_tokens=False, + ) + + def parser(): + return cls(tok, tools=request.tools, chat_template_kwargs=kwargs) + + token_texts = [] + previous = None + prefix_offset = read_offset = 0 + for i in range(len(completion)): + added, delta, prefix_offset, read_offset = detokenize_incrementally( + tok, + prompt + completion[: i + 1], + previous, + prefix_offset, + read_offset, + skip_special_tokens=False, + ) + previous = (previous or []) + added + token_texts.append(delta) + serving_text = "".join(token_texts) + outputs = { + "complete": None, + "stream": {}, + "stream_errors": {}, + "detokenized_text": serving_text, + "decode_agrees": serving_text == text, + "parser_kwargs": kwargs, + } + try: + reason, content, calls = parser().parse( + serving_text, + request, + enable_auto_tools=True, + model_output_token_ids=completion, + ) + outputs["complete"] = projection( + reason, content, [c.model_dump() for c in calls or []] + ) + except Exception as exc: + outputs["complete_error"] = f"{type(exc).__name__}: {exc}" + + def stream(chunk): + instance = parser() + reason_parts, content_parts, tool_parts = [], [], {} + for start in range(0, len(completion), chunk): + end = min(start + chunk, len(completion)) + delta = instance.parse_delta( + "".join(token_texts[start:end]), + completion[start:end], + request, + prompt_token_ids=prompt, + finished=end == len(completion), + ) + if delta is None: + continue + reason_parts.append(delta.reasoning or "") + content_parts.append(delta.content or "") + for tc in delta.tool_calls or []: + fn = tc.function + record = tool_parts.setdefault(tc.index, {"name": "", "arguments": ""}) + if fn: + record["name"] += fn.name or "" + record["arguments"] += fn.arguments or "" + return projection( + "".join(reason_parts), + "".join(content_parts), + [tool_parts[i] for i in sorted(tool_parts)], + ) + + for chunk in (1, 2, 5, len(completion) or 1): + try: + outputs["stream"][str(chunk)] = stream(chunk) + except Exception as exc: + outputs["stream_errors"][str(chunk)] = f"{type(exc).__name__}: {exc}" + return outputs