diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
index 34e81487..29c673cc 100644
--- a/.github/workflows/test.yml
+++ b/.github/workflows/test.yml
@@ -9,7 +9,7 @@ on:
- "uv.lock"
- ".github/workflows/test.yml"
pull_request:
- branches: [main]
+ branches: [main, "codex/**"]
paths:
- "**.py"
- "pyproject.toml"
diff --git a/tests/conformance/__init__.py b/tests/conformance/__init__.py
new file mode 100644
index 00000000..8169ee34
--- /dev/null
+++ b/tests/conformance/__init__.py
@@ -0,0 +1 @@
+"""Independent reference tests through the public renderer interface."""
diff --git a/tests/conformance/conftest.py b/tests/conformance/conftest.py
new file mode 100644
index 00000000..110bb4b5
--- /dev/null
+++ b/tests/conformance/conftest.py
@@ -0,0 +1,14 @@
+"""Release real tokenizer assets between conformance modules."""
+
+import gc
+
+import pytest
+
+from tests.conformance.harness import tokenizer
+
+
+@pytest.fixture(scope="module", autouse=True)
+def release_reference_assets():
+ yield
+ tokenizer.cache_clear()
+ gc.collect()
diff --git a/tests/conformance/encoder.py b/tests/conformance/encoder.py
new file mode 100644
index 00000000..2766149e
--- /dev/null
+++ b/tests/conformance/encoder.py
@@ -0,0 +1,87 @@
+"""Execute the reviewed, pinned model-supplied Python encoder directly."""
+
+import copy
+import hashlib
+import importlib.util
+import json
+from functools import cache
+from pathlib import Path
+
+from huggingface_hub import hf_hub_download
+
+from tests.model_assets import model_revision
+
+DEEPSEEK_ENCODER = {
+ "repo": "deepseek-ai/DeepSeek-V4-Flash-0731",
+ "file": "encoding/encoding_dsv4.py",
+ "sha256": "abc0d26120250dda0ae077dc64aa28836026e61e970854aaeb792445e6a0dde6",
+}
+
+
+@cache
+def encoder():
+ source = DEEPSEEK_ENCODER
+ path = hf_hub_download(
+ source["repo"],
+ source["file"],
+ revision=model_revision(source["repo"]),
+ )
+ assert hashlib.sha256(Path(path).read_bytes()).hexdigest() == source["sha256"]
+ spec = importlib.util.spec_from_file_location("pinned_deepseek_encoder", path)
+ assert spec is not None and spec.loader is not None
+ module = importlib.util.module_from_spec(spec)
+ spec.loader.exec_module(module)
+ return module
+
+
+def deepseek_reference(
+ tok,
+ messages,
+ *,
+ tools=None,
+ add_generation_prompt=False,
+ enable_thinking=False,
+ drop_thinking=True,
+ reasoning_effort="low",
+):
+ upstream = encoder()
+ messages = copy.deepcopy(messages)
+ if tools:
+ target = next((m for m in messages if m["role"] == "developer"), None)
+ if target is None:
+ target = next((m for m in messages if m["role"] == "system"), None)
+ if target is None:
+ target = {"role": "system", "content": ""}
+ messages.insert(0, target)
+ target["tools"] = tools
+ for message in messages:
+ for call in message.get("tool_calls") or []:
+ function = call["function"]
+ if not isinstance(function["arguments"], str):
+ function["arguments"] = json.dumps(
+ function["arguments"], ensure_ascii=False
+ )
+ text = upstream.encode_messages(
+ messages,
+ thinking_mode="thinking" if enable_thinking else "chat",
+ drop_thinking=drop_thinking,
+ reasoning_effort=reasoning_effort,
+ )
+ # Upstream's API always opens generation after a terminal query. The
+ # renderer API also exposes a completed-transcript view. Remove only that
+ # terminal generation prefill; all internal transitions remain upstream's.
+ if not add_generation_prompt:
+ merged = upstream.merge_tool_messages(messages)
+ if (
+ merged
+ and merged[-1]["role"] in {"user", "developer"}
+ and not merged[-1].get("task")
+ ):
+ suffix = upstream.ASSISTANT_SP_TOKEN + (
+ upstream.thinking_start_token
+ if enable_thinking
+ else upstream.thinking_end_token
+ )
+ assert text.endswith(suffix)
+ text = text[: -len(suffix)]
+ return tok.encode(text, add_special_tokens=False)
diff --git a/tests/conformance/harness.py b/tests/conformance/harness.py
new file mode 100644
index 00000000..d61eb659
--- /dev/null
+++ b/tests/conformance/harness.py
@@ -0,0 +1,102 @@
+"""No renderer internals: official encoder → token backend → public client."""
+
+from __future__ import annotations
+
+import json
+from functools import lru_cache
+from typing import Any
+
+import httpx
+
+from renderers import config_from_name, create_renderer
+from tests.conformance.encoder import deepseek_reference
+from tests.model_assets import load_test_tokenizer
+from tests.parity import ModelCase
+from tests.reference_rendering import render_reference
+
+
+@lru_cache(maxsize=1)
+def tokenizer(model: str):
+ return load_test_tokenizer(model)
+
+
+def renderer(case: ModelCase, kwargs: dict[str, Any]):
+ return create_renderer(
+ tokenizer(case.model),
+ config_from_name(case.resolved_renderer),
+ chat_template_kwargs=kwargs or None,
+ )
+
+
+def reference(case: ModelCase, messages: list, kwargs: dict, **render_kwargs):
+ options = dict(case.oracle_defaults) | kwargs
+ options = {key: value for key, value in options.items() if value is not None}
+ if case.oracle == "deepseek-v4":
+ return deepseek_reference(
+ tokenizer(case.model), messages, **options, **render_kwargs
+ )
+ return render_reference(
+ tokenizer(case.model), messages, oracle=case.oracle, **options, **render_kwargs
+ )
+
+
+def assert_tokens(actual: list[int], expected: list[int], case: ModelCase):
+ if actual == expected:
+ return
+ first = next(
+ (i for i, (a, b) in enumerate(zip(actual, expected)) if a != b),
+ min(len(actual), len(expected)),
+ )
+ tok = tokenizer(case.model)
+ window = slice(max(0, first - 12), first + 24)
+ raise AssertionError(
+ f"{case.model}: first differing token {first}; "
+ f"lengths native={len(actual)}, reference={len(expected)}\n"
+ f"native: {tok.decode(actual[window], skip_special_tokens=False)!r}\n"
+ f"reference: {tok.decode(expected[window], skip_special_tokens=False)!r}"
+ )
+
+
+def assert_attribution(rendered):
+ size = len(rendered.token_ids)
+ for field in ("message_indices",):
+ assert len(getattr(rendered, field)) == size, field
+ assert len(rendered.sampled_mask) in {0, size}
+ assert len(rendered.is_content) in {0, size}
+
+
+class TokenBackend:
+ """Script sampling only. Rendering and parsing remain the real implementation."""
+
+ base_url = "http://conformance.invalid/v1"
+
+ def __init__(self, prompt: list[int], completion: list[int], finish="stop"):
+ self.prompt = prompt
+ self.completion = completion
+ self.finish = finish
+ self.requests: list[dict] = []
+
+ async def post(self, endpoint, *, body, cast_to, **kwargs):
+ assert endpoint == "http://conformance.invalid/inference/v1/generate"
+ assert body["token_ids"] == self.prompt
+ assert body["sampling_params"]["skip_special_tokens"] is False
+ self.requests.append(body)
+ return httpx.Response(
+ 200,
+ content=json.dumps(
+ {
+ "choices": [
+ {
+ "token_ids": self.completion,
+ "finish_reason": self.finish,
+ "logprobs": {
+ "content": [
+ {"token": f"token_id:{t}", "logprob": -0.5}
+ for t in self.completion
+ ]
+ },
+ }
+ ]
+ }
+ ).encode(),
+ )
diff --git a/tests/conformance/test_rollouts.py b/tests/conformance/test_rollouts.py
new file mode 100644
index 00000000..3470af14
--- /dev/null
+++ b/tests/conformance/test_rollouts.py
@@ -0,0 +1,267 @@
+"""Production client + actual generation boundary + three successive turns."""
+
+import asyncio
+
+import numpy as np
+import pytest
+
+from renderers.client import generate
+from tests.conformance.harness import (
+ TokenBackend,
+ assert_attribution,
+ assert_tokens,
+ reference,
+ renderer,
+ tokenizer,
+)
+from tests.parity import MODEL_CATALOG, kwarg_combinations
+
+pytestmark = pytest.mark.model
+
+
+def matrix():
+ for case in MODEL_CATALOG:
+ for kwargs in kwarg_combinations(case):
+ # These knobs accept pre-serialized assistant bodies, rather than
+ # the structured assistant messages returned by generate(). Their
+ # complete matrix is exercised by test_finished_trace.
+ if (
+ kwargs.get("is_training")
+ or kwargs.get("render_assistant_messages_raw")
+ or kwargs.get("raw_last_assistant")
+ ):
+ continue
+ yield pytest.param(case, kwargs, id=f"{case.model}/{kwargs!r}")
+
+
+def canonical_turn(case, kwargs, native, history):
+ prompt = reference(case, history, kwargs, add_generation_prompt=True)
+ # Some templates require an open reasoning channel, others prefill a closed
+ # channel. Select a message representable at that exact reference boundary.
+ # The generation prompt is NEVER replaced with a historical-message prefix.
+ for assistant in (
+ {"role": "assistant", "content": "Four.", "reasoning_content": "Compute."},
+ {"role": "assistant", "content": "Four."},
+ ):
+ finished = reference(case, history + [assistant], kwargs)
+ if finished[: len(prompt)] == prompt:
+ break
+ else:
+ if case.resolved_renderer == "deepseek-r1":
+ # R1 deliberately removes ALL historical reasoning, so historical
+ # serialization cannot produce a live completion. This is its
+ # official prefill + vLLM's deepseek_r1 output grammar.
+ completion = tokenizer(case.model).encode(
+ "Compute.Four.<｜end▁of▁sentence｜>", add_special_tokens=False
+ )
+ return prompt, completion, assistant, "Compute.", "Four."
+ if case.resolved_renderer == "gpt-oss":
+ # Harmony's live prefix is analysis; a finished content-only
+ # history is final. Use the official Harmony channel transition.
+ completion = tokenizer(case.model).encode(
+ "Compute.<|end|><|start|>assistant<|channel|>final<|message|>Four.<|return|>",
+ add_special_tokens=False,
+ )
+ return prompt, completion, assistant, "Compute.", "Four."
+ closed_channels = ("", "", "")
+ if (
+ tokenizer(case.model)
+ .decode(prompt, skip_special_tokens=False)
+ .rstrip()
+ .endswith(closed_channels)
+ ):
+ assistant = {"role": "assistant", "content": "Four."}
+ finished = reference(case, history + [assistant], kwargs)
+ text = tokenizer(case.model).decode(finished, skip_special_tokens=False)
+ body = text[text.rindex("Four.") :]
+ completion = tokenizer(case.model).encode(body, add_special_tokens=False)
+ return finish_turn(
+ case,
+ kwargs,
+ native,
+ history,
+ prompt,
+ finished,
+ completion,
+ assistant,
+ None,
+ "Four.",
+ )
+ assert_tokens(finished[: len(prompt)], prompt, case)
+ raise AssertionError(
+ "No canonical completion at the official generation boundary"
+ )
+
+ completion = finished[len(prompt) :]
+ reason = assistant.get("reasoning_content")
+ if (
+ reference(case, history + [{"role": "assistant", "content": "Four."}], kwargs)
+ == finished
+ ):
+ reason = None # The reference template ignores this metadata field.
+ content = "Four."
+ if case.resolved_renderer == "glm-4.5":
+ reason = "\n" + reason if reason else None
+ content = "\n" + content
+ return finish_turn(
+ case,
+ kwargs,
+ native,
+ history,
+ prompt,
+ finished,
+ completion,
+ assistant,
+ reason,
+ content,
+ )
+
+
+def finish_turn(
+ case,
+ kwargs,
+ native,
+ history,
+ prompt,
+ finished,
+ completion,
+ assistant,
+ reason,
+ content,
+):
+ stops = set(native.get_stop_token_ids())
+ end = next(
+ (i + 1 for i in range(len(completion) - 1, -1, -1) if completion[i] in stops),
+ None,
+ )
+ if end is not None:
+ completion = completion[:end]
+ else:
+ extended = reference(
+ case, history + [assistant, {"role": "user", "content": "Next."}], kwargs
+ )
+ # A new query may drop historical reasoning, so its prefix need not
+ # equal the live turn. GLM samples the next user role as its stop.
+ stop = max(i for i, token in enumerate(extended) if token in stops)
+ completion += extended[stop : stop + 1]
+ return prompt, completion, assistant, reason, content
+
+
+def expected_extension(case, kwargs, history, assistant, new, completion, native):
+ following = reference(
+ case, history + [assistant] + new, kwargs, add_generation_prompt=True
+ )
+ # Perturb only the previous assistant body. The common suffix locates the
+ # actual turn-close boundary IN THE FOLLOWING REFERENCE RENDER, even when
+ # the template drops historical reasoning upon arrival of a new query.
+ probe = dict(assistant, content="BOUNDARY_PROBE_endXYZ")
+ other = reference(case, history + [probe] + new, kwargs, add_generation_prompt=True)
+ common = 0
+ while (
+ common < min(len(following), len(other))
+ and following[-common - 1] == other[-common - 1]
+ ):
+ common += 1
+ suffix = following[len(following) - common :]
+ stops = set(native.get_stop_token_ids())
+ if case.resolved_renderer == "gpt-oss":
+ # Harmony history closes a message with end, whereas sampled final
+ # answers close the turn with return. The reference owns both forms.
+ stops.add(tokenizer(case.model).convert_tokens_to_ids("<|end|>"))
+ stop = next(i for i, token in enumerate(suffix) if token in stops)
+ return suffix[stop + 1 :]
+
+
+@pytest.mark.parametrize("case,kwargs", tuple(matrix()))
+def test_three_turn_client_rollout(case, kwargs):
+ native = renderer(case, kwargs)
+ history = [{"role": "user", "content": "What is two plus two?"}]
+ next_prompt = None
+ for turn in range(3):
+ prompt, completion, assistant, reasoning, content = canonical_turn(
+ case, kwargs, native, history
+ )
+ if next_prompt is not None:
+ prompt = next_prompt.token_ids
+ backend = TokenBackend(prompt, completion)
+ result = asyncio.run(
+ generate(
+ client=backend,
+ renderer=native,
+ messages=history,
+ model=case.model,
+ prompt_ids=next_prompt.token_ids if next_prompt is not None else None,
+ prompt_attribution=next_prompt,
+ max_prompt_len=1_000_000,
+ )
+ )
+ assert result["prompt_ids"] == prompt
+ assert result["completion_ids"] == completion
+ np.testing.assert_array_equal(
+ result["completion_logprobs"], [-0.5] * len(completion)
+ )
+ assert result["content"] == content
+ # Empty channel versus absent channel has the same OpenAI projection;
+ # do not normalize ANY nonempty reasoning or content whitespace.
+ assert (result["reasoning_content"] or None) == reasoning
+ assert result["reasoning_complete"] is (
+ None if case.resolved_renderer == "default" else True
+ )
+ assert not result["tool_calls"]
+ assert len(backend.requests) == 1
+ assert_attribution(result["prompt_attribution"])
+ # Reuse the PUBLIC parser result, just as a real conversation does.
+ # Reusing the fixture message here would hide parser/template drift.
+ assistant = {"role": "assistant", "content": result["content"]}
+ if result["reasoning_content"] is not None:
+ assistant["reasoning_content"] = result["reasoning_content"]
+ new = [{"role": "user", "content": f"Next query {turn}."}]
+ bridge = native.bridge_to_next_turn(prompt, completion, new)
+ extension = expected_extension(
+ case, kwargs, history, assistant, new, completion, native
+ )
+ history += [assistant] + new
+ expected = reference(case, history, kwargs, add_generation_prompt=True)
+ if bridge is not None:
+ # For this canonical stream no serialization normalization is
+ # needed. Both prefix preservation AND complete template parity
+ # must hold, not just a plausible-looking decoded suffix.
+ assert (
+ bridge.token_ids[: len(prompt) + len(completion)] == prompt + completion
+ )
+ assert_tokens(bridge.token_ids, prompt + completion + extension, case)
+ assert_attribution(bridge)
+ assert not any(bridge.sampled_mask)
+ next_prompt = bridge
+ else:
+ next_prompt = native.render(history, add_generation_prompt=True)
+ assert_tokens(next_prompt.token_ids, expected, case)
+
+
+@pytest.mark.parametrize("case,kwargs", tuple(matrix()))
+def test_budget_stop_preserves_sampled_prefix(case, kwargs):
+ native = renderer(case, kwargs)
+ history = [{"role": "user", "content": "What is two plus two?"}]
+ prompt, completed, _, _, _ = canonical_turn(case, kwargs, native, history)
+ sampled = completed[: max(1, len(completed) // 2)]
+ backend = TokenBackend(prompt, sampled, finish="length")
+ result = asyncio.run(
+ generate(
+ client=backend,
+ renderer=native,
+ messages=history,
+ model=case.model,
+ max_prompt_len=1_000_000,
+ )
+ )
+ assert result["prompt_ids"] == prompt
+ assert result["completion_ids"] == sampled
+ np.testing.assert_array_equal(result["completion_logprobs"], [-0.5] * len(sampled))
+ assert result["finish_reason"] == "length"
+ bridge = native.bridge_to_next_turn(
+ prompt, sampled, [{"role": "user", "content": "Continue."}]
+ )
+ if bridge is not None:
+ assert bridge.token_ids[: len(prompt) + len(sampled)] == prompt + sampled
+ assert_attribution(bridge)
+ assert not any(bridge.sampled_mask)
diff --git a/tests/conformance/test_tool_cycles.py b/tests/conformance/test_tool_cycles.py
new file mode 100644
index 00000000..07494ea8
--- /dev/null
+++ b/tests/conformance/test_tool_cycles.py
@@ -0,0 +1,213 @@
+"""Two tool cycles through the production client and exact reference bridges."""
+
+import asyncio
+
+import numpy as np
+import pytest
+
+from renderers import config_from_name, create_renderer
+from renderers.base import ToolCallParseStatus
+from renderers.client import generate
+from tests.conformance.harness import (
+ TokenBackend,
+ assert_attribution,
+ assert_tokens,
+ reference,
+ renderer,
+ tokenizer,
+)
+from tests.parity import MODEL_CATALOG, TOOLS
+
+pytestmark = pytest.mark.model
+
+
+def cases():
+ for case in MODEL_CATALOG:
+ if "full-tool-cycle" not in case.excluded_scenarios:
+ yield pytest.param(case, id=case.model)
+
+
+def tool_renderer(case, kwargs):
+ if case.resolved_renderer == "default":
+ return create_renderer(
+ tokenizer(case.model),
+ config_from_name("default").model_copy(update={"tool_parser": "qwen3"}),
+ )
+ return renderer(case, kwargs)
+
+
+def tool_turn(case, turn, city):
+ call_id = (
+ f"functions.get_weather:{turn}"
+ if case.resolved_renderer in {"kimi-k2", "kimi-k2.5"}
+ else f"call_{turn}"
+ )
+ assistant = {
+ "role": "assistant",
+ "content": "",
+ "reasoning_content": "Check weather.",
+ "tool_calls": [
+ {
+ "id": call_id,
+ "type": "function",
+ "function": {"name": "get_weather", "arguments": {"city": city}},
+ }
+ ],
+ }
+ new = [
+ {
+ "role": "tool",
+ "name": "get_weather",
+ "tool_call_id": call_id,
+ "content": "Sunny.",
+ }
+ ]
+ return assistant, new
+
+
+def reference_turn(case, native, history, assistant, new, kwargs):
+ prompt = reference(case, history, kwargs, tools=TOOLS, add_generation_prompt=True)
+ finished = reference(case, history + [assistant], kwargs, tools=TOOLS)
+ if finished[: len(prompt)] == prompt:
+ completion = finished[len(prompt) :]
+ else:
+ # A closed live prefill may be absent in history. Locate the start of
+ # the tool output by comparing two official completed serializations.
+ assistant = dict(assistant)
+ assistant.pop("reasoning_content", None)
+ finished = reference(case, history + [assistant], kwargs, tools=TOOLS)
+ empty = reference(
+ case, history + [{"role": "assistant", "content": ""}], kwargs, tools=TOOLS
+ )
+ start = next(
+ (i for i, (a, b) in enumerate(zip(finished, empty)) if a != b),
+ min(len(finished), len(empty)),
+ )
+ completion = finished[start:]
+ stops = set(native.get_stop_token_ids())
+ end = next(
+ (i + 1 for i in range(len(completion) - 1, -1, -1) if completion[i] in stops),
+ None,
+ )
+ if end is not None:
+ completion = completion[:end]
+ else:
+ extended = reference(case, history + [assistant] + new, kwargs, tools=TOOLS)
+ stop = max(i for i, token in enumerate(extended) if token in stops)
+ completion += extended[stop : stop + 1]
+ return prompt, completion, assistant
+
+
+def tool_extension(case, native, history, assistant, new, kwargs):
+ following = reference(
+ case,
+ history + [assistant] + new,
+ kwargs,
+ tools=TOOLS,
+ add_generation_prompt=True,
+ )
+ probe = dict(
+ assistant,
+ tool_calls=[
+ {
+ "id": "call_probe",
+ "type": "function",
+ "function": {
+ "name": "get_weather",
+ "arguments": {"city": "BOUNDARY_PROBE_endXYZ"},
+ },
+ }
+ ],
+ )
+ other = reference(
+ case, history + [probe] + new, kwargs, tools=TOOLS, add_generation_prompt=True
+ )
+ common = 0
+ while (
+ common < min(len(following), len(other))
+ and following[-common - 1] == other[-common - 1]
+ ):
+ common += 1
+ suffix = following[len(following) - common :]
+ stops = set(native.get_stop_token_ids())
+ stop = next(i for i, token in enumerate(suffix) if token in stops)
+ return suffix[stop + 1 :]
+
+
+@pytest.mark.parametrize("case", tuple(cases()))
+@pytest.mark.parametrize("city", ["Paris", "", " Paris\nMünchen ☁ "])
+def test_two_tool_cycles(case, city):
+ kwargs = {"is_training": False} if case.resolved_renderer == "hy3" else {}
+ native = tool_renderer(case, kwargs)
+ history = [
+ {"role": "system", "content": "Use the weather tool."},
+ {"role": "user", "content": "Weather?"},
+ ]
+ next_prompt = None
+ for turn in range(2):
+ assistant, new = tool_turn(case, turn, city)
+ prompt, completion, assistant = reference_turn(
+ case, native, history, assistant, new, kwargs
+ )
+ if next_prompt is not None:
+ prompt = next_prompt.token_ids
+ backend = TokenBackend(prompt, completion)
+ result = asyncio.run(
+ generate(
+ client=backend,
+ renderer=native,
+ messages=history,
+ model=case.model,
+ tools=TOOLS,
+ max_prompt_len=1_000_000,
+ prompt_ids=next_prompt.token_ids if next_prompt is not None else None,
+ prompt_attribution=next_prompt,
+ )
+ )
+ assert result["prompt_ids"] == prompt
+ assert result["completion_ids"] == completion
+ np.testing.assert_array_equal(
+ result["completion_logprobs"], [-0.5] * len(completion)
+ )
+ assert len(result["tool_calls"]) == 1
+ call = result["tool_calls"][0]
+ assert call.status == ToolCallParseStatus.OK
+ assert call.name == "get_weather"
+ assert call.arguments == {"city": city}
+ assert result["finish_reason"] == "tool_calls"
+ assistant = {
+ "role": "assistant",
+ "content": result["content"],
+ "reasoning_content": result["reasoning_content"],
+ "tool_calls": [
+ {
+ "id": call.id or new[0]["tool_call_id"],
+ "type": "function",
+ "function": {"name": call.name, "arguments": call.arguments},
+ }
+ ],
+ }
+ new[0]["tool_call_id"] = assistant["tool_calls"][0]["id"]
+ bridge = native.bridge_to_next_turn(prompt, completion, new, tools=TOOLS)
+ if bridge is not None:
+ extension = tool_extension(case, native, history, assistant, new, kwargs)
+ assert_tokens(bridge.token_ids, prompt + completion + extension, case)
+ assert_attribution(bridge)
+ assert not any(bridge.sampled_mask)
+ next_prompt = bridge
+ else:
+ next_prompt = native.render(
+ history + [assistant] + new, tools=TOOLS, add_generation_prompt=True
+ )
+ assert_tokens(
+ next_prompt.token_ids,
+ reference(
+ case,
+ history + [assistant] + new,
+ kwargs,
+ tools=TOOLS,
+ add_generation_prompt=True,
+ ),
+ case,
+ )
+ history += [assistant] + new
diff --git a/tests/conformance/test_traces.py b/tests/conformance/test_traces.py
new file mode 100644
index 00000000..c92e80e5
--- /dev/null
+++ b/tests/conformance/test_traces.py
@@ -0,0 +1,37 @@
+"""Reference coverage and the exact supported generation profiles."""
+
+from typing import get_args
+
+import pytest
+
+from renderers.configs import RendererConfig
+from tests.conformance.harness import (
+ assert_tokens,
+ reference,
+ renderer,
+)
+from tests.conformance.test_rollouts import matrix as inference_matrix
+from tests.model_assets import MODEL_REVISIONS
+from tests.parity import MODEL_CATALOG
+
+pytestmark = pytest.mark.model
+
+
+def test_every_renderer_has_a_reference_checkpoint():
+ union = get_args(RendererConfig)[0]
+ declared = {cls.model_fields["name"].default for cls in get_args(union)}
+ # Factory choices are not wire formats.
+ assert declared - {"auto", "custom"} == {
+ case.resolved_renderer for case in MODEL_CATALOG
+ }
+ assert {case.model for case in MODEL_CATALOG} <= MODEL_REVISIONS.keys()
+
+
+@pytest.mark.parametrize("case,kwargs", tuple(inference_matrix()))
+def test_generation_prefill_matches_reference_profile(case, kwargs):
+ messages = [{"role": "user", "content": "Test."}]
+ assert_tokens(
+ renderer(case, kwargs).render_ids(messages, add_generation_prompt=True),
+ reference(case, messages, kwargs, add_generation_prompt=True),
+ case,
+ )
diff --git a/tests/test_parity.py b/tests/test_parity.py
index 928934bd..6a9aed2d 100644
--- a/tests/test_parity.py
+++ b/tests/test_parity.py
@@ -29,6 +29,8 @@
from renderers import create_renderer
from renderers.base import MODEL_RENDERER_MAP
from renderers.configs import RendererConfig, _config_class_for
+from tests.conformance.harness import assert_attribution, reference
+from tests.conformance.harness import tokenizer as reference_tokenizer
from tests.model_assets import load_test_tokenizer as load_tokenizer
from tests.reference_rendering import (
DEFAULT_REFERENCE_ORACLE,
@@ -36,7 +38,6 @@
RENDERER_ORACLE_ROUTES,
reference_oracle_for_model,
reference_oracle_for_renderer,
- render_reference,
)
pytestmark = pytest.mark.model
@@ -199,26 +200,16 @@ def test_renderer_matches_reference(
scenario: Scenario,
kwargs: Mapping[str, Any],
):
- tokenizer = _tokenizer(case.model)
renderer = _renderer(case.model, case.renderer, tuple(kwargs.items()))
for key, value in kwargs.items():
if value is not None:
assert getattr(renderer.config, key) == value
- oracle_kwargs = dict(case.oracle_defaults)
- oracle_kwargs.update(kwargs)
- # Config ``None`` values mean "defer to the oracle default"; omission
- # expresses the same state for every registered adapter.
- oracle_kwargs = {
- key: value for key, value in oracle_kwargs.items() if value is not None
- }
- expected = render_reference(
- tokenizer,
- [dict(message) for message in scenario.messages],
- oracle=case.oracle,
- **oracle_kwargs,
- **scenario.render_kwargs,
- )
+ messages = [dict(message) for message in scenario.messages]
+ expected = reference(case, messages, dict(kwargs), **scenario.render_kwargs)
+ rendered = renderer.render(messages, **scenario.render_kwargs)
+ assert rendered.token_ids == expected
+ assert_attribution(rendered)
got = renderer.render_ids(list(scenario.messages), **scenario.render_kwargs)
assert got == expected, (
f"{case.model} / {scenario.id} / {dict(kwargs)!r}: renderer diverged "
@@ -231,3 +222,4 @@ def teardown_module():
"""Release model resources when this module finishes, not at process exit."""
_tokenizer.cache_clear()
_renderer.cache_clear()
+ reference_tokenizer.cache_clear()