diff --git a/Cargo.lock b/Cargo.lock index 8f42e8d..d48d4b3 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -965,9 +965,9 @@ dependencies = [ [[package]] name = "cc" -version = "1.3.0" +version = "1.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c89588d05638b5b4594a3348a2d6c20277e43a7f5c5202b05cc56888475a47b8" +checksum = "5add81bb678e6cb321aff7fa0dc7689ad82b112dbc032cea19f91d6b8e3582b9" dependencies = [ "find-msvc-tools", "jobserver", @@ -2156,9 +2156,9 @@ checksum = "e1d926b4d407d372f141f93bb444696142c29d32962ccbd3531117cf3aa0bfa9" [[package]] name = "either" -version = "1.16.0" +version = "1.17.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "91622ff5e7162018101f2fea40d6ebf4a78bbe5a49736a2020649edf9693679e" +checksum = "9e5e8f6c15a24b9a3ee5efec809ccd006d3b30e8b3bb63c39af737c7f87daa1d" [[package]] name = "embassy-futures" @@ -4816,9 +4816,9 @@ dependencies = [ [[package]] name = "rustls-pki-types" -version = "1.15.0" +version = "1.15.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "764899a24af3980067ee14bc143654f297b22eaebfe3c7b6b211920a5a59b046" +checksum = "2f4925028c7eb5d1fcdaf196971378ed9d2c1c4efc7dc5d011256f76c99c0a96" dependencies = [ "web-time", "zeroize", diff --git a/README.md b/README.md index e62a45e..12e3e9b 100644 --- a/README.md +++ b/README.md @@ -91,7 +91,12 @@ It exists because two local-first apps — **[laurelane](https://github.com/phys - **Template byte gate** — the hardcoded renderers are proven **byte-identical to `transformers.apply_chat_template`** rendering the checkpoint's own imported `chat_template` (via the `hf-chat-template` dev-dependency): plain, multi-turn, the full Hermes `# Tools` block, - and function-call history all match byte-for-byte on Qwen3-0.6B (`tests/template_gate.rs`). + and function-call history match byte-for-byte on **Qwen3-0.6B, Qwen2.5-1.5B, and LFM2.5-1.2B** + (LFM's legs cover both tool conventions, its `tool` role turns, its history think-stripping, and + the standalone `chat_template.jinja` import path), `tests/template_gate.rs`. The known family + divergences are pinned to their exact deltas so any other drift fails loudly: Qwen2.5's no-system + branding preamble vs our neutral one, Qwen3's no-system no-preamble, and Qwen3's history + think-stripping (ours re-renders history verbatim — a `ChatMl::qwen3()` is a roadmap item). Prompt JSON deliberately serializes with Python `json.dumps` spacing and insertion-order keys (serde_json `preserve_order`) — the exact bytes the reference stack renders and models emit back. - **f16 inference, validated** — Qwen2.5-1.5B runs coherently on `GpuF16` (weights + KV in f16, the diff --git a/ROADMAP.md b/ROADMAP.md index 7a34156..591f80b 100644 --- a/ROADMAP.md +++ b/ROADMAP.md @@ -77,7 +77,7 @@ a benchmark holds/improves its budget; README perf claims link an artifact. on an otherwise-idle GPU (5% util). Decode throughput tracking CPU availability is what a dispatch-bound path looks like. Operationally: **run the budget gates on a quiet machine** or they report contention as a regression.* -- [ ] Evaluate Burn 0.21's `burn.toml` project config — per-subsystem tuning + a CubeCL kernel-validation +- [x] Evaluate Burn 0.21's `burn.toml` project config — per-subsystem tuning + a CubeCL kernel-validation layer without recompiling; useful as a debug switch for kernel-level parity hunts — https://burn.dev/blog/release-0.21.0/ *(2026-07-17 research)* Concretely, a `burn.toml` dropped at the project root parameterizes every internal subsystem with no code change / no recompile: @@ -96,14 +96,21 @@ a benchmark holds/improves its budget; README perf claims link an artifact. "enforce"|"validate"|"auto"`, `cache`; `[cubecl.streaming] max_streams`; `[cubecl.memory] persistent_memory = "enabled"|"disabled"|"enforced"`. Validation is the `[cubecl.compilation] check_mode` knob (`"validate"` = the OOB-catching layer). So the item's "ON for test / OFF for bench - **profiles**" premise isn't directly expressible — one checked-in file applies to *every* cargo - invocation, benches included. Workable routes when picked up: (a) accept global `check_mode = "validate"` - only if a bench A/B shows the budgets are unmoved by it (measure the overhead first — it may be - negligible and the whole ON/OFF split moot); (b) if it does move the budgets, drive it from an - **env-var/CI-only file** (drop `burn.toml` in the test/CI working dir but not where `cargo bench` runs, - or gate it behind a CI env var the bench job clears) rather than committing a repo-root file. Confirm - whether Burn reads `burn.toml` from cwd or repo-root (determines whether the per-dir trick works) before - adopting either. + **profiles**" premise isn't directly expressible in ONE file — the open question was whether the + discovery rule allows a per-directory split. + *(2026-07-24)* **Shipped — the per-directory split works.** The discovery rule read from + cubecl-runtime 0.10 source answers the 07-21 question: `RuntimeConfig::from_current_dir` walks UP + from the process CWD, `cubecl.toml` checked before `burn.toml` at each level, first hit wins — and + cargo runs each crate's tests/benches with CWD = the package dir. So a repo-root **`burn.toml`** + sets `[cubecl.compilation] check_mode = "validate"` (bounds-check every launch AND validate + explicitly-unchecked kernels for OOB) arming the `crates/mummu` parity/real-model suites, while + **`crates/mummu-bench/cubecl.toml`** opts the budget/bench crate back to the `auto` default so + recorded numbers never carry validation overhead. Consumers run from their own CWD — untouched. + Verified live: an A/B with a malformed root burn.toml makes a GPU test fail at config load naming + the bad key (proof of discovery + parse, from BOTH crates' CWDs), the real-model GPU suite passes + with validation armed (no OOB found — clean bill), and the budget gates hold their numbers from + the opted-out bench crate (12.4 tok/s with the files ≈ 12.1 without; first run after a config + change can read ~30% low while autotune re-tunes — re-run before believing a regression). - [ ] Evaluate **CubeCL's now-complete flash-attention kernel** for the decode/prefill attention step — the releases page reports a full implementation (causal **masking**, partitions, row-wise reductions, multi-plane ops). Mummu currently materializes attention explicitly (q·kᵀ → f32 softmax @@ -608,11 +615,27 @@ The subsystem that turns "a model on HuggingFace or on disk" into a loaded, pari `{"name": "get_weather", ...}` from the new prompt bytes on the 4070 Ti SUPER. Payoff (2) — a general fallback renderer for checkpoints without a hardcoded family renderer — is split below. + *(2026-07-24, merged from the parallel nightly)* **Gate coverage extended beyond Qwen3**: the same + in-process harness now byte-verifies **Qwen2.5-1.5B** (plain / tools / FC history) and **LFM2.5-1.2B** + (plain±system, tools±system in the LFM bare-JSON convention, history think-stripping, pythonic + `<|tool_call_start|>` + `tool` role turns — LFM's legs also exercise the standalone + `chat_template.jinja` import fallback on the real checkpoint, whose template file was fetched into + the local cache). Known family divergences are PINNED to their exact deltas so any other drift still + fails: Qwen2.5's no-system branding preamble ("You are Qwen, …") vs our neutral one (with tools AND + the plain injected default turn), Qwen3's no-system no-preamble, Qwen3's history think-stripping. - [ ] **General fallback chat renderer via `hf-chat-template`** — payoff (2) of the evaluation above: for a checkpoint whose family has no hardcoded `chat` renderer, render prompts from its own imported `chat_template` (the byte gate proved fidelity on Qwen3). Weigh promoting the dep from dev to optional runtime feature vs the from-scratch ethos; needs the P8/consumer-facing API decision of when to prefer the imported template over a family renderer. *(2026-07-23, split from the evaluation.)* +- [ ] **`ChatMl::qwen3()` with history think-stripping** — the byte gate documented that Qwen3's template + strips `…` reasoning from assistant turns at/before the last user query while our + shared `ChatMl::qwen2()` renderer re-renders history verbatim (fine for fresh prompts + tool loops, + wrong for long multi-turn chats with a thinking Qwen3). A `qwen3()` constructor wants the LFM-style + strip (the machinery exists — `turn_content` already does it for `Lfm`) but keyed to Qwen3's + "at/before the last user query" rule rather than LFM's "every but the last assistant turn"; gate it + on the template byte gate's think case flipping from documented-divergence to byte-equal. + *(2026-07-24, found by the byte gate.)* - [x] **Model registry / manifest** — a declarative `ModelSpec` (repo, architecture, weight format, dtype, tokenizer, chat template, size tier) + a small built-in catalog of known-good models (Qwen2.5, LFM2.5, MiniLM, …); adding a model = a manifest entry. *(2026-07-10) `mummu::registry`: `ModelSpec` @@ -670,6 +693,11 @@ The subsystem that turns "a model on HuggingFace or on disk" into a loaded, pari `<|tool_call_start|>[get_weather(city="Paris")]<|tool_call_end|>` from our rendered prompt and the parser round-tripped it; the Qwen2 parity gate re-passed both legs after the template refactor (max |Δlogit| 2.670e-5, Ollama greedy byte-identical). 16 new unit tests (109 total).* + *(2026-07-23/24) Template-embedded tool JSON now spells `json.dumps` separators (`python_json` — + `{"a": 1}`, not compact `{"a":1}`) in BOTH conventions, with insertion-order keys (serde_json + `preserve_order`, a workspace feature): the P3 template byte gate proved the checkpoints' own + templates (Jinja `tojson`) and the models' own emissions use that spelling, and the renders are + now byte-identical to `transformers.apply_chat_template` — see the P3 gate item.* *(2026-07-10 research)* 2026 community numbers back the plan: Qwen3-8B keeps tool-calling score through Q4_K_M (0.919 quantized vs 0.933 full — quant does NOT cost tool reliability, good news for P9); BFCL shows a capability cliff below ~7B (Qwen3.5-9B 66.1% vs 4B 50.3%), so the zoo's @@ -752,10 +780,35 @@ that fits the model AND uses every device to the fullest. decide the policy explicitly — call `set_default_dtypes` per device at model-load/planner level (or pin every runtime tensor creation site's dtype) so in-process mixed-precision is defined behavior, and add a two-alias regression test once it is. +- [ ] **Placement plan** — given model size + KV-cache + display headroom and the device set, choose a **fit-and-fill** plan: single GPU when it fits; **shard layers across multiple GPUs** (pipeline/ layer-parallel over Burn's multi-device tensors — Burn gives the multi-device *primitives*, not automatic tensor-parallel, so we place modules on devices ourselves); **spill cold layers to CPU** (GGUF-style hybrid) when total VRAM is short. Largest-model-that-fits, every device busy. + *(2026-07-24, design settled)* **Heterogeneous per-device precision comes from ONE source file — + no new format.** The checkpoint stores weights once (bf16 safetensors, or a GGUF); each pipeline + stage *derives* its own in-memory representation at load: cast bf16→f32 for the big GPU and + bf16→f16 for a SHADER_F16-capable iGPU (both = the existing `CastFloatAdapter` path, quality-free + casts), and quantize int8/int4 for the CPU stage (the P9 keep-quantized leg; naive round-to-nearest + from bf16 is worse than a calibrated GPTQ/K-quant artifact, so prefer an on-the-fly *block-wise* + Q4_K-style quant — llama.cpp's offline K-quants are data-free, same math). The inverse also holds + and already runs: one Q4_K_M GGUF can serve f32/f16 stages by dequant/upcast (at Q4 quality) and + the CPU stage keep-quantized — so "one file" is a quality-vs-disk choice (bf16 = quality-max, + GGUF = size-min), never a format question. The only format-adjacent addition is a later + **derived-artifact cache** (don't re-quantize 30 layers per launch): a per-user cache dir of + ordinary safetensors/GGUF shards keyed by (source hash, dtype, layer range) — a cache, not a + format. The real work is runtime, in dependency order: (1) test the dtype-policy hazard above + *across* devices first — the flip is per-device, so f32-on-discrete + f16-on-iGPU in one process + is expected to work but is exactly the unproven experiment; (2) the stage-composed model type — + `CausalLm` is generic over ONE backend, a GPU+iGPU split can stay one `Wgpu` backend with + per-tensor dtypes (Burn 0.21 multi-dtype, the f32-softmax island already does per-tensor casts), + but the CPU stage is a different backend *type* (`burn-flex`), so the GPU→CPU seam is a + host-memory transfer between two backend generics; (3) activations cast at stage seams (small + tensors, cheap); (4) per-stage KV-cache shards + the micro-batch schedule (the multi-GPU item). + Expectation to encode in the planner: pipeline throughput = the slowest stage, so iGPU/CPU stages + exist to make a model FIT, not to make a fitting model faster — fit-and-fill, per-device precision + picked by what fits + what the device advertises (`inventory()` already records SHADER_F16 + + max_buffer_bytes + true VRAM per adapter). - [ ] **Multi-GPU execution** — run the sharded plan: per-device sub-modules, activations handed across the device boundary between stages, KV-cache per shard, and a micro-batch/pipeline schedule so the GPUs overlap rather than idle. *(Tensor-parallel within a layer is the stretch goal; layer/pipeline split is @@ -836,6 +889,16 @@ The VRAM lever the P6 planner pulls to make the largest useful model fit the use the kernel substrate a Q4-weights × f16-activations decode path would ride (vs hand-writing a dequant-fused kernel); gate any adoption on the parity harness + `bench/BASELINE.md` — https://github.com/tracel-ai/cubecl/releases · https://burn.dev/blog/release-0.21.0/ +- [ ] **KV-cache quantization (FP8/e4m3)** — quantize the KV cache (and optionally the QK/ScoreV attention + matmuls) to 8-bit, halving per-token cache footprint — the *other* VRAM lever besides weights, and the + one that grows with context length. vLLM shipped exactly this (April 2026) and published the lessons + that transfer: uncalibrated per-head e4m3 scales recover 97%+ on reasoning tasks and 94–98% AUC at + 1M-token contexts; **two-level accumulation is critical** (intermediate f32 writes on long contexts — + the same failure our f32-softmax island guards); layer-selective beats uniform (sliding-window layers + pay overhead for no benefit); head_dim 256 loses at prefill (~1.6× register pressure) while 64/128 win. + For Mummu: our KV cache is f16 on `GpuF16` — an e4m3-quantized cache would halve it again; gate on the + parity harness + budgets like every numeric change. — https://vllm.ai/blog/2026-04-22-fp8-kvcache + *(2026-07-24 research)* - [ ] **Auto-quantize-to-fit** — the planner picks the *highest* precision that fits the detected VRAM (f16 → int8 → int4), reports the quality/size trade, and never silently ships a worse tier than asked. diff --git a/burn.toml b/burn.toml new file mode 100644 index 0000000..41309e7 --- /dev/null +++ b/burn.toml @@ -0,0 +1,17 @@ +# Burn/CubeCL project config (Burn 0.21+). Discovered at runtime by walking UP +# from the process CWD (cubecl.toml is checked before burn.toml at each level), +# so this file governs every process launched from inside this repo — i.e. the +# `crates/mummu` test suites (cargo runs tests with CWD = the package dir, and +# the walk-up lands here). Consumers of the library run from their own +# directories and are untouched. +# +# `validate` = enforce bounds checks on EVERY kernel launch AND validate +# explicitly-unchecked kernels for out-of-bounds accesses — the failure class +# behind silent wrong-logits drift or a GPU crash. We want that armed under +# the parity/real-model suites, where correctness is the whole point. +# +# The perf-budget/benchmark crate must NOT pay validation overhead in its +# recorded numbers: `crates/mummu-bench/cubecl.toml` opts that crate back out +# (its CWD finds the crate-local file first and stops the walk-up). +[cubecl.compilation] +check_mode = "validate" diff --git a/crates/mummu-bench/cubecl.toml b/crates/mummu-bench/cubecl.toml new file mode 100644 index 0000000..a1a3a62 --- /dev/null +++ b/crates/mummu-bench/cubecl.toml @@ -0,0 +1,9 @@ +# Opt the benchmark/budget crate OUT of the repo-root burn.toml's +# `check_mode = "validate"`: CubeCL's config discovery walks up from the +# process CWD and stops at the FIRST cubecl.toml/burn.toml it finds, and +# cargo runs this crate's benches + budget-gate tests with CWD = this +# directory. Bench numbers must never include kernel-validation overhead — +# `auto` is CubeCL's default (bounds-check standard launches, trust +# explicitly-unchecked ones). +[compilation] +check_mode = "auto" diff --git a/crates/mummu/tests/template_gate.rs b/crates/mummu/tests/template_gate.rs index acccb9c..23a5a21 100644 --- a/crates/mummu/tests/template_gate.rs +++ b/crates/mummu/tests/template_gate.rs @@ -219,3 +219,382 @@ fn qwen3_tool_history_render_byte_matches_the_imported_template() { println!("{}", diff_context("tool-history", &ours, &reference)); assert_eq!(ours, reference, "FC history render must byte-match"); } + +// --------------------------------------------------------------------------- +// Coverage beyond Qwen3 (2026-07-24): Qwen2.5 + LFM2.5 legs, and the known +// family divergences PINNED to their exact deltas so any other drift fails. +// Extra env keys: MUMMU_QWEN2_DIR, MUMMU_LFM2_DIR (LFM's legs also exercise +// the standalone `chat_template.jinja` import fallback — the checkpoint dir +// carries the template only as that file). +// --------------------------------------------------------------------------- + +fn dir_from(var: &str) -> Option { + let dir = PathBuf::from(std::env::var_os(var)?); + dir.is_dir().then_some(dir) +} + +fn weather_spec() -> ToolSpec { + ToolSpec { + name: "get_weather".into(), + description: "Get the current weather in a city.".into(), + parameters: serde_json::json!({ + "type": "object", + "properties": { + "city": {"type": "string", "description": "City name"} + }, + "required": ["city"] + }), + } +} + +/// The Hermes wire JSON transformers hands the template as one `tools` entry. +fn hermes_tool_json() -> serde_json::Value { + serde_json::json!({ + "type": "function", + "function": { + "name": "get_weather", + "description": "Get the current weather in a city.", + "parameters": { + "type": "object", + "properties": { + "city": {"type": "string", "description": "City name"} + }, + "required": ["city"] + } + } + }) +} + +/// LFM tools are BARE tool JSON (no Hermes wrapper) — the template runs +/// `tool | tojson` on whatever is passed; the model card shows bare. +fn lfm_tool_json() -> serde_json::Value { + serde_json::json!({ + "name": "get_weather", + "description": "Get the current weather in a city.", + "parameters": { + "type": "object", + "properties": { + "city": {"type": "string", "description": "City name"} + }, + "required": ["city"] + } + }) +} + +/// LFM's template opens with `{{- bos_token -}}` — pass it in the context, +/// exactly as transformers does from the tokenizer's special tokens. +fn lfm_input(messages: Vec, tools: Vec) -> RenderInput { + let mut extra = serde_json::Map::new(); + extra.insert( + "bos_token".into(), + serde_json::Value::String("<|startoftext|>".into()), + ); + RenderInput { + messages, + tools, + add_generation_prompt: true, + extra, + ..RenderInput::default() + } +} + +const SYSTEM: &str = "You are a helpful assistant."; +const USER: &str = "List the first five prime numbers."; + +// ---- Qwen2.5 --------------------------------------------------------------- + +/// Qwen2.5's plain + tools + FC-history legs, byte-identical like Qwen3's. +#[test] +#[ignore = "needs the local Qwen2.5 checkpoint dir (MUMMU_QWEN2_DIR)"] +fn qwen2_renders_byte_match_the_imported_template() { + let dir = dir_from("MUMMU_QWEN2_DIR").expect("set MUMMU_QWEN2_DIR"); + let template = imported_template(&dir); + + // Plain — the exact shape the Qwen2 parity gate commits. + let reference = template + .render(&RenderInput { + messages: vec![Message::system(SYSTEM), Message::user(USER)], + add_generation_prompt: true, + ..RenderInput::default() + }) + .expect("reference render succeeds"); + let ours = ChatMl::qwen2().render(&[Turn::system(SYSTEM), Turn::user(USER)]); + println!("{}", diff_context("qwen2 plain", &ours, &reference)); + assert_eq!(ours, reference, "Qwen2.5 plain render must byte-match"); + + // Tools. + let reference = template + .render(&RenderInput { + messages: vec![Message::system(SYSTEM), Message::user(USER)], + tools: vec![hermes_tool_json()], + add_generation_prompt: true, + ..RenderInput::default() + }) + .expect("reference render succeeds"); + let ours = ChatMl::qwen2() + .render_with_tools(&[weather_spec()], &[Turn::system(SYSTEM), Turn::user(USER)]); + println!("{}", diff_context("qwen2 tools", &ours, &reference)); + assert_eq!(ours, reference, "Qwen2.5 tools render must byte-match"); + + // FC history: a turn + two tool responses (merged user turn). + let mut assistant_call = Message::assistant(""); + assistant_call.content = None; + assistant_call.tool_calls = + vec![serde_json::json!({"name": "get_weather", "arguments": {"city": "Paris"}})]; + let reference = template + .render(&RenderInput { + messages: vec![ + Message::system(SYSTEM), + Message::user("What's the weather in Paris?"), + assistant_call, + Message::new("tool", "{\"temp_c\": 21}"), + Message::new("tool", "{\"temp_c\": 24}"), + ], + tools: vec![hermes_tool_json()], + add_generation_prompt: true, + ..RenderInput::default() + }) + .expect("reference render succeeds"); + let calls = [mummu::chat::ToolCall { + name: "get_weather".into(), + arguments: serde_json::json!({"city": "Paris"}), + }]; + let ours = ChatMl::qwen2().render_with_tools( + &[weather_spec()], + &[ + Turn::system(SYSTEM), + Turn::user("What's the weather in Paris?"), + Turn::assistant_tool_calls(&calls), + Turn::tool_response("{\"temp_c\": 21}"), + Turn::tool_response("{\"temp_c\": 24}"), + ], + ); + println!("{}", diff_context("qwen2 fc-history", &ours, &reference)); + assert_eq!(ours, reference, "Qwen2.5 FC history render must byte-match"); +} + +/// Documented divergence, pinned exactly: without a system turn Qwen2.5's +/// template injects its branding preamble where we inject a neutral one (one +/// renderer serves Qwen2 AND Qwen3, and Qwen3 injects nothing — no single +/// default can match both). The delta must be EXACTLY the preamble swap. +#[test] +#[ignore = "needs the local Qwen2.5 checkpoint dir (MUMMU_QWEN2_DIR)"] +fn qwen2_no_system_defaults_diverge_only_by_the_documented_preamble() { + let dir = dir_from("MUMMU_QWEN2_DIR").expect("set MUMMU_QWEN2_DIR"); + let template = imported_template(&dir); + const QWEN_PREAMBLE: &str = + "You are Qwen, created by Alibaba Cloud. You are a helpful assistant."; + + // With tools: both sides synthesize a system turn; preambles differ. + let reference = template + .render(&RenderInput { + messages: vec![Message::user(USER)], + tools: vec![hermes_tool_json()], + add_generation_prompt: true, + ..RenderInput::default() + }) + .expect("reference render succeeds"); + let ours = ChatMl::qwen2().render_with_tools(&[weather_spec()], &[Turn::user(USER)]); + let ours_with_qwen_preamble = ours.replacen(SYSTEM, QWEN_PREAMBLE, 1); + assert_ne!( + ours_with_qwen_preamble, ours, + "our preamble must be present" + ); + assert_eq!( + ours_with_qwen_preamble, reference, + "no-system tools render must diverge ONLY by the default preamble" + ); + + // Without tools: the template injects a whole default system turn; our + // render() injects nothing (explicit turns are the caller's contract). + let reference = template + .render(&RenderInput { + messages: vec![Message::user(USER)], + add_generation_prompt: true, + ..RenderInput::default() + }) + .expect("reference render succeeds"); + let ours = ChatMl::qwen2().render(&[Turn::user(USER)]); + assert_eq!( + format!("<|im_start|>system\n{QWEN_PREAMBLE}<|im_end|>\n{ours}"), + reference, + "no-system plain render must diverge ONLY by the injected default turn" + ); +} + +// ---- Qwen3 divergences ------------------------------------------------------ + +/// Qwen3's documented divergences, pinned exactly: (a) with tools and no +/// system turn its template injects NO preamble (ours injects the neutral +/// one); (b) it strips `` reasoning from assistant turns at/before the +/// last user query (ours re-renders history verbatim — `ChatMl::qwen3()` with +/// the strip is a ROADMAP item; flip this case to byte-equal when it lands). +#[test] +#[ignore = "needs the local Qwen3 checkpoint dir (MUMMU_QWEN3_DIR)"] +fn qwen3_divergences_are_exactly_the_documented_ones() { + let dir = dir().expect("set MUMMU_QWEN3_DIR to a Qwen3 checkpoint dir"); + let template = imported_template(&dir); + + let reference = template + .render(&RenderInput { + messages: vec![Message::user(USER)], + tools: vec![hermes_tool_json()], + add_generation_prompt: true, + ..RenderInput::default() + }) + .expect("reference render succeeds"); + let ours = ChatMl::qwen2().render_with_tools(&[weather_spec()], &[Turn::user(USER)]); + assert_eq!( + ours.replacen(&format!("{SYSTEM}\n\n"), "", 1), + reference, + "Qwen3 no-system tools render must diverge ONLY by our neutral preamble" + ); + + let think_turn = "2 then 3.The first primes are 2 and 3."; + let reference = template + .render(&RenderInput { + messages: vec![ + Message::system(SYSTEM), + Message::user(USER), + Message::assistant(think_turn), + Message::user("And the next two?"), + ], + add_generation_prompt: true, + ..RenderInput::default() + }) + .expect("reference render succeeds"); + let ours = ChatMl::qwen2().render(&[ + Turn::system(SYSTEM), + Turn::user(USER), + Turn::assistant(think_turn), + Turn::user("And the next two?"), + ]); + assert!(ours.contains(""), "ours re-renders history verbatim"); + assert!( + !reference.contains(""), + "Qwen3's template strips history reasoning" + ); + assert_eq!( + ours.replacen("2 then 3.", "", 1), + reference, + "the think block must be the ONLY delta" + ); +} + +// ---- LFM2.5 ----------------------------------------------------------------- + +/// LFM2.5 plain legs (±system) — these also prove the standalone +/// `chat_template.jinja` import fallback end-to-end on a real checkpoint: +/// the dir ships NO `chat_template` JSON key, only the sibling file. +#[test] +#[ignore = "needs the local LFM2.5 checkpoint dir (MUMMU_LFM2_DIR) with chat_template.jinja"] +fn lfm2_plain_renders_byte_match_the_imported_template() { + let dir = dir_from("MUMMU_LFM2_DIR").expect("set MUMMU_LFM2_DIR"); + let template = imported_template(&dir); + + let reference = template + .render(&lfm_input(vec![Message::user(USER)], vec![])) + .expect("reference render succeeds"); + let ours = ChatMl::lfm2().render(&[Turn::user(USER)]); + println!("{}", diff_context("lfm plain", &ours, &reference)); + assert_eq!(ours, reference, "LFM no-system render must byte-match"); + + let reference = template + .render(&lfm_input( + vec![Message::system(SYSTEM), Message::user(USER)], + vec![], + )) + .expect("reference render succeeds"); + let ours = ChatMl::lfm2().render(&[Turn::system(SYSTEM), Turn::user(USER)]); + println!("{}", diff_context("lfm system", &ours, &reference)); + assert_eq!(ours, reference, "LFM system render must byte-match"); +} + +/// LFM2.5 tools legs (±system, bare tool JSON): with no system turn BOTH +/// sides inject nothing — LFM has no default preamble, so unlike the Qwen +/// families this case is byte-equal, not a pinned divergence. +#[test] +#[ignore = "needs the local LFM2.5 checkpoint dir (MUMMU_LFM2_DIR) with chat_template.jinja"] +fn lfm2_tools_renders_byte_match_the_imported_template() { + let dir = dir_from("MUMMU_LFM2_DIR").expect("set MUMMU_LFM2_DIR"); + let template = imported_template(&dir); + + let reference = template + .render(&lfm_input( + vec![Message::system(SYSTEM), Message::user(USER)], + vec![lfm_tool_json()], + )) + .expect("reference render succeeds"); + let ours = ChatMl::lfm2() + .render_with_tools(&[weather_spec()], &[Turn::system(SYSTEM), Turn::user(USER)]); + println!("{}", diff_context("lfm tools+system", &ours, &reference)); + assert_eq!(ours, reference, "LFM tools+system render must byte-match"); + + let reference = template + .render(&lfm_input(vec![Message::user(USER)], vec![lfm_tool_json()])) + .expect("reference render succeeds"); + let ours = ChatMl::lfm2().render_with_tools(&[weather_spec()], &[Turn::user(USER)]); + println!("{}", diff_context("lfm tools no-system", &ours, &reference)); + assert_eq!( + ours, reference, + "LFM tools no-system render must byte-match" + ); +} + +/// LFM2.5 history semantics: past assistant turns lose their `` +/// reasoning on BOTH sides (keep_past_thinking=false), the LAST assistant +/// turn keeps it; pythonic call turns + real `tool` role turns round-trip. +#[test] +#[ignore = "needs the local LFM2.5 checkpoint dir (MUMMU_LFM2_DIR) with chat_template.jinja"] +fn lfm2_history_renders_byte_match_the_imported_template() { + let dir = dir_from("MUMMU_LFM2_DIR").expect("set MUMMU_LFM2_DIR"); + let template = imported_template(&dir); + + let past = "2, 3.\n\nThe first two primes are 2 and 3."; + let last = "5, 7 next.\n\n5 and 7."; + let turns = [ + Turn::user(USER), + Turn::assistant(past), + Turn::user("And the next two?"), + Turn::assistant(last), + Turn::user("Thanks — one more?"), + ]; + let messages = vec![ + Message::user(USER), + Message::assistant(past), + Message::user("And the next two?"), + Message::assistant(last), + Message::user("Thanks — one more?"), + ]; + let reference = template + .render(&lfm_input(messages, vec![])) + .expect("reference render succeeds"); + let ours = ChatMl::lfm2().render(&turns); + println!("{}", diff_context("lfm think-strip", &ours, &reference)); + assert_eq!(ours, reference, "LFM think-stripping must byte-match"); + + // Pythonic call turn + a real `tool` role turn: the LFM template renders + // assistant content verbatim, so the reference sees the emitted text. + let calls = [mummu::chat::ToolCall { + name: "get_weather".into(), + arguments: serde_json::json!({"city": "Paris"}), + }]; + let call_turn = Turn::assistant_tool_calls_lfm(&calls); + let reference = template + .render(&lfm_input( + vec![ + Message::user("Weather in Paris?"), + Message::assistant(call_turn.content.clone()), + Message::new("tool", "{\"temp_c\": 21}"), + ], + vec![], + )) + .expect("reference render succeeds"); + let ours = ChatMl::lfm2().render(&[ + Turn::user("Weather in Paris?"), + call_turn, + Turn::tool_response("{\"temp_c\": 21}"), + ]); + println!("{}", diff_context("lfm pythonic+tool", &ours, &reference)); + assert_eq!(ours, reference, "LFM pythonic/tool turns must byte-match"); +}