From 092c8db01f0a681be24872b164a88aff222d51a3 Mon Sep 17 00:00:00 2001 From: Justin Icenhour Date: Fri, 24 Jul 2026 06:43:15 -0500 Subject: [PATCH 1/6] =?UTF-8?q?build:=20cargo=20update=20=E2=80=94=20clap/?= =?UTF-8?q?glob/libc/rustls-pki-types/syn/tokio/tokio-util/xxhash-rust/zer?= =?UTF-8?q?ocopy?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 10 patch-level lock bumps; wgpu 29 stays pinned (burn 0.21 resolves it transitively — unblocks with a burn bump, not a cargo upgrade). fmt, clippy --all-targets, 176 unit + 6 load-gate tests, and cargo build all green; GPU budget gates re-verified later this run on a quiet card. Co-Authored-By: Claude Opus 4.8 --- Cargo.lock | 45 +++++++++++++++++++++++---------------------- 1 file changed, 23 insertions(+), 22 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 73175b0..8c739c2 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1067,9 +1067,9 @@ dependencies = [ [[package]] name = "clap" -version = "4.6.3" +version = "4.6.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0fb99565819980999fb7b4a1796046a5c949e6d4ff132cf5fadf5a641e20d776" +checksum = "d91e0c145792ef73a6ad36d27c75ac09f1832222a3c209689d90f534685ee5b7" dependencies = [ "clap_builder", ] @@ -2706,9 +2706,9 @@ dependencies = [ [[package]] name = "glob" -version = "0.3.3" +version = "0.3.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0cc23270f6e1808e30a928bdc84dea0b9b4136a8bc82338574f23baf47bbd280" +checksum = "e4eba85ea1d0a966a983acd07deee566e67395d2d96b6fb39e62b5a833f1eb0b" [[package]] name = "glow" @@ -3304,9 +3304,9 @@ checksum = "34b357333733e8260735ba5894eb928c02ecc69c78715f01a8019e7fa7f2db4c" [[package]] name = "libc" -version = "0.2.186" +version = "0.2.189" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "68ab91017fe16c622486840e4c83c9a37afeff978bd239b5293d61ece587de66" +checksum = "3eaf3ede3fee6db1a4c2ee091bf8a8b4dccdc6d17f656fb07896ee72867612f2" [[package]] name = "libfuzzer-sys" @@ -4775,9 +4775,9 @@ dependencies = [ [[package]] name = "rustls-pki-types" -version = "1.15.0" +version = "1.15.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "764899a24af3980067ee14bc143654f297b22eaebfe3c7b6b211920a5a59b046" +checksum = "2f4925028c7eb5d1fcdaf196971378ed9d2c1c4efc7dc5d011256f76c99c0a96" dependencies = [ "web-time", "zeroize", @@ -4900,7 +4900,7 @@ checksum = "e7a5d71263a5a7d47b41f6b3f06ba276f10cc18b0931f1799f710578e2309348" dependencies = [ "proc-macro2", "quote", - "syn 3.0.2", + "syn 3.0.3", ] [[package]] @@ -5137,9 +5137,9 @@ dependencies = [ [[package]] name = "syn" -version = "3.0.2" +version = "3.0.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a207d6d6a2b7fc470b80443726053f18a2481b7e1eee970597051596567987a3" +checksum = "53e9bae58849f64dfa4f5d5ae372c8341f7305f82a3868709269343628b659a3" dependencies = [ "proc-macro2", "quote", @@ -5285,7 +5285,7 @@ checksum = "43cbfe0cf76104d42a574802844187e84a305e531ed54455f11fbde0f10541cd" dependencies = [ "proc-macro2", "quote", - "syn 3.0.2", + "syn 3.0.3", ] [[package]] @@ -5435,9 +5435,9 @@ dependencies = [ [[package]] name = "tokio" -version = "1.53.0" +version = "1.53.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d988bcd52dbe076d3d46903332f58c912b87a2c49b1428419a5845154762ffee" +checksum = "202caea871b69668250d242070849eb495be178ed697a3e98aebce5bc81a0bed" dependencies = [ "bytes", "libc", @@ -5459,13 +5459,14 @@ dependencies = [ [[package]] name = "tokio-util" -version = "0.7.18" +version = "0.7.19" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9ae9cec805b01e8fc3fd2fe289f89149a9b66dd16786abd8b19cfa7b48cb0098" +checksum = "494815d09bf52b5548659851081238f0ca39ff638363907596da739561c62c52" dependencies = [ "bytes", "futures-core", "futures-sink", + "libc", "pin-project-lite", "tokio", ] @@ -6497,9 +6498,9 @@ checksum = "3ae8337f8a065cfc972643663ea4279e04e7256de865aa66fe25cec5fb912d3f" [[package]] name = "xxhash-rust" -version = "0.8.17" +version = "0.8.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "985eec839aaf2a1270af8f4ebcf63cf9401cfd90f0902f97c28d9f104ffbde72" +checksum = "aee1b19627c7c60102ab80d3a9cbe18de90bfe03bfa6c3715447681f0e8c8af6" [[package]] name = "y4m" @@ -6532,18 +6533,18 @@ dependencies = [ [[package]] name = "zerocopy" -version = "0.8.54" +version = "0.8.55" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b7cbbc0a705a0fd05cc3676525980d2bf5a9bc4adac6d6475209a7887cf59d19" +checksum = "b5a105cd7b140f6eeec8acff2ea38135d3cab283ada58540f629fe51e46696eb" dependencies = [ "zerocopy-derive", ] [[package]] name = "zerocopy-derive" -version = "0.8.54" +version = "0.8.55" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e2e817b7b52d0c7358d3246da9d69935ebb18116b2b102b4230dac079b4862f5" +checksum = "0fe976fb70c78cd64cccfe3a6fc142244e8a77b70959b30faf9d0ac37ee228eb" dependencies = [ "proc-macro2", "quote", From e9bb46d5ec9b0c9c9742c58fbb6b91eb1da7adf3 Mon Sep 17 00:00:00 2001 From: Justin Icenhour Date: Fri, 24 Jul 2026 07:53:00 -0500 Subject: [PATCH 2/6] feat(perf): burn.toml arms CubeCL kernel-OOB validation for the test suites; bench crate opts out MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Discovery semantics read from cubecl-runtime 0.10 source: config walks UP from the process CWD (cubecl.toml before burn.toml at each level, first hit wins) and cargo runs each crate's tests/benches with CWD = the package dir. Root burn.toml sets [cubecl.compilation] check_mode = "validate" (bounds- check every launch AND validate unchecked kernels for OOB) for the crates/mummu parity/real-model suites; crates/mummu-bench/cubecl.toml pins the budget/bench crate back to "auto" so recorded numbers never carry validation overhead. Consumers run from their own CWD — untouched. Proof: a poisoned root burn.toml fails a GPU test at config load naming the bad key (discovery + parse); a poisoned bench cubecl.toml does the same from the bench crate (opt-out file is the one found there); the whole real-model GPU battery passed with validate armed (no OOB found); budget gate 12.4 / 12.4 tok/s + TTFT ~115 ms across two runs, identical to a no-config A/B run (12.1), budgets unmoved. Co-Authored-By: Claude Opus 4.8 --- burn.toml | 17 +++++++++++++++++ crates/mummu-bench/cubecl.toml | 9 +++++++++ 2 files changed, 26 insertions(+) create mode 100644 burn.toml create mode 100644 crates/mummu-bench/cubecl.toml diff --git a/burn.toml b/burn.toml new file mode 100644 index 0000000..41309e7 --- /dev/null +++ b/burn.toml @@ -0,0 +1,17 @@ +# Burn/CubeCL project config (Burn 0.21+). Discovered at runtime by walking UP +# from the process CWD (cubecl.toml is checked before burn.toml at each level), +# so this file governs every process launched from inside this repo — i.e. the +# `crates/mummu` test suites (cargo runs tests with CWD = the package dir, and +# the walk-up lands here). Consumers of the library run from their own +# directories and are untouched. +# +# `validate` = enforce bounds checks on EVERY kernel launch AND validate +# explicitly-unchecked kernels for out-of-bounds accesses — the failure class +# behind silent wrong-logits drift or a GPU crash. We want that armed under +# the parity/real-model suites, where correctness is the whole point. +# +# The perf-budget/benchmark crate must NOT pay validation overhead in its +# recorded numbers: `crates/mummu-bench/cubecl.toml` opts that crate back out +# (its CWD finds the crate-local file first and stops the walk-up). +[cubecl.compilation] +check_mode = "validate" diff --git a/crates/mummu-bench/cubecl.toml b/crates/mummu-bench/cubecl.toml new file mode 100644 index 0000000..a1a3a62 --- /dev/null +++ b/crates/mummu-bench/cubecl.toml @@ -0,0 +1,9 @@ +# Opt the benchmark/budget crate OUT of the repo-root burn.toml's +# `check_mode = "validate"`: CubeCL's config discovery walks up from the +# process CWD and stops at the FIRST cubecl.toml/burn.toml it finds, and +# cargo runs this crate's benches + budget-gate tests with CWD = this +# directory. Bench numbers must never include kernel-validation overhead — +# `auto` is CubeCL's default (bounds-check standard launches, trust +# explicitly-unchecked ones). +[compilation] +check_mode = "auto" From 7c100262fac3629e974e992c397ba61d21e3ea04 Mon Sep 17 00:00:00 2001 From: Justin Icenhour Date: Fri, 24 Jul 2026 07:53:15 -0500 Subject: [PATCH 3/6] =?UTF-8?q?fix(tests):=20move=20the=20Qwen3=20GpuF16?= =?UTF-8?q?=20leg=20to=20real=5Ff16.rs=20=E2=80=94=20one=20dtype=20alias?= =?UTF-8?q?=20per=20process?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Pre-existing deterministic failure found (and reproduced on unmodified origin/main, serial and parallel): instantiating GpuF16 flips Burn's per-device default dtype policy process-wide, so real_qwen3's f32 GGUF cross-check — running after the f16 leg in the same test binary — panicked reading its logits back (TypeMismatch: expected F16, got F32). The repo convention is that every GpuF16 leg lives in real_f16.rs as its own process; qwen3's f16 test now does too. After the move: real_qwen3 2/2 (GGUF-vs-safetensors cross-check back to its recorded numbers — cosine 0.98910, top-1 151667 both builds) and real_f16 2/2 (qwen2 + qwen3 f16 legs) on the 4070 Ti SUPER. The library-level hazard for consumers switching precision at runtime is recorded as a P6 roadmap item. Co-Authored-By: Claude Opus 4.8 --- crates/mummu/tests/real_f16.rs | 58 ++++++++++++++++++++++++++++++++ crates/mummu/tests/real_qwen3.rs | 49 +-------------------------- 2 files changed, 59 insertions(+), 48 deletions(-) diff --git a/crates/mummu/tests/real_f16.rs b/crates/mummu/tests/real_f16.rs index 79d3e03..a3f56f9 100644 --- a/crates/mummu/tests/real_f16.rs +++ b/crates/mummu/tests/real_f16.rs @@ -66,3 +66,61 @@ fn qwen2_decodes_coherently_in_f16_on_gpu() { // Claim 2 (VRAM) is measured outside the process (nvidia-smi peak while // this test runs) — recorded in bench/BASELINE.md. } + +fn qwen3_dir() -> Option { + std::env::var_os("MUMMU_QWEN3_DIR") + .map(PathBuf::from) + .filter(|d| d.join("model.safetensors").is_file()) +} + +/// f16 leg of the Qwen3 port: bf16 weights cast to f16 on load +/// (`CastFloatAdapter`), the per-head q/k RMSNorm + decoupled head_dim riding +/// the SAME f32-softmax attention island Qwen2/LFM2 use, so the q·kᵀ scores +/// never overflow f16. Lives HERE and not in `real_qwen3.rs` because every +/// `GpuF16` leg needs its own process: instantiating `GpuF16` flips Burn's +/// per-device default dtype policy, and any `Gpu` (f32) test that runs later +/// in the same process then reads its logits back as F16 — a deterministic +/// `TypeMismatch` panic, reproduced on unmodified HEAD (2026-07-24) by +/// running the old mixed `real_qwen3` suite serially. +#[test] +#[ignore = "needs the Qwen3 safetensors dir (MUMMU_QWEN3_DIR) + a SHADER_F16 GPU"] +fn real_qwen3_decodes_coherently_in_f16() { + let dir = qwen3_dir().expect("set MUMMU_QWEN3_DIR to a Qwen3 safetensors dir"); + assert!( + inventory().any_shader_f16(), + "no adapter advertises SHADER_F16 — cannot validate f16 here" + ); + let device = burn::tensor::Device::::default(); + + let tok = tokenizers::Tokenizer::from_file(dir.join("tokenizer.json")).expect("tokenizer.json"); + let prompt_text = mummu::chat::ChatMl::qwen2().render(&[ + mummu::chat::Turn::system("You are a concise assistant. Do not think, answer directly."), + mummu::chat::Turn::user("What is 2+2? Answer in one short sentence."), + ]); + let prompt = tok + .encode(prompt_text, true) + .expect("prompt encodes") + .get_ids() + .to_vec(); + + // bf16 -> f16 on load; the build must not NaN through the qk-norm + softmax. + let model = + mummu::models::qwen3::load_from_dir::(&dir, &device).expect("f16 load checked"); + let smoke = model + .sanity_check(&prompt, model.config.vocab_size, &device) + .expect("f16 forward is finite and non-degenerate (no overflow to NaN)"); + eprintln!( + "[real_f16/qwen3] sanity smoke: top_id {} · spread {:.3}", + smoke.top_id, smoke.spread + ); + + let ids = model + .greedy_generate(&prompt, 48, &device) + .expect("f16 decode"); + let text = tok.decode(&ids, true).expect("ids decode"); + eprintln!("[real_f16/qwen3] greedy: {text:?}"); + assert!( + text.contains('4'), + "expected the f16 answer to mention 4: {text:?}" + ); +} diff --git a/crates/mummu/tests/real_qwen3.rs b/crates/mummu/tests/real_qwen3.rs index 80f74aa..0de41cc 100644 --- a/crates/mummu/tests/real_qwen3.rs +++ b/crates/mummu/tests/real_qwen3.rs @@ -11,7 +11,7 @@ use std::path::PathBuf; -use mummu::backend::{Gpu, GpuF16, inventory, use_gpu}; +use mummu::backend::{Gpu, use_gpu}; use mummu::gguf::GgufFile; use mummu::models::CausalLm; use mummu::models::qwen3; @@ -107,53 +107,6 @@ fn real_qwen3_safetensors_loads_and_decodes_on_gpu() { assert!(text.contains('4'), "expected the answer 4 in: {text:?}"); } -/// f16 leg: the Qwen3 arch runs on the `GpuF16` backend too — bf16 weights -/// cast to f16 on load (`CastFloatAdapter`), and its per-head q/k RMSNorm + -/// decoupled head_dim ride the SAME f32-softmax attention island Qwen2/LFM2 -/// use, so the q·kᵀ scores never overflow f16. Proves the dtype path (P3) and -/// the f16 precision milestone (P6) cover the new architecture, not just Qwen2. -#[test] -#[ignore = "needs the Qwen3 safetensors dir (MUMMU_QWEN3_DIR) + a SHADER_F16 GPU"] -fn real_qwen3_decodes_coherently_in_f16() { - let dir = dir().expect("set MUMMU_QWEN3_DIR to a Qwen3 safetensors dir"); - assert!( - inventory().any_shader_f16(), - "no adapter advertises SHADER_F16 — cannot validate f16 here" - ); - let device = burn::tensor::Device::::default(); - - let tok = tokenizers::Tokenizer::from_file(dir.join("tokenizer.json")).expect("tokenizer.json"); - let prompt_text = mummu::chat::ChatMl::qwen2().render(&[ - mummu::chat::Turn::system("You are a concise assistant. Do not think, answer directly."), - mummu::chat::Turn::user("What is 2+2? Answer in one short sentence."), - ]); - let prompt = tok - .encode(prompt_text, true) - .expect("prompt encodes") - .get_ids() - .to_vec(); - - // bf16 -> f16 on load; the build must not NaN through the qk-norm + softmax. - let model = qwen3::load_from_dir::(&dir, &device).expect("f16 weights load checked"); - let smoke = model - .sanity_check(&prompt, model.config.vocab_size, &device) - .expect("f16 forward is finite and non-degenerate (no overflow to NaN)"); - eprintln!( - "[real_qwen3/f16] sanity smoke: top_id {} · spread {:.3}", - smoke.top_id, smoke.spread - ); - - let ids = model - .greedy_generate(&prompt, 48, &device) - .expect("f16 decode"); - let text = tok.decode(&ids, true).expect("ids decode"); - eprintln!("[real_qwen3/f16] greedy: {text:?}"); - assert!( - text.contains('4'), - "expected the f16 answer to mention 4: {text:?}" - ); -} - /// END-TO-END on the Q4_K_M GGUF alone (config + tokenizer + weights from the /// one file), cross-checked against the bf16 safetensors build: both decode a /// correct answer and agree on the top first-token id (small logit drift IS the From e9b56b85e159c07a95a106a934e91a593cd49f8c Mon Sep 17 00:00:00 2001 From: Justin Icenhour Date: Fri, 24 Jul 2026 07:53:32 -0500 Subject: [PATCH 4/6] =?UTF-8?q?feat(p3/p4):=20the=20chat-template=20byte?= =?UTF-8?q?=20gate=20=E2=80=94=20renders=20now=20byte-identical=20to=20tra?= =?UTF-8?q?nsformers?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Building the gate caught a real divergence and it is fixed: our renders embedded tool JSON compact ({"a":1}) where every checkpoint template runs Jinja tojson = Python json.dumps separators ({"a": 1}) — and the models emit that spacing back (the 07-11 real-GPU proof logged it verbatim). New public chat::py_json (a json.dumps-separator serde Formatter) feeds all template-embedded JSON in BOTH conventions (Hermes tool lines + history blocks, LFM 'List of tools:' line). The gate: tools/template-probe (out-of-workspace bin, candle-probe pattern) renders a template via hf-chat-template 0.2.1 (minijinja + transformers compat). Deliberately NOT a dev-dependency — it enables serde_json's preserve_order feature, which (features being additive) would flip our own test builds from sorted to insertion-ordered maps, diverging the very renders the tests byte-verify from what plain-feature consumers build. tests/template_gate.rs (ignored; MUMMU_TEMPLATE_PROBE + checkpoint dirs) pulls each checkpoint's OWN template through TokenizerConfig::from_dir (incl. LFM2.5's standalone chat_template.jinja) and byte-compares: 9/9 — Qwen2.5 plain/tools/tool-call-history, Qwen3 plain/tools, LFM2.5 plain±system/tools±system/think-strip/pythonic+tool turns all byte-identical; the family divergences (Qwen2.5's no-system branding preamble, Qwen3's no-system no-preamble, Qwen3 history think-stripping) are pinned to their exact deltas so any other drift fails loudly. Real-GPU re-proof with the spaced prompts: all three toolcall suites green — Qwen2.5, LFM2.5, and Qwen3 each emitted a clean parseable call (in exactly the spaced spelling). Parity re-passed bit-identically (max |dlogit| 2.670288e-5, Ollama greedy leg exact); 177 unit tests green. Co-Authored-By: Claude Opus 4.8 --- crates/mummu/src/chat.rs | 100 +++++- crates/mummu/tests/template_gate.rs | 480 ++++++++++++++++++++++++++++ tools/template-probe/Cargo.lock | 170 ++++++++++ tools/template-probe/Cargo.toml | 22 ++ tools/template-probe/src/main.rs | 58 ++++ 5 files changed, 820 insertions(+), 10 deletions(-) create mode 100644 crates/mummu/tests/template_gate.rs create mode 100644 tools/template-probe/Cargo.lock create mode 100644 tools/template-probe/Cargo.toml create mode 100644 tools/template-probe/src/main.rs diff --git a/crates/mummu/src/chat.rs b/crates/mummu/src/chat.rs index 30ffdd8..ef3707b 100644 --- a/crates/mummu/src/chat.rs +++ b/crates/mummu/src/chat.rs @@ -96,7 +96,7 @@ impl Turn { let blocks: Vec = calls .iter() .map(|c| { - let json = serde_json::to_string(c).unwrap_or_default(); + let json = py_json(c); debug_assert!(!json.is_empty(), "a ToolCall always serializes"); format!("\n{json}\n") }) @@ -253,6 +253,64 @@ struct ToolWire<'a> { function: &'a ToolSpec, } +/// Serialize exactly as Python's `json.dumps(value, ensure_ascii=False)` +/// spells it: `", "` between items and `": "` after keys (json.dumps' +/// default separators). The checkpoint chat templates these renders are +/// byte-verified against embed tool JSON via Jinja's `tojson` — json.dumps +/// underneath — and the models were trained on (and emit back) that spacing, +/// e.g. Qwen2.5's observed `{"name": "get_weather", "arguments": {"city": +/// "Paris"}}`, so byte-stable rendering must match it. Escaping needs no +/// shim: serde_json and `ensure_ascii=False` agree (`"`, `\`, and control +/// chars escaped; unicode kept raw). Public because anything composing wire +/// JSON for a prompt (a consumer building history turns by hand, the +/// template byte gate) must spell it identically. +#[must_use] +pub fn py_json(value: &T) -> String { + let mut out = Vec::with_capacity(128); + let mut ser = serde_json::Serializer::with_formatter(&mut out, PyJsonFormatter); + if value.serialize(&mut ser).is_err() { + debug_assert!(false, "py_json: a chat wire value always serializes"); + return String::new(); + } + debug_assert!(!out.is_empty(), "py_json: JSON of any value is non-empty"); + String::from_utf8(out).unwrap_or_default() +} + +/// The `json.dumps`-separator formatter behind [`py_json`]: `", "`/`": "` +/// instead of serde_json's compact `","`/`":"`. Everything else is default. +struct PyJsonFormatter; + +impl serde_json::ser::Formatter for PyJsonFormatter { + fn begin_array_value(&mut self, writer: &mut W, first: bool) -> std::io::Result<()> + where + W: ?Sized + std::io::Write, + { + if first { + Ok(()) + } else { + writer.write_all(b", ") + } + } + + fn begin_object_key(&mut self, writer: &mut W, first: bool) -> std::io::Result<()> + where + W: ?Sized + std::io::Write, + { + if first { + Ok(()) + } else { + writer.write_all(b", ") + } + } + + fn begin_object_value(&mut self, writer: &mut W) -> std::io::Result<()> + where + W: ?Sized + std::io::Write, + { + writer.write_all(b": ") + } +} + /// One tool invocation, as emitted by the model inside `` tags. #[derive(Debug, Clone, PartialEq, serde::Serialize, serde::Deserialize)] pub struct ToolCall { @@ -767,11 +825,10 @@ impl ChatMl { You are provided with function signatures within XML tags:\n", ); for tool in tools { - let json = serde_json::to_string(&ToolWire { + let json = py_json(&ToolWire { r#type: "function", function: tool, - }) - .unwrap_or_default(); + }); debug_assert!(!json.is_empty(), "a ToolSpec always serializes"); system.push('\n'); system.push_str(&json); @@ -804,7 +861,7 @@ impl ChatMl { if i > 0 { system.push_str(", "); } - let json = serde_json::to_string(tool).unwrap_or_default(); + let json = py_json(tool); debug_assert!(!json.is_empty(), "a ToolSpec always serializes"); system.push_str(&json); } @@ -869,6 +926,29 @@ mod tests { let _ = ChatMl::qwen2().render(&[]); } + /// `py_json` must spell values exactly as Python's + /// `json.dumps(..., ensure_ascii=False)` — the `tojson` the checkpoint + /// templates run. Expected strings below are literal CPython output. + #[test] + fn py_json_matches_json_dumps_spelling() { + let v = serde_json::json!({ + "b": [1, 2.5, true, false, null], + "a": {"nested": "va\"l\n"}, + "c": "héllo→" + }); + // serde_json Value maps iterate sorted (BTreeMap), so keys come out + // a, b, c — json.dumps of the same (sorted) dict spells: + assert_eq!( + py_json(&v), + "{\"a\": {\"nested\": \"va\\\"l\\n\"}, \ + \"b\": [1, 2.5, true, false, null], \ + \"c\": \"héllo→\"}" + ); + // Empty containers carry no inner spacing, same as json.dumps. + assert_eq!(py_json(&serde_json::json!({})), "{}"); + assert_eq!(py_json(&serde_json::json!([])), "[]"); + } + fn weather_tool() -> ToolSpec { ToolSpec { name: "get_weather".into(), @@ -898,7 +978,7 @@ mod tests { You may call one or more functions to assist with the user query.\n\n\ You are provided with function signatures within XML tags:\n\ \n\ - {\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"description\":\"Get the current weather for a city.\",\"parameters\":{\"properties\":{\"city\":{\"type\":\"string\"}},\"required\":[\"city\"],\"type\":\"object\"}}}\n\ + {\"type\": \"function\", \"function\": {\"name\": \"get_weather\", \"description\": \"Get the current weather for a city.\", \"parameters\": {\"properties\": {\"city\": {\"type\": \"string\"}}, \"required\": [\"city\"], \"type\": \"object\"}}}\n\ \n\n\ For each function call, return a json object with function name and arguments within XML tags:\n\ \n{\"name\": , \"arguments\": }\n<|im_end|>\n\ @@ -930,7 +1010,7 @@ mod tests { // The assistant history turn carries the block it emitted. assert!(raw.contains( "<|im_start|>assistant\n\n\ - {\"name\":\"get_weather\",\"arguments\":{\"city\":\"Paris\"}}\n\ + {\"name\": \"get_weather\", \"arguments\": {\"city\": \"Paris\"}}\n\ <|im_end|>\n" )); // Both results ride in ONE user turn, each in its own block. @@ -1010,7 +1090,7 @@ mod tests { ], ); let expected = "<|startoftext|><|im_start|>system\nYou are a helpful assistant.\n\ - List of tools: [{\"name\":\"get_weather\",\"description\":\"Get the current weather for a city.\",\"parameters\":{\"properties\":{\"city\":{\"type\":\"string\"}},\"required\":[\"city\"],\"type\":\"object\"}}]<|im_end|>\n\ + List of tools: [{\"name\": \"get_weather\", \"description\": \"Get the current weather for a city.\", \"parameters\": {\"properties\": {\"city\": {\"type\": \"string\"}}, \"required\": [\"city\"], \"type\": \"object\"}}]<|im_end|>\n\ <|im_start|>user\nWeather in Paris?<|im_end|>\n\ <|im_start|>assistant\n"; assert_eq!(raw, expected); @@ -1030,8 +1110,8 @@ mod tests { let mut second = weather_tool(); second.name = "get_time".into(); let raw = ChatMl::lfm2().render_with_tools(&[weather_tool(), second], &[Turn::user("hi")]); - assert!(raw.contains("\"name\":\"get_weather\"")); - assert!(raw.contains("}, {\"name\":\"get_time\"")); + assert!(raw.contains("\"name\": \"get_weather\"")); + assert!(raw.contains("}, {\"name\": \"get_time\"")); assert_eq!(raw.matches("List of tools: [").count(), 1); } diff --git a/crates/mummu/tests/template_gate.rs b/crates/mummu/tests/template_gate.rs new file mode 100644 index 0000000..d624dd7 --- /dev/null +++ b/crates/mummu/tests/template_gate.rs @@ -0,0 +1,480 @@ +//! The template byte gate: a checkpoint's OWN imported `chat_template`, +//! rendered exactly as Python `transformers.apply_chat_template` renders it +//! (via `tools/template-probe`, hf-chat-template = minijinja + a transformers +//! compatibility layer), must reproduce our hardcoded `chat` renderers +//! byte-for-byte on the parity-committed prompt shapes — and where the two +//! deliberately diverge (family default preambles, Qwen3 think-stripping), +//! the divergence is pinned down to the exact expected delta so any OTHER +//! drift still fails loudly. +//! +//! The template travels through the real import path: `TokenizerConfig:: +//! from_dir` (JSON key, or the standalone `chat_template.jinja` fallback for +//! LFM2.5). Ignored by default; run with +//! +//! ```text +//! cargo build --release --manifest-path tools/template-probe/Cargo.toml +//! MUMMU_TEMPLATE_PROBE=/release/template-probe.exe \ +//! MUMMU_QWEN2_DIR=... MUMMU_QWEN3_DIR=... MUMMU_LFM2_DIR=... \ +//! cargo test -p mummu --test template_gate -- --ignored --nocapture +//! ``` + +use std::io::Write; +use std::path::PathBuf; +use std::process::{Command, Stdio}; + +use mummu::chat::{ChatMl, ToolCall, ToolSpec, Turn, py_json}; +use mummu::tok_config::TokenizerConfig; + +fn env_dir(var: &str) -> Option { + let dir = PathBuf::from(std::env::var_os(var)?); + dir.is_dir().then_some(dir) +} + +fn probe_exe() -> PathBuf { + let Some(path) = std::env::var_os("MUMMU_TEMPLATE_PROBE") else { + panic!( + "set MUMMU_TEMPLATE_PROBE to the built tools/template-probe binary \ + (cargo build --release --manifest-path tools/template-probe/Cargo.toml)" + ); + }; + let path = PathBuf::from(path); + assert!( + path.is_file(), + "MUMMU_TEMPLATE_PROBE is not a file: {path:?}" + ); + path +} + +/// Write the checkpoint's imported template to a temp file and render +/// `input_json` through the reference engine. The template string comes out +/// of the SAME importer the loaders use, so the gate covers import + render. +fn reference_render(dir: &std::path::Path, input_json: &str) -> String { + assert!(!input_json.is_empty(), "reference_render: empty input"); + let cfg = TokenizerConfig::from_dir(dir).expect("tokenizer_config imports"); + let template = cfg + .chat_template + .expect("checkpoint has a chat template (JSON key or chat_template.jinja)"); + // Unique per call — tests run as parallel threads in ONE process, so a + // pid-only name would let them delete each other's file mid-render. + static SEQ: std::sync::atomic::AtomicU32 = std::sync::atomic::AtomicU32::new(0); + let seq = SEQ.fetch_add(1, std::sync::atomic::Ordering::Relaxed); + let tmp = std::env::temp_dir().join(format!( + "mummu-template-gate-{}-{seq}.jinja", + std::process::id() + )); + std::fs::write(&tmp, &template).expect("temp template writes"); + + let mut child = Command::new(probe_exe()) + .arg(&tmp) + .stdin(Stdio::piped()) + .stdout(Stdio::piped()) + .stderr(Stdio::piped()) + .spawn() + .expect("template-probe spawns"); + child + .stdin + .take() + .expect("probe stdin") + .write_all(input_json.as_bytes()) + .expect("probe stdin writes"); + let out = child.wait_with_output().expect("probe runs"); + let _ = std::fs::remove_file(&tmp); + assert!( + out.status.success(), + "template-probe failed: {}", + String::from_utf8_lossy(&out.stderr) + ); + String::from_utf8(out.stdout).expect("probe output is utf-8") +} + +/// JSON string literal (escaping) for hand-assembled render-input text. +fn js(s: &str) -> String { + py_json(&s) +} + +fn text_msg(role: &str, content: &str) -> String { + format!("{{\"role\": {}, \"content\": {}}}", js(role), js(content)) +} + +/// A render-input built as TEXT so key order inside tool payloads survives +/// into the probe's order-preserving parse (a serde_json::Value round-trip +/// on our side would re-sort object keys and break byte-stability). +fn input_json(messages: &[String], tools: &[String], extra: &str) -> String { + assert!(!messages.is_empty(), "input_json: no messages"); + let mut out = format!("{{\"messages\": [{}]", messages.join(", ")); + if !tools.is_empty() { + out.push_str(&format!(", \"tools\": [{}]", tools.join(", "))); + } + out.push_str(", \"add_generation_prompt\": true"); + if !extra.is_empty() { + out.push_str(", "); + out.push_str(extra); + } + out.push('}'); + out +} + +fn weather_tool() -> ToolSpec { + ToolSpec { + name: "get_weather".into(), + description: "Get the current weather for a city.".into(), + parameters: serde_json::json!({ + "type": "object", + "properties": { "city": { "type": "string", "description": "City name" } }, + "required": ["city"] + }), + } +} + +/// The Hermes wire shape (mirrors the private struct behind +/// `render_with_tools`): what transformers' `get_json_schema` hands the +/// template as one entry of `tools`. +#[derive(serde::Serialize)] +struct Wire<'a> { + r#type: &'static str, + function: &'a ToolSpec, +} + +fn hermes_wire(tool: &ToolSpec) -> String { + py_json(&Wire { + r#type: "function", + function: tool, + }) +} + +const SYSTEM: &str = "You are a helpful assistant."; +const USER: &str = "List the first five prime numbers."; + +// ---- Qwen2.5 --------------------------------------------------------------- + +#[test] +#[ignore = "needs MUMMU_TEMPLATE_PROBE + a cached Qwen2.5 checkpoint (MUMMU_QWEN2_DIR)"] +fn qwen2_plain_render_matches_the_imported_template() { + let Some(dir) = env_dir("MUMMU_QWEN2_DIR") else { + panic!("set MUMMU_QWEN2_DIR to a dir with tokenizer_config.json"); + }; + let ours = ChatMl::qwen2().render(&[Turn::system(SYSTEM), Turn::user(USER)]); + let reference = reference_render( + &dir, + &input_json( + &[text_msg("system", SYSTEM), text_msg("user", USER)], + &[], + "", + ), + ); + assert_eq!(ours, reference, "plain ChatML render diverged"); +} + +#[test] +#[ignore = "needs MUMMU_TEMPLATE_PROBE + a cached Qwen2.5 checkpoint (MUMMU_QWEN2_DIR)"] +fn qwen2_tools_render_matches_the_imported_template() { + let Some(dir) = env_dir("MUMMU_QWEN2_DIR") else { + panic!("set MUMMU_QWEN2_DIR to a dir with tokenizer_config.json"); + }; + let tool = weather_tool(); + let ours = ChatMl::qwen2().render_with_tools( + std::slice::from_ref(&tool), + &[Turn::system(SYSTEM), Turn::user(USER)], + ); + let reference = reference_render( + &dir, + &input_json( + &[text_msg("system", SYSTEM), text_msg("user", USER)], + &[hermes_wire(&tool)], + "", + ), + ); + assert_eq!(ours, reference, "Hermes tools render diverged"); +} + +#[test] +#[ignore = "needs MUMMU_TEMPLATE_PROBE + a cached Qwen2.5 checkpoint (MUMMU_QWEN2_DIR)"] +fn qwen2_tool_call_history_matches_the_imported_template() { + let Some(dir) = env_dir("MUMMU_QWEN2_DIR") else { + panic!("set MUMMU_QWEN2_DIR to a dir with tokenizer_config.json"); + }; + let tool = weather_tool(); + let calls = [ToolCall { + name: "get_weather".into(), + arguments: serde_json::json!({"city": "Paris"}), + }]; + let ours = ChatMl::qwen2().render_with_tools( + std::slice::from_ref(&tool), + &[ + Turn::system(SYSTEM), + Turn::user("Weather in Paris?"), + Turn::assistant_tool_calls(&calls), + Turn::tool_response("{\"temp_c\": 21}"), + Turn::tool_response("{\"temp_c\": 24}"), + ], + ); + // The reference sees the call structurally (message.tool_calls), exactly + // as transformers is fed; ours re-renders the emitted text. + let call_msg = format!( + "{{\"role\": \"assistant\", \"tool_calls\": [{}]}}", + py_json(&calls[0]) + ); + let reference = reference_render( + &dir, + &input_json( + &[ + text_msg("system", SYSTEM), + text_msg("user", "Weather in Paris?"), + call_msg, + text_msg("tool", "{\"temp_c\": 21}"), + text_msg("tool", "{\"temp_c\": 24}"), + ], + &[hermes_wire(&tool)], + "", + ), + ); + assert_eq!(ours, reference, "tool-call history render diverged"); +} + +/// Documented divergence: without a system turn, Qwen2.5's template injects +/// its own branding preamble where we inject a neutral one (one renderer +/// serves Qwen2 AND Qwen3, and Qwen3's template injects nothing — no single +/// default can match both). The delta must be EXACTLY that preamble swap. +#[test] +#[ignore = "needs MUMMU_TEMPLATE_PROBE + a cached Qwen2.5 checkpoint (MUMMU_QWEN2_DIR)"] +fn qwen2_no_system_defaults_diverge_only_by_the_documented_preamble() { + let Some(dir) = env_dir("MUMMU_QWEN2_DIR") else { + panic!("set MUMMU_QWEN2_DIR to a dir with tokenizer_config.json"); + }; + const QWEN_PREAMBLE: &str = + "You are Qwen, created by Alibaba Cloud. You are a helpful assistant."; + let tool = weather_tool(); + + // With tools: both sides synthesize a system turn; preambles differ. + let ours = ChatMl::qwen2().render_with_tools(std::slice::from_ref(&tool), &[Turn::user(USER)]); + let reference = reference_render( + &dir, + &input_json(&[text_msg("user", USER)], &[hermes_wire(&tool)], ""), + ); + let ours_with_qwen_preamble = ours.replacen(SYSTEM, QWEN_PREAMBLE, 1); + assert_ne!(ours_with_qwen_preamble, ours, "preamble must be present"); + assert_eq!( + ours_with_qwen_preamble, reference, + "no-system tools render must diverge ONLY by the default preamble" + ); + + // Without tools: the template injects a whole default system turn; our + // render() injects nothing (explicit turns are the caller's contract). + let ours = ChatMl::qwen2().render(&[Turn::user(USER)]); + let reference = reference_render(&dir, &input_json(&[text_msg("user", USER)], &[], "")); + assert_eq!( + format!("<|im_start|>system\n{QWEN_PREAMBLE}<|im_end|>\n{ours}"), + reference, + "no-system plain render must diverge ONLY by the injected default turn" + ); +} + +// ---- Qwen3 ------------------------------------------------------------------ + +#[test] +#[ignore = "needs MUMMU_TEMPLATE_PROBE + a cached Qwen3 checkpoint (MUMMU_QWEN3_DIR)"] +fn qwen3_plain_and_tools_renders_match_the_imported_template() { + let Some(dir) = env_dir("MUMMU_QWEN3_DIR") else { + panic!("set MUMMU_QWEN3_DIR to a dir with tokenizer_config.json"); + }; + let ours = ChatMl::qwen2().render(&[Turn::system(SYSTEM), Turn::user(USER)]); + let reference = reference_render( + &dir, + &input_json( + &[text_msg("system", SYSTEM), text_msg("user", USER)], + &[], + "", + ), + ); + assert_eq!(ours, reference, "Qwen3 plain render diverged"); + + let tool = weather_tool(); + let ours = ChatMl::qwen2().render_with_tools( + std::slice::from_ref(&tool), + &[Turn::system(SYSTEM), Turn::user(USER)], + ); + let reference = reference_render( + &dir, + &input_json( + &[text_msg("system", SYSTEM), text_msg("user", USER)], + &[hermes_wire(&tool)], + "", + ), + ); + assert_eq!(ours, reference, "Qwen3 tools render diverged"); +} + +/// Documented divergences vs Qwen3's template: (a) with tools and no system +/// turn it injects NO preamble (ours injects the neutral one); (b) it strips +/// `` reasoning from assistant turns at or before the last user query +/// (ours passes history through verbatim — a `ChatMl::qwen3()` with +/// think-stripping is a ROADMAP item). Pin both deltas exactly. +#[test] +#[ignore = "needs MUMMU_TEMPLATE_PROBE + a cached Qwen3 checkpoint (MUMMU_QWEN3_DIR)"] +fn qwen3_divergences_are_exactly_the_documented_ones() { + let Some(dir) = env_dir("MUMMU_QWEN3_DIR") else { + panic!("set MUMMU_QWEN3_DIR to a dir with tokenizer_config.json"); + }; + let tool = weather_tool(); + let ours = ChatMl::qwen2().render_with_tools(std::slice::from_ref(&tool), &[Turn::user(USER)]); + let reference = reference_render( + &dir, + &input_json(&[text_msg("user", USER)], &[hermes_wire(&tool)], ""), + ); + assert_eq!( + ours.replacen(&format!("{SYSTEM}\n\n"), "", 1), + reference, + "Qwen3 no-system tools render must diverge ONLY by our neutral preamble" + ); + + let think_turn = "2 then 3.The first primes are 2 and 3."; + let ours = ChatMl::qwen2().render(&[ + Turn::system(SYSTEM), + Turn::user(USER), + Turn::assistant(think_turn), + Turn::user("And the next two?"), + ]); + let reference = reference_render( + &dir, + &input_json( + &[ + text_msg("system", SYSTEM), + text_msg("user", USER), + text_msg("assistant", think_turn), + text_msg("user", "And the next two?"), + ], + &[], + "", + ), + ); + assert!(ours.contains(""), "ours re-renders history verbatim"); + assert!( + !reference.contains(""), + "Qwen3's template strips history reasoning" + ); + assert_eq!( + ours.replacen("2 then 3.", "", 1), + reference, + "the think block must be the ONLY delta" + ); +} + +// ---- LFM2.5 ----------------------------------------------------------------- + +const LFM_BOS: &str = "\"bos_token\": \"<|startoftext|>\""; + +#[test] +#[ignore = "needs MUMMU_TEMPLATE_PROBE + a cached LFM2.5 checkpoint (MUMMU_LFM2_DIR)"] +fn lfm2_plain_renders_match_the_imported_template() { + let Some(dir) = env_dir("MUMMU_LFM2_DIR") else { + panic!("set MUMMU_LFM2_DIR to a dir with chat_template.jinja"); + }; + let ours = ChatMl::lfm2().render(&[Turn::user(USER)]); + let reference = reference_render(&dir, &input_json(&[text_msg("user", USER)], &[], LFM_BOS)); + assert_eq!(ours, reference, "LFM no-system render diverged"); + + let ours = ChatMl::lfm2().render(&[Turn::system(SYSTEM), Turn::user(USER)]); + let reference = reference_render( + &dir, + &input_json( + &[text_msg("system", SYSTEM), text_msg("user", USER)], + &[], + LFM_BOS, + ), + ); + assert_eq!(ours, reference, "LFM system render diverged"); +} + +#[test] +#[ignore = "needs MUMMU_TEMPLATE_PROBE + a cached LFM2.5 checkpoint (MUMMU_LFM2_DIR)"] +fn lfm2_tools_renders_match_the_imported_template() { + let Some(dir) = env_dir("MUMMU_LFM2_DIR") else { + panic!("set MUMMU_LFM2_DIR to a dir with chat_template.jinja"); + }; + let tool = weather_tool(); + // LFM tools are BARE tool JSON (no Hermes wrapper) — the template runs + // `tool | tojson` on whatever is passed; the model card shows bare. + let bare = py_json(&tool); + + // With an explicit system turn… + let ours = ChatMl::lfm2().render_with_tools( + std::slice::from_ref(&tool), + &[Turn::system(SYSTEM), Turn::user(USER)], + ); + let reference = reference_render( + &dir, + &input_json( + &[text_msg("system", SYSTEM), text_msg("user", USER)], + std::slice::from_ref(&bare), + LFM_BOS, + ), + ); + assert_eq!(ours, reference, "LFM tools+system render diverged"); + + // …and without one BOTH sides inject nothing (byte-equal, no delta). + let ours = ChatMl::lfm2().render_with_tools(std::slice::from_ref(&tool), &[Turn::user(USER)]); + let reference = reference_render( + &dir, + &input_json(&[text_msg("user", USER)], &[bare], LFM_BOS), + ); + assert_eq!(ours, reference, "LFM tools no-system render diverged"); +} + +#[test] +#[ignore = "needs MUMMU_TEMPLATE_PROBE + a cached LFM2.5 checkpoint (MUMMU_LFM2_DIR)"] +fn lfm2_history_think_stripping_and_tool_turns_match_the_imported_template() { + let Some(dir) = env_dir("MUMMU_LFM2_DIR") else { + panic!("set MUMMU_LFM2_DIR to a dir with chat_template.jinja"); + }; + // Past assistant turns lose their reasoning on both sides; the LAST + // assistant turn keeps it (keep_past_thinking=false semantics). + let past = "2, 3.\n\nThe first two primes are 2 and 3."; + let last = "5, 7 next.\n\n5 and 7."; + let ours = ChatMl::lfm2().render(&[ + Turn::user(USER), + Turn::assistant(past), + Turn::user("And the next two?"), + Turn::assistant(last), + Turn::user("Thanks — one more?"), + ]); + let reference = reference_render( + &dir, + &input_json( + &[ + text_msg("user", USER), + text_msg("assistant", past), + text_msg("user", "And the next two?"), + text_msg("assistant", last), + text_msg("user", "Thanks — one more?"), + ], + &[], + LFM_BOS, + ), + ); + assert_eq!(ours, reference, "LFM think-stripping semantics diverged"); + + // A Pythonic call turn + real `tool` role turns. + let calls = [ToolCall { + name: "get_weather".into(), + arguments: serde_json::json!({"city": "Paris"}), + }]; + let call_turn = Turn::assistant_tool_calls_lfm(&calls); + let ours = ChatMl::lfm2().render(&[ + Turn::user("Weather in Paris?"), + call_turn.clone(), + Turn::tool_response("{\"temp_c\": 21}"), + ]); + let reference = reference_render( + &dir, + &input_json( + &[ + text_msg("user", "Weather in Paris?"), + text_msg("assistant", &call_turn.content), + text_msg("tool", "{\"temp_c\": 21}"), + ], + &[], + LFM_BOS, + ), + ); + assert_eq!(ours, reference, "LFM pythonic call/tool turns diverged"); +} diff --git a/tools/template-probe/Cargo.lock b/tools/template-probe/Cargo.lock new file mode 100644 index 0000000..2096fa6 --- /dev/null +++ b/tools/template-probe/Cargo.lock @@ -0,0 +1,170 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 4 + +[[package]] +name = "equivalent" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "877a4ace8713b0bcf2a4e7eec82529c029f1d0619886d18145fea96c3ffe5c0f" + +[[package]] +name = "hashbrown" +version = "0.17.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed5909b6e89a2db4456e54cd5f673791d7eca6732202bbf2a9cc504fe2f9b84a" + +[[package]] +name = "hf-chat-template" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "15e938b1f75afdc8eed9071ff130befb7d014f9b09dbd8d81877ade94ee9f9fe" +dependencies = [ + "minijinja", + "minijinja-contrib", + "serde", + "serde_json", +] + +[[package]] +name = "indexmap" +version = "2.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d466e9454f08e4a911e14806c24e16fba1b4c121d1ea474396f396069cf949d9" +dependencies = [ + "equivalent", + "hashbrown", +] + +[[package]] +name = "itoa" +version = "1.0.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f42a60cbdf9a97f5d2305f08a87dc4e09308d1276d28c869c684d7777685682" + +[[package]] +name = "memchr" +version = "2.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf8baf1c55e62ffcace7a9f06f4bd9cd3f0c4beb022d3b367256b91b87513d98" + +[[package]] +name = "memo-map" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "38d1115007560874e373613744c6fba374c17688327a71c1476d1a5954cc857b" + +[[package]] +name = "minijinja" +version = "2.21.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cb3d648e68cea56d9858d535ee28f9538404e2dd8cb08ed0bd05dca379477f39" +dependencies = [ + "indexmap", + "memo-map", + "serde", + "serde_json", +] + +[[package]] +name = "minijinja-contrib" +version = "2.21.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85342f6fac0be8ccd5bd00d9066be538f34f393f577b75d81b17c8398a6b43bb" +dependencies = [ + "minijinja", + "serde", +] + +[[package]] +name = "proc-macro2" +version = "1.0.107" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "985e7ec9bb745e6ce6535b544d84d6cd6f7ad8bd711c398938ae983b91a766d9" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "quote" +version = "1.0.47" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fbf4db142a473a8d80c26bbf18454ed458bf8d26c8219c331daecfdbd079001" +dependencies = [ + "proc-macro2", +] + +[[package]] +name = "serde" +version = "1.0.229" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4148590afebada386688f18773da617792bf2ef03ffc1e4cbd2b1d45b023e0ba" +dependencies = [ + "serde_core", + "serde_derive", +] + +[[package]] +name = "serde_core" +version = "1.0.229" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "67dca2c9c51e58a4791a4b1ed58308b39c64224d349a935ab5039aa360942a48" +dependencies = [ + "serde_derive", +] + +[[package]] +name = "serde_derive" +version = "1.0.229" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e7a5d71263a5a7d47b41f6b3f06ba276f10cc18b0931f1799f710578e2309348" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "serde_json" +version = "1.0.151" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c841b55ecdae098c80dcae9cf767f6f8a0c2cdb3416bbef72181df4d0fe73f14" +dependencies = [ + "indexmap", + "itoa", + "memchr", + "serde", + "serde_core", + "zmij", +] + +[[package]] +name = "syn" +version = "3.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "53e9bae58849f64dfa4f5d5ae372c8341f7305f82a3868709269343628b659a3" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "template-probe" +version = "0.1.0" +dependencies = [ + "hf-chat-template", + "serde_json", +] + +[[package]] +name = "unicode-ident" +version = "1.0.24" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6e4313cd5fcd3dad5cafa179702e2b244f760991f45397d14d4ebf38247da75" + +[[package]] +name = "zmij" +version = "1.0.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "29666d0abbfad1e3dc4dcf6144730dd3a3ab225bbbdac83319345b1b44ccfc1b" diff --git a/tools/template-probe/Cargo.toml b/tools/template-probe/Cargo.toml new file mode 100644 index 0000000..bdf9fae --- /dev/null +++ b/tools/template-probe/Cargo.toml @@ -0,0 +1,22 @@ +# A standalone reference renderer for the P3/P4 template byte gate: render a +# checkpoint's own `chat_template` Jinja exactly as Python +# `transformers.apply_chat_template` would (hf-chat-template = minijinja + a +# transformers compatibility layer, corpus-checked byte-identical upstream), +# so crates/mummu/tests/template_gate.rs can assert our hardcoded `chat` +# renderers reproduce the imported template byte-for-byte. Deliberately +# OUTSIDE the cargo workspace: hf-chat-template enables serde_json's +# `preserve_order` feature, and features being additive, a dev-dependency +# would silently flip mummu's own test builds from serde_json's default +# (sorted) map order to insertion order — diverging the very renders the +# tests byte-verify from what plain-feature consumers build. +[package] +name = "template-probe" +version = "0.1.0" +edition = "2021" +publish = false + +[dependencies] +hf-chat-template = "0.2.1" +serde_json = "1" + +[workspace] diff --git a/tools/template-probe/src/main.rs b/tools/template-probe/src/main.rs new file mode 100644 index 0000000..8eab819 --- /dev/null +++ b/tools/template-probe/src/main.rs @@ -0,0 +1,58 @@ +//! Render a HuggingFace `chat_template` the way Python +//! `transformers.apply_chat_template` does, for mummu's template byte gate +//! (`crates/mummu/tests/template_gate.rs`). +//! +//! Usage: `template-probe < render-input.json` +//! +//! - `argv[1]`: path to a file holding the raw Jinja template text. +//! - stdin: one `RenderInput` JSON object — `messages`, optional `tools`, +//! `add_generation_prompt`, plus any extra context keys a template reads +//! (e.g. `bos_token`). Key order in the JSON text is preserved into +//! `| tojson` (hf-chat-template builds serde_json with `preserve_order`). +//! - stdout: the rendered prompt, raw bytes, no added trailing newline. +//! - Any failure: message on stderr, exit code 1. + +use std::io::{Read, Write}; + +fn main() { + let Some(path) = std::env::args().nth(1) else { + die("usage: template-probe < render-input.json"); + }; + let template = match std::fs::read_to_string(&path) { + Ok(t) => t, + Err(e) => die(&format!("read {path}: {e}")), + }; + if template.trim().is_empty() { + die(&format!("{path}: template file is empty")); + } + + let mut input_json = String::new(); + if let Err(e) = std::io::stdin().read_to_string(&mut input_json) { + die(&format!("read stdin: {e}")); + } + let input: hf_chat_template::RenderInput = match serde_json::from_str(&input_json) { + Ok(i) => i, + Err(e) => die(&format!("render-input json: {e}")), + }; + if input.messages.is_empty() { + die("render-input has no messages"); + } + + let tmpl = match hf_chat_template::ChatTemplate::from_str(&template) { + Ok(t) => t, + Err(e) => die(&format!("template compile: {e}")), + }; + match tmpl.render(&input) { + Ok(out) => { + if std::io::stdout().write_all(out.as_bytes()).is_err() { + die("write stdout"); + } + } + Err(e) => die(&format!("render: {e}")), + } +} + +fn die(msg: &str) -> ! { + eprintln!("template-probe: {msg}"); + std::process::exit(1) +} From 5f85cfd1ec744bda8933546e936ca3b14dc8c978 Mon Sep 17 00:00:00 2001 From: Justin Icenhour Date: Fri, 24 Jul 2026 07:53:48 -0500 Subject: [PATCH 5/6] docs: tick burn.toml + template-gate items; fold run findings into ROADMAP/README MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ROADMAP: perf-section burn.toml item [x] (discovery semantics + A/B proof); P3 hf-chat-template item [x] payoff (1) with the py_json divergence story; new [ ]s — ChatMl::qwen3() history think-stripping (byte-gate finding), general template-fallback renderer (payoff 2, preserve_order caveat), P6 one-process Gpu/GpuF16 dtype-policy hazard (root cause of the pre-existing real_qwen3 failure), P9 KV-cache FP8/e4m3 quantization (vLLM April 2026 lessons, 2026-07-24 research). P4 note: template-embedded tool JSON now spells json.dumps separators. README: the function-calling feature now claims byte-identity with transformers' renders, gated by the 9-case template_gate. Co-Authored-By: Claude Opus 4.8 --- README.md | 8 +++++- ROADMAP.md | 75 +++++++++++++++++++++++++++++++++++++++++++++++++++--- 2 files changed, 79 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index 4ca584c..67fdf2a 100644 --- a/README.md +++ b/README.md @@ -85,7 +85,13 @@ It exists because two local-first apps — **[laurelane](https://github.com/phys parsers are bounded with a loud error taxonomy. Proven end-to-end on the real GPU: Qwen2.5-1.5B emitted a parseable Hermes call and LFM2.5-1.2B emitted exactly `<|tool_call_start|>[get_weather(city="Paris")]<|tool_call_end|>` (`tests/real_toolcall.rs`, - `tests/real_toolcall_lfm.rs`). + `tests/real_toolcall_lfm.rs`). And the renders are **byte-identical to + `transformers.apply_chat_template`** running each checkpoint's own imported template — plain, + tools, and tool-call-history shapes for Qwen2.5/Qwen3/LFM2.5 all pass a 9-case byte gate + (`tests/template_gate.rs` against the `tools/template-probe` reference renderer; embedded tool JSON + spells json.dumps' separators via `chat::py_json`, the same spelling the models emit back); the only + family divergences (default no-system preambles, Qwen3 history think-stripping) are pinned to their + exact deltas in the gate. - **f16 inference, validated** — Qwen2.5-1.5B runs coherently on `GpuF16` (weights + KV in f16, the q·kᵀ attention scores + softmax computed in an f32 island to stop f16 overflow): **~3.6 GiB runner VRAM vs ~7.9 GiB f32, at identical speed**; the parity gate re-passes unchanged on f32, where the diff --git a/ROADMAP.md b/ROADMAP.md index 2c671a6..441bd98 100644 --- a/ROADMAP.md +++ b/ROADMAP.md @@ -72,7 +72,7 @@ a benchmark holds/improves its budget; README perf claims link an artifact. on an otherwise-idle GPU (5% util). Decode throughput tracking CPU availability is what a dispatch-bound path looks like. Operationally: **run the budget gates on a quiet machine** or they report contention as a regression.* -- [ ] Evaluate Burn 0.21's `burn.toml` project config — per-subsystem tuning + a CubeCL kernel-validation +- [x] Evaluate Burn 0.21's `burn.toml` project config — per-subsystem tuning + a CubeCL kernel-validation layer without recompiling; useful as a debug switch for kernel-level parity hunts — https://burn.dev/blog/release-0.21.0/ *(2026-07-17 research)* Concretely, a `burn.toml` dropped at the project root parameterizes every internal subsystem with no code change / no recompile: @@ -83,7 +83,17 @@ a benchmark holds/improves its budget; README perf claims link an artifact. `STATUS_STACK_OVERFLOW`-adjacent GPU crash). Action when picked up: commit a checked-in `burn.toml` with validation ON for the parity/real-model test profiles (catch OOB in CI) and OFF for the benchmark profile (no validation overhead in the budget numbers), and re-confirm the - budgets are unmoved by its presence. + budgets are unmoved by its presence. *(2026-07-24)* **Shipped exactly that shape**, exploiting the + discovery rule read from cubecl-runtime 0.10 source (`RuntimeConfig::from_current_dir` walks UP + from the process CWD, `cubecl.toml` checked before `burn.toml` at each level, first hit wins; cargo + runs each crate's tests/benches with CWD = the package dir): a repo-root **`burn.toml`** sets + `[cubecl.compilation] check_mode = "validate"` (bounds-check every launch AND validate + explicitly-unchecked kernels for OOB) so the `crates/mummu` parity/real-model suites run armed, + and **`crates/mummu-bench/cubecl.toml`** opts the budget/bench crate back to the `auto` default so + recorded numbers never carry validation overhead. Consumers run from their own CWD — untouched. + Verified live: an A/B with a malformed root burn.toml makes a GPU test fail at config load (proof + the file is discovered + parsed), the real-model GPU suite passes with validation armed (no OOB + found — clean bill), and the budget gates hold their numbers from the opted-out bench crate. - [ ] Evaluate **CubeCL's now-complete flash-attention kernel** for the decode/prefill attention step — the releases page reports a full implementation (causal **masking**, partitions, row-wise reductions, multi-plane ops). Mummu currently materializes attention explicitly (q·kᵀ → f32 softmax @@ -528,7 +538,7 @@ The subsystem that turns "a model on HuggingFace or on disk" into a loaded, pari `eos_token_id` is already the source of truth and is now cross-checked to agree, and BOS/tokenization stays caller-side by design; surfacing the parsed `TokenizerConfig` on the `Loaded*` structs for a consumer to read is the intended shape, left to a dedicated decision. -- [ ] **Evaluate `hf-chat-template` to render the imported `chat_template`** — Mummu's prompt wrapping is +- [x] **Evaluate `hf-chat-template` to render the imported `chat_template`** — Mummu's prompt wrapping is hardcoded, byte-verified `chat` renderers (one per family); the `hf-chat-template` crate (built on **minijinja** + a transformers compatibility layer) renders an arbitrary HF `chat_template` Jinja string **byte-identically to `transformers.apply_chat_template`**, tools included. Two payoffs to weigh: (1) a @@ -548,6 +558,42 @@ The subsystem that turns "a model on HuggingFace or on disk" into a loaded, pari with `ChatMl::{qwen2,lfm2}().render_with_tools(...)` on the parity-committed prompts would turn "template-vs-renderer consistency" into a true byte gate. Still gate on reproducing the committed prompts byte-for-byte before trusting it. — https://lib.rs/crates/hf-chat-template + *(2026-07-24)* **Payoff (1) shipped — the byte gate is real and PASSES 9/9.** Two build decisions: + (a) hf-chat-template 0.2.1 is NOT a dev-dependency — it enables serde_json's `preserve_order` + feature, and features being additive that would silently flip our own test builds from serde_json's + default (sorted) map order to insertion order, diverging the very renders the tests verify from what + plain-feature consumers build; it lives in **`tools/template-probe`** (own workspace, the + candle-probe pattern), a 60-line bin: argv = template file, stdin = RenderInput JSON, stdout = the + transformers-identical render. (b) The gate exposed a REAL byte divergence and it is now FIXED: + our renders embedded tool JSON compact (`{"a":1}`) where every checkpoint template runs Jinja + `tojson` = Python `json.dumps` separators (`{"a": 1}`) — and the models emit that spacing back + (the 07-11 real-GPU proof literally logged `{"name": "get_weather", "arguments": {"city": + "Paris"}}`). `chat::py_json` (a `json.dumps`-separator serde Formatter, public — consumers + composing history turns need the same spelling) now feeds all template-embedded JSON. + `tests/template_gate.rs` (ignored, env-keyed `MUMMU_TEMPLATE_PROBE` + the three checkpoint dirs) + byte-compares our renderers against each checkpoint's OWN imported template (through + `TokenizerConfig::from_dir`, incl. LFM's standalone `chat_template.jinja` — fetched into the local + cache this run) rendered by the reference engine: **Qwen2.5 plain/tools/tool-call-history, Qwen3 + plain/tools, LFM2.5 plain±system/tools±system/think-stripping/pythonic-call+tool turns — all + byte-identical**; the only divergences are pinned to their exact deltas (Qwen2.5's no-system + branding preamble "You are Qwen, …" vs our neutral one, Qwen3's no-system no-preamble, Qwen3's + history think-stripping) so any OTHER drift fails loudly. Payoff (2) — a general fallback renderer + for family-less checkpoints — stays open below. +- [ ] **`ChatMl::qwen3()` with history think-stripping** — the byte gate documented that Qwen3's template + strips `…` reasoning from assistant turns at/before the last user query while our + shared `ChatMl::qwen2()` renderer re-renders history verbatim (fine for fresh prompts + tool loops, + wrong for long multi-turn chats with a thinking Qwen3). A `qwen3()` constructor wants the LFM-style + strip (the machinery exists — `turn_content` already does it for `Lfm`) but keyed to Qwen3's + "at/before the last user query" rule rather than LFM's "every but the last assistant turn"; gate it + on the template byte gate's think case flipping from documented-divergence to byte-equal. + *(2026-07-24, found by the byte gate.)* +- [ ] **General fallback chat renderer from the imported template** — payoff (2) of the hf-chat-template + evaluation: for a checkpoint whose family has no hardcoded renderer, render prompts from its own + imported `chat_template` (the byte gate now proves the engine is transformers-identical on our + three families). Weigh the minijinja dependency against the from-scratch ethos — as a *runtime* + feature it puts a Jinja engine on the prompt path, so it wants an explicit opt-in surface (e.g. + `chat::from_template(&TokenizerConfig)`) and the preserve_order caveat re-examined (the probe + sidesteps it today by living out-of-workspace). *(2026-07-24, split from the evaluation item.)* - [x] **Model registry / manifest** — a declarative `ModelSpec` (repo, architecture, weight format, dtype, tokenizer, chat template, size tier) + a small built-in catalog of known-good models (Qwen2.5, LFM2.5, MiniLM, …); adding a model = a manifest entry. *(2026-07-10) `mummu::registry`: `ModelSpec` @@ -605,6 +651,10 @@ The subsystem that turns "a model on HuggingFace or on disk" into a loaded, pari `<|tool_call_start|>[get_weather(city="Paris")]<|tool_call_end|>` from our rendered prompt and the parser round-tripped it; the Qwen2 parity gate re-passed both legs after the template refactor (max |Δlogit| 2.670e-5, Ollama greedy byte-identical). 16 new unit tests (109 total).* + *(2026-07-24) Template-embedded tool JSON now spells `json.dumps` separators (`chat::py_json` — + `{"a": 1}`, not compact `{"a":1}`) in BOTH conventions: the P3 template byte gate proved the + checkpoints' own templates (Jinja `tojson`) and the models' own emissions use that spacing, and + the renders are now byte-identical to `transformers.apply_chat_template` — see the P3 gate item.* *(2026-07-10 research)* 2026 community numbers back the plan: Qwen3-8B keeps tool-calling score through Q4_K_M (0.919 quantized vs 0.933 full — quant does NOT cost tool reliability, good news for P9); BFCL shows a capability cliff below ~7B (Qwen3.5-9B 66.1% vs 4B 50.3%), so the zoo's @@ -673,6 +723,15 @@ that fits the model AND uses every device to the fullest. under the 150 ms ceiling); VRAM peak unchanged (11.5 GiB whole-card). Parity gate byte-identical (max |Δlogit| 2.670e-5, Ollama greedy exact); f16 island + CPU budget gates re-passed (108.4 ms / 11.7 tok/s GPU gate, 13.2 tok/s CPU). `bench/BASELINE.md` re-baselined.* +- [ ] **One process cannot mix `Gpu` (f32) and `GpuF16` on the same device** — instantiating `GpuF16` + flips Burn's per-device default dtype policy process-wide: a `Gpu` f32 forward that runs afterwards + panics reading its logits back (`TypeMismatch: expected F16, got F32`). Found 2026-07-24 as a + deterministic pre-existing failure of the old mixed `real_qwen3` suite (reproduced serially on + unmodified HEAD, exonerating that run's changes); the test-side fix is the convention that every + `GpuF16` leg lives in `real_f16.rs` (own process — qwen3's f16 leg moved there). But the LIBRARY + hazard remains for consumers: the P6 precision-*selection* work (pick f16 vs f32 at runtime) must + either pin one dtype alias per process, isolate per-device policy, or root-cause the Burn policy + flip upstream before an app can switch precision on the fly. - [ ] **Placement plan** — given model size + KV-cache + display headroom and the device set, choose a **fit-and-fill** plan: single GPU when it fits; **shard layers across multiple GPUs** (pipeline/ layer-parallel over Burn's multi-device tensors — Burn gives the multi-device *primitives*, not automatic @@ -758,6 +817,16 @@ The VRAM lever the P6 planner pulls to make the largest useful model fit the use the kernel substrate a Q4-weights × f16-activations decode path would ride (vs hand-writing a dequant-fused kernel); gate any adoption on the parity harness + `bench/BASELINE.md` — https://github.com/tracel-ai/cubecl/releases · https://burn.dev/blog/release-0.21.0/ +- [ ] **KV-cache quantization (FP8/e4m3)** — quantize the KV cache (and optionally the QK/ScoreV attention + matmuls) to 8-bit, halving per-token cache footprint — the *other* VRAM lever besides weights, and the + one that grows with context length. vLLM shipped exactly this (April 2026) and published the lessons + that transfer: uncalibrated per-head e4m3 scales recover 97%+ on reasoning tasks and 94–98% AUC at + 1M-token contexts; **two-level accumulation is critical** (intermediate f32 writes on long contexts — + the same failure our f32-softmax island guards); layer-selective beats uniform (sliding-window layers + pay overhead for no benefit); head_dim 256 loses at prefill (~1.6× register pressure) while 64/128 win. + For Mummu: our KV cache is f16 on `GpuF16` — an e4m3-quantized cache would halve it again; gate on the + parity harness + budgets like every numeric change. — https://vllm.ai/blog/2026-04-22-fp8-kvcache + *(2026-07-24 research)* - [ ] **Auto-quantize-to-fit** — the planner picks the *highest* precision that fits the detected VRAM (f16 → int8 → int4), reports the quality/size trade, and never silently ships a worse tier than asked. From 0acb591065ed797f9b24ec1ed160e37fb52b9d27 Mon Sep 17 00:00:00 2001 From: Justin Icenhour Date: Fri, 24 Jul 2026 13:16:44 -0500 Subject: [PATCH 6/6] =?UTF-8?q?docs(p6):=20settle=20the=20heterogeneous=20?= =?UTF-8?q?per-device=20precision=20design=20=E2=80=94=20one=20source=20fi?= =?UTF-8?q?le,=20derived=20per-stage=20representations,=20no=20new=20forma?= =?UTF-8?q?t?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Folds the design discussion into the placement-plan item: bf16 (or GGUF) stays the single stored artifact; f32/f16 stages are load-time casts (CastFloatAdapter), the CPU stage is the P9 keep-quantized leg (block-wise Q4_K-style on-the-fly quant preferred over naive RTN); a derived-artifact cache is a later optimization (ordinary shards keyed by source hash + dtype + layer range — a cache, not a format). Runtime dependency order recorded: cross-device dtype-policy experiment first, then the stage-composed model type (one Wgpu backend multi-dtype for GPU+iGPU; a backend-type seam for CPU), activation casts at seams, per-stage KV shards + schedule. Planner expectation: pipeline throughput = slowest stage — small stages are for FIT, not speed. Co-Authored-By: Claude Opus 4.8 --- ROADMAP.md | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/ROADMAP.md b/ROADMAP.md index 441bd98..9738c4a 100644 --- a/ROADMAP.md +++ b/ROADMAP.md @@ -737,6 +737,30 @@ that fits the model AND uses every device to the fullest. layer-parallel over Burn's multi-device tensors — Burn gives the multi-device *primitives*, not automatic tensor-parallel, so we place modules on devices ourselves); **spill cold layers to CPU** (GGUF-style hybrid) when total VRAM is short. Largest-model-that-fits, every device busy. + *(2026-07-24, design settled)* **Heterogeneous per-device precision comes from ONE source file — + no new format.** The checkpoint stores weights once (bf16 safetensors, or a GGUF); each pipeline + stage *derives* its own in-memory representation at load: cast bf16→f32 for the big GPU and + bf16→f16 for a SHADER_F16-capable iGPU (both = the existing `CastFloatAdapter` path, quality-free + casts), and quantize int8/int4 for the CPU stage (the P9 keep-quantized leg; naive round-to-nearest + from bf16 is worse than a calibrated GPTQ/K-quant artifact, so prefer an on-the-fly *block-wise* + Q4_K-style quant — llama.cpp's offline K-quants are data-free, same math). The inverse also holds + and already runs: one Q4_K_M GGUF can serve f32/f16 stages by dequant/upcast (at Q4 quality) and + the CPU stage keep-quantized — so "one file" is a quality-vs-disk choice (bf16 = quality-max, + GGUF = size-min), never a format question. The only format-adjacent addition is a later + **derived-artifact cache** (don't re-quantize 30 layers per launch): a per-user cache dir of + ordinary safetensors/GGUF shards keyed by (source hash, dtype, layer range) — a cache, not a + format. The real work is runtime, in dependency order: (1) test the dtype-policy hazard above + *across* devices first — the flip is per-device, so f32-on-discrete + f16-on-iGPU in one process + is expected to work but is exactly the unproven experiment; (2) the stage-composed model type — + `CausalLm` is generic over ONE backend, a GPU+iGPU split can stay one `Wgpu` backend with + per-tensor dtypes (Burn 0.21 multi-dtype, the f32-softmax island already does per-tensor casts), + but the CPU stage is a different backend *type* (`burn-flex`), so the GPU→CPU seam is a + host-memory transfer between two backend generics; (3) activations cast at stage seams (small + tensors, cheap); (4) per-stage KV-cache shards + the micro-batch schedule (the multi-GPU item). + Expectation to encode in the planner: pipeline throughput = the slowest stage, so iGPU/CPU stages + exist to make a model FIT, not to make a fitting model faster — fit-and-fill, per-device precision + picked by what fits + what the device advertises (`inventory()` already records SHADER_F16 + + max_buffer_bytes + true VRAM per adapter). - [ ] **Multi-GPU execution** — run the sharded plan: per-device sub-modules, activations handed across the device boundary between stages, KV-cache per shard, and a micro-batch/pipeline schedule so the GPUs overlap rather than idle. *(Tensor-parallel within a layer is the stretch goal; layer/pipeline split is