diff --git a/Cargo.lock b/Cargo.lock index 73175b0..8f42e8d 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1067,9 +1067,9 @@ dependencies = [ [[package]] name = "clap" -version = "4.6.3" +version = "4.6.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0fb99565819980999fb7b4a1796046a5c949e6d4ff132cf5fadf5a641e20d776" +checksum = "d91e0c145792ef73a6ad36d27c75ac09f1832222a3c209689d90f534685ee5b7" dependencies = [ "clap_builder", ] @@ -2706,9 +2706,9 @@ dependencies = [ [[package]] name = "glob" -version = "0.3.3" +version = "0.3.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0cc23270f6e1808e30a928bdc84dea0b9b4136a8bc82338574f23baf47bbd280" +checksum = "e4eba85ea1d0a966a983acd07deee566e67395d2d96b6fb39e62b5a833f1eb0b" [[package]] name = "glow" @@ -2855,6 +2855,18 @@ version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "dfa686283ad6dd069f105e5ab091b04c62850d3e4cf5d67debad1933f55023df" +[[package]] +name = "hf-chat-template" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "15e938b1f75afdc8eed9071ff130befb7d014f9b09dbd8d81877ade94ee9f9fe" +dependencies = [ + "minijinja", + "minijinja-contrib", + "serde", + "serde_json", +] + [[package]] name = "hmac" version = "0.12.1" @@ -3304,9 +3316,9 @@ checksum = "34b357333733e8260735ba5894eb928c02ecc69c78715f01a8019e7fa7f2db4c" [[package]] name = "libc" -version = "0.2.186" +version = "0.2.189" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "68ab91017fe16c622486840e4c83c9a37afeff978bd239b5293d61ece587de66" +checksum = "3eaf3ede3fee6db1a4c2ee091bf8a8b4dccdc6d17f656fb07896ee72867612f2" [[package]] name = "libfuzzer-sys" @@ -3517,6 +3529,34 @@ dependencies = [ "stable_deref_trait", ] +[[package]] +name = "memo-map" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "38d1115007560874e373613744c6fba374c17688327a71c1476d1a5954cc857b" + +[[package]] +name = "minijinja" +version = "2.21.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cb3d648e68cea56d9858d535ee28f9538404e2dd8cb08ed0bd05dca379477f39" +dependencies = [ + "indexmap", + "memo-map", + "serde", + "serde_json", +] + +[[package]] +name = "minijinja-contrib" +version = "2.21.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85342f6fac0be8ccd5bd00d9066be538f34f393f577b75d81b17c8398a6b43bb" +dependencies = [ + "minijinja", + "serde", +] + [[package]] name = "minimal-lexical" version = "0.2.1" @@ -3590,6 +3630,7 @@ dependencies = [ "burn-flex", "burn-store", "half", + "hf-chat-template", "once_cell", "pollster", "serde", @@ -4900,7 +4941,7 @@ checksum = "e7a5d71263a5a7d47b41f6b3f06ba276f10cc18b0931f1799f710578e2309348" dependencies = [ "proc-macro2", "quote", - "syn 3.0.2", + "syn 3.0.3", ] [[package]] @@ -4909,6 +4950,7 @@ version = "1.0.151" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c841b55ecdae098c80dcae9cf767f6f8a0c2cdb3416bbef72181df4d0fe73f14" dependencies = [ + "indexmap", "itoa", "memchr", "serde", @@ -5137,9 +5179,9 @@ dependencies = [ [[package]] name = "syn" -version = "3.0.2" +version = "3.0.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a207d6d6a2b7fc470b80443726053f18a2481b7e1eee970597051596567987a3" +checksum = "53e9bae58849f64dfa4f5d5ae372c8341f7305f82a3868709269343628b659a3" dependencies = [ "proc-macro2", "quote", @@ -5285,7 +5327,7 @@ checksum = "43cbfe0cf76104d42a574802844187e84a305e531ed54455f11fbde0f10541cd" dependencies = [ "proc-macro2", "quote", - "syn 3.0.2", + "syn 3.0.3", ] [[package]] @@ -5435,9 +5477,9 @@ dependencies = [ [[package]] name = "tokio" -version = "1.53.0" +version = "1.53.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d988bcd52dbe076d3d46903332f58c912b87a2c49b1428419a5845154762ffee" +checksum = "202caea871b69668250d242070849eb495be178ed697a3e98aebce5bc81a0bed" dependencies = [ "bytes", "libc", @@ -5459,13 +5501,14 @@ dependencies = [ [[package]] name = "tokio-util" -version = "0.7.18" +version = "0.7.19" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9ae9cec805b01e8fc3fd2fe289f89149a9b66dd16786abd8b19cfa7b48cb0098" +checksum = "494815d09bf52b5548659851081238f0ca39ff638363907596da739561c62c52" dependencies = [ "bytes", "futures-core", "futures-sink", + "libc", "pin-project-lite", "tokio", ] @@ -6497,9 +6540,9 @@ checksum = "3ae8337f8a065cfc972643663ea4279e04e7256de865aa66fe25cec5fb912d3f" [[package]] name = "xxhash-rust" -version = "0.8.17" +version = "0.8.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "985eec839aaf2a1270af8f4ebcf63cf9401cfd90f0902f97c28d9f104ffbde72" +checksum = "aee1b19627c7c60102ab80d3a9cbe18de90bfe03bfa6c3715447681f0e8c8af6" [[package]] name = "y4m" @@ -6532,18 +6575,18 @@ dependencies = [ [[package]] name = "zerocopy" -version = "0.8.54" +version = "0.8.55" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b7cbbc0a705a0fd05cc3676525980d2bf5a9bc4adac6d6475209a7887cf59d19" +checksum = "b5a105cd7b140f6eeec8acff2ea38135d3cab283ada58540f629fe51e46696eb" dependencies = [ "zerocopy-derive", ] [[package]] name = "zerocopy-derive" -version = "0.8.54" +version = "0.8.55" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e2e817b7b52d0c7358d3246da9d69935ebb18116b2b102b4230dac079b4862f5" +checksum = "0fe976fb70c78cd64cccfe3a6fc142244e8a77b70959b30faf9d0ac37ee228eb" dependencies = [ "proc-macro2", "quote", diff --git a/Cargo.toml b/Cargo.toml index 6287bdc..86052cc 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -70,7 +70,14 @@ ureq = "3" sha2 = "0.11" serde = { version = "1", features = ["derive"] } -serde_json = "1" +# `preserve_order` (insertion-order maps) is deliberate and load-bearing for +# prompt bytes: Python/transformers renders tool JSON in dict insertion order, +# so our tool-bearing prompts only byte-match the reference stack's rendering +# with it on. It is also what the hf-chat-template dev-dependency (the +# template byte gate) requires — without it here, test builds would unify the +# feature on while production consumer builds leave it off, and the two would +# render DIFFERENT prompt bytes (found 2026-07-23 when the gate landed). +serde_json = { version = "1", features = ["preserve_order"] } thiserror = "2" once_cell = "1" diff --git a/README.md b/README.md index 4ca584c..e62a45e 100644 --- a/README.md +++ b/README.md @@ -48,7 +48,9 @@ It exists because two local-first apps — **[laurelane](https://github.com/phys family's renderer, or — when a `tokenizer.json` sits beside it — whose declared added-token ids don't match that real tokenizer, is a loud `ImportError::Inconsistent` instead of a model that silently mis-stops, mis-templates, or mis-tokenizes. Both sibling files are optional (a GGUF-derived dir has - neither → no behavior change). + neither → no behavior change). On a successful safetensors load the parsed `TokenizerConfig` is surfaced + on the returned `Loaded{Qwen2,Qwen3,Lfm2}` struct (`tokenizer_config`), so a consumer reads config-driven + EOS/BOS/PAD ids straight off the model; a GGUF load surfaces `None` (self-contained). - **Import validation** — a two-stage error taxonomy: `ImportError` for the file→module stage (missing file, parse, load, and an `Incomplete` per-tensor missing/errored diff) and `SanityError` for the runtime liveness a checked load can't see — NaN/Inf logits, a vocab-width mismatch, or a @@ -86,6 +88,12 @@ It exists because two local-first apps — **[laurelane](https://github.com/phys emitted a parseable Hermes call and LFM2.5-1.2B emitted exactly `<|tool_call_start|>[get_weather(city="Paris")]<|tool_call_end|>` (`tests/real_toolcall.rs`, `tests/real_toolcall_lfm.rs`). +- **Template byte gate** — the hardcoded renderers are proven **byte-identical to + `transformers.apply_chat_template`** rendering the checkpoint's own imported `chat_template` + (via the `hf-chat-template` dev-dependency): plain, multi-turn, the full Hermes `# Tools` block, + and function-call history all match byte-for-byte on Qwen3-0.6B (`tests/template_gate.rs`). + Prompt JSON deliberately serializes with Python `json.dumps` spacing and insertion-order keys + (serde_json `preserve_order`) — the exact bytes the reference stack renders and models emit back. - **f16 inference, validated** — Qwen2.5-1.5B runs coherently on `GpuF16` (weights + KV in f16, the q·kᵀ attention scores + softmax computed in an f32 island to stop f16 overflow): **~3.6 GiB runner VRAM vs ~7.9 GiB f32, at identical speed**; the parity gate re-passes unchanged on f32, where the diff --git a/ROADMAP.md b/ROADMAP.md index 2c671a6..7a34156 100644 --- a/ROADMAP.md +++ b/ROADMAP.md @@ -24,7 +24,12 @@ still pass, and every parity number is **bit-identical to 0.22** — LFM2.5-1.2B (lfm2), Qwen2-vs-Candle 2.29e-5 with the Ollama greedy leg exact; both tool-call emissions unchanged; budgets 105.5 ms / 12.2 tok/s GPU, 11.74 tok/s CPU. **wgpu 30 stays held** — it is not ours to pick, burn 0.21 resolves wgpu 29 transitively, so it unblocks with a burn bump, not a -`cargo upgrade`.* +`cargo upgrade`.* *(2026-07-22) Pin watch re-checked: burn is still 0.21 (no 0.22), tokenizers still +0.23.1 — the pinned combo is current. Worth knowing for the eventual burn bump: wgpu 30's changelog +lifts `SHADER_F16` to **all** shader kinds (WGSL included — previously SPIR-V passthrough only) and +adds the Vulkan f16 IO polyfill (PR #7884, already confirmed live here) — so when burn moves to +wgpu 30, f16 stops being SPIR-V-only and the WGSL fallback path (non-Vulkan APIs) can run f16 too — +https://github.com/gfx-rs/wgpu/blob/trunk/CHANGELOG.md.* ## North Star @@ -84,6 +89,21 @@ a benchmark holds/improves its budget; README perf claims link an artifact. `burn.toml` with validation ON for the parity/real-model test profiles (catch OOB in CI) and OFF for the benchmark profile (no validation overhead in the budget numbers), and re-confirm the budgets are unmoved by its presence. + *(2026-07-21 research)* **Caveat found — `burn.toml` has no per-profile mechanism.** The documented + schema is a single, global project-root file with flat sections (verified against the release notes): + `[fusion.beam_search] max_blocks`; `[cubecl.autotune] level = "minimal"|"balanced"|"extensive"|"full"`, + `cache = "local"|"target"|"global"|{file=…}`; `[cubecl.compilation] check_mode = + "enforce"|"validate"|"auto"`, `cache`; `[cubecl.streaming] max_streams`; `[cubecl.memory] + persistent_memory = "enabled"|"disabled"|"enforced"`. Validation is the `[cubecl.compilation] + check_mode` knob (`"validate"` = the OOB-catching layer). So the item's "ON for test / OFF for bench + **profiles**" premise isn't directly expressible — one checked-in file applies to *every* cargo + invocation, benches included. Workable routes when picked up: (a) accept global `check_mode = "validate"` + only if a bench A/B shows the budgets are unmoved by it (measure the overhead first — it may be + negligible and the whole ON/OFF split moot); (b) if it does move the budgets, drive it from an + **env-var/CI-only file** (drop `burn.toml` in the test/CI working dir but not where `cargo bench` runs, + or gate it behind a CI env var the bench job clears) rather than committing a repo-root file. Confirm + whether Burn reads `burn.toml` from cwd or repo-root (determines whether the per-dir trick works) before + adopting either. - [ ] Evaluate **CubeCL's now-complete flash-attention kernel** for the decode/prefill attention step — the releases page reports a full implementation (causal **masking**, partitions, row-wise reductions, multi-plane ops). Mummu currently materializes attention explicitly (q·kᵀ → f32 softmax @@ -240,6 +260,13 @@ a benchmark holds/improves its budget; README perf claims link an artifact. Qwen3.6 (35B-A3B) is also out now but is MoE and well past the single-card tier this zoo targets — https://huggingface.co/unsloth/Qwen3.5-4B-GGUF · https://unsloth.ai/docs/models/qwen3.5/gguf-benchmarks + *(2026-07-22 research)* The mid-2026 function-calling field guides converge on the same picture: for + ≤ 8 GB cards **Qwen3.5-9B** is the general-purpose FC pick and **Qwen3.5-4B** the CPU-only pick + (reinforces the 4B/9B target of this item); the Qwen3.6 tier (27B dense / 35B-A3B MoE) ships a new + `qwen3_coder` tool-call parser in vLLM/SGLang — if its emission format differs from Hermes ChatML, + a Qwen3.6-class port would need a new `chat` convention entry, so check the template's markers before + assuming Hermes covers it — https://insiderllm.com/guides/function-calling-local-llms/ · + https://www.popularai.org/p/best-cpu-only-local-llm-2026 - [x] **Qwen3 strict parity gate PASSED** — the Qwen3 dense arch is now through the P7 trust gate. `tests/parity_gguf.rs` gained a `qwen3` leg on the `llama_ref` harness: `llama-server` (Ollama's bundled binary) runs the SAME local Qwen3-0.6B Q4_K_M our loader loads, RAW `/completion` with @@ -502,7 +529,7 @@ The subsystem that turns "a model on HuggingFace or on disk" into a loaded, pari green); the real Qwen3-0.6B safetensors GPU load+decode re-passed unchanged with the gate live (the cached checkpoint's `tokenizer_config` EOS `<|im_end|>`→151645 agrees with `config.json` and its ChatML template is Hermes). What remains is split to the item below. -- [ ] **Loaders open the tokenizer for `check_ids_against` + config-driven BOS** — the 2026-07-19 gate is +- [x] **Loaders open the tokenizer for `check_ids_against` + config-driven BOS** — the 2026-07-19 gate is tokenizer-free (it cross-checks `tokenizer_config.json` ↔ `config.json` only). The remaining half of the wiring needs the loader to actually *open* the HF `tokenizer.json`: run `check_ids_against(token_to_id)` (every added-token id vs the real tokenizer) at load, and *drive* the model's EOS/BOS ids from @@ -528,7 +555,20 @@ The subsystem that turns "a model on HuggingFace or on disk" into a loaded, pari `eos_token_id` is already the source of truth and is now cross-checked to agree, and BOS/tokenization stays caller-side by design; surfacing the parsed `TokenizerConfig` on the `Loaded*` structs for a consumer to read is the intended shape, left to a dedicated decision. -- [ ] **Evaluate `hf-chat-template` to render the imported `chat_template`** — Mummu's prompt wrapping is + *(2026-07-21)* **Config-driven EOS/BOS shipped — the intended shape, exactly as scoped.** + `validate_checkpoint_dir` already parsed and *returned* the sibling `TokenizerConfig`, but all three + safetensors loaders discarded it; they now capture it and surface it as a new public + `tokenizer_config: Option` field on `LoadedQwen2`/`LoadedQwen3`/`LoadedLfm2`. A consumer + reads config-driven EOS/BOS/PAD straight off the loaded model (`m.tokenizer_config.as_ref().and_then(|c| + c.eos_id())`, `.bos_id()`, `.add_bos_token`) instead of hardcoding. Deliberately *additive*: the model's + internal `is_eos` still rides `config.json`'s `eos_token_id` (already the cross-checked source of truth per + the 2026-07-19 gate), so decode/parity behavior is byte-unchanged; a debug-assert in each safetensors + loader upholds the invariant that any surfaced EOS agrees with `config.json`. GGUF loads surface `None` + (self-contained — that path reads no sibling `tokenizer_config.json`; EOS rides GGUF metadata). REAL-GPU + proof (`tests/real_qwen3.rs`): the safetensors leg asserts the surfaced config's `eos_id()` == 151645 + (`<|im_end|>`) and agrees with `config.json`; the GGUF leg asserts `tokenizer_config.is_none()`. 176 unit + tests + parity + budget gates unmoved (additive field, off every hot path). +- [x] **Evaluate `hf-chat-template` to render the imported `chat_template`** — Mummu's prompt wrapping is hardcoded, byte-verified `chat` renderers (one per family); the `hf-chat-template` crate (built on **minijinja** + a transformers compatibility layer) renders an arbitrary HF `chat_template` Jinja string **byte-identically to `transformers.apply_chat_template`**, tools included. Two payoffs to weigh: (1) a @@ -548,6 +588,31 @@ The subsystem that turns "a model on HuggingFace or on disk" into a loaded, pari with `ChatMl::{qwen2,lfm2}().render_with_tools(...)` on the parity-committed prompts would turn "template-vs-renderer consistency" into a true byte gate. Still gate on reproducing the committed prompts byte-for-byte before trusting it. — https://lib.rs/crates/hf-chat-template + *(2026-07-22)* Crate is now at **0.2.1** (June 20, 2026 — a same-day patch over the 0.2.0 evaluated + above; same `RenderInput` surface). Version to use when this is picked up. + *(2026-07-23)* **Adopted as a dev-dependency; payoff (1) SHIPPED as `tests/template_gate.rs` — and it + caught two real divergences.** The gate renders the cached Qwen3-0.6B checkpoint's own imported + template through `hf-chat-template` 0.2.1 (default features on — real HF templates call minijinja's + Python-compat string methods, `startswith` included) and byte-compares against `ChatMl::qwen2()`: + **all four legs are byte-identical** — plain (142 B), multi-turn (201 B), full Hermes tools block + (748 B), and FC history (`` turn + tool response, 324 B). Getting there surfaced: (a) + `hf-chat-template` hard-requires `serde_json/preserve_order`, so as a dev-dep it silently flipped + test builds to insertion-order JSON while production builds stayed alphabetical — prompt bytes would + have differed between what tests verify and what consumers ship. Resolved by making `preserve_order` + a first-class workspace feature (insertion order is what Python/transformers renders — the training + distribution). (b) our tool JSON was serde-compact (`{"a":1}`) where transformers' `tojson` emits + Python `json.dumps` spacing (`{"a": 1}` — the spacing models emit back in their own tool calls); + fixed with a ~30-line `python_json` serializer (custom `serde_json::ser::Formatter`) now used for + every prompt-JSON site (Hermes `` block, LFM `List of tools:` line, `` history + blocks). Live re-proof after both changes: Qwen3-0.6B greedy-emitted a clean parseable + `{"name": "get_weather", ...}` from the new prompt bytes on the 4070 Ti SUPER. + Payoff (2) — a general fallback renderer for checkpoints without a hardcoded family renderer — is + split below. +- [ ] **General fallback chat renderer via `hf-chat-template`** — payoff (2) of the evaluation above: for a + checkpoint whose family has no hardcoded `chat` renderer, render prompts from its own imported + `chat_template` (the byte gate proved fidelity on Qwen3). Weigh promoting the dep from dev to optional + runtime feature vs the from-scratch ethos; needs the P8/consumer-facing API decision of when to prefer + the imported template over a family renderer. *(2026-07-23, split from the evaluation.)* - [x] **Model registry / manifest** — a declarative `ModelSpec` (repo, architecture, weight format, dtype, tokenizer, chat template, size tier) + a small built-in catalog of known-good models (Qwen2.5, LFM2.5, MiniLM, …); adding a model = a manifest entry. *(2026-07-10) `mummu::registry`: `ModelSpec` @@ -673,7 +738,20 @@ that fits the model AND uses every device to the fullest. under the 150 ms ceiling); VRAM peak unchanged (11.5 GiB whole-card). Parity gate byte-identical (max |Δlogit| 2.670e-5, Ollama greedy exact); f16 island + CPU budget gates re-passed (108.4 ms / 11.7 tok/s GPU gate, 13.2 tok/s CPU). `bench/BASELINE.md` re-baselined.* -- [ ] **Placement plan** — given model size + KV-cache + display headroom and the device set, choose a +- [ ] **Per-device default-dtype policy when mixing precisions in one process** — Burn 0.21 resolves + unspecified-dtype tensor creation against a per-DEVICE settings policy (`get_device_settings` / + `set_default_dtypes`), not the backend type alias: a `GpuF16` client and a `Gpu` client sharing the + same device inside one process flip each other's ambient float dtype. *(2026-07-23, found live)*: with + the three `real_qwen3` GPU tests in one binary, running the f16 leg before the f32 GGUF leg made the + GGUF load's zeros-probed `target_float` come back **F16**, so the whole f32 model loaded/ran in f16 and + the strict f32 logits readback died `TypeMismatch("expected F16, got F32")` — deterministic + sequentially, racy in parallel; A/B-confirmed pre-existing at f1e547a. Two mitigations landed + (2026-07-23): every loader now takes `target_float` from the TYPE (`::dtype()`, + never a probe tensor), and all `GpuF16` legs live in their own test binary (`real_f16.rs`) = separate + process. What remains for the P6 planner (which will legitimately run f32 and f16 models side by side): + decide the policy explicitly — call `set_default_dtypes` per device at model-load/planner level (or pin + every runtime tensor creation site's dtype) so in-process mixed-precision is defined behavior, and add a + two-alias regression test once it is. **fit-and-fill** plan: single GPU when it fits; **shard layers across multiple GPUs** (pipeline/ layer-parallel over Burn's multi-device tensors — Burn gives the multi-device *primitives*, not automatic tensor-parallel, so we place modules on devices ourselves); **spill cold layers to CPU** (GGUF-style diff --git a/crates/mummu/Cargo.toml b/crates/mummu/Cargo.toml index 99562d0..3d14925 100644 --- a/crates/mummu/Cargo.toml +++ b/crates/mummu/Cargo.toml @@ -21,6 +21,15 @@ once_cell = { workspace = true } burn-flex = { workspace = true } burn-store = { workspace = true } +[dev-dependencies] +# Test-only reference renderer for the imported `chat_template` Jinja: claims +# byte-identical output to `transformers.apply_chat_template`. Powers the +# template-vs-renderer BYTE gate (tests/template_gate.rs); production prompt +# wrapping stays the hardcoded, byte-verified `chat` renderers. +# Default features stay ON: they carry minijinja's Python-compat layer +# (str.startswith etc.), which real HF templates (Qwen3's included) call. +hf-chat-template = "0.2.1" + [target.'cfg(windows)'.dependencies] # One syscall (GlobalMemoryStatusEx) for the device inventory's total-RAM # figure; the narrow feature keeps the compile cost near zero. (The DXGI diff --git a/crates/mummu/src/chat.rs b/crates/mummu/src/chat.rs index 30ffdd8..ff7a5d6 100644 --- a/crates/mummu/src/chat.rs +++ b/crates/mummu/src/chat.rs @@ -96,7 +96,7 @@ impl Turn { let blocks: Vec = calls .iter() .map(|c| { - let json = serde_json::to_string(c).unwrap_or_default(); + let json = python_json(c); debug_assert!(!json.is_empty(), "a ToolCall always serializes"); format!("\n{json}\n") }) @@ -144,6 +144,57 @@ impl Turn { /// past any real argument payload, and the recursion bound for both. const MAX_VALUE_DEPTH: usize = 8; +/// Serialize a value the way Python's `json.dumps` does by default — `", "` +/// between items, `": "` after keys. Prompt JSON renders this way +/// deliberately: it is byte-for-byte what `transformers.apply_chat_template` +/// produces through Jinja's `tojson` (and the spacing the models emit back in +/// their own `` JSON), pinned by the template byte gate +/// (`tests/template_gate.rs`). +fn python_json(value: &T) -> String { + struct PySeparators; + impl serde_json::ser::Formatter for PySeparators { + fn begin_array_value(&mut self, writer: &mut W, first: bool) -> std::io::Result<()> + where + W: ?Sized + std::io::Write, + { + if !first { + writer.write_all(b", ")?; + } + Ok(()) + } + + fn begin_object_key(&mut self, writer: &mut W, first: bool) -> std::io::Result<()> + where + W: ?Sized + std::io::Write, + { + if !first { + writer.write_all(b", ")?; + } + Ok(()) + } + + fn begin_object_value(&mut self, writer: &mut W) -> std::io::Result<()> + where + W: ?Sized + std::io::Write, + { + writer.write_all(b": ") + } + } + + let mut out = Vec::with_capacity(128); + let mut ser = serde_json::Serializer::with_formatter(&mut out, PySeparators); + let serialized = serde::Serialize::serialize(value, &mut ser).is_ok(); + debug_assert!(serialized, "prompt JSON values always serialize"); + debug_assert!( + !out.is_empty() || !serialized, + "a serialized value is non-empty" + ); + if !serialized { + return String::new(); + } + String::from_utf8(out).unwrap_or_default() +} + /// Render tool calls as LFM's Pythonic call list: `[name(k=v, …), …]`. /// JSON scalars map to Python spellings (`true`→`True`, `null`→`None`); /// strings/lists/objects render as Python literals. @@ -767,11 +818,10 @@ impl ChatMl { You are provided with function signatures within XML tags:\n", ); for tool in tools { - let json = serde_json::to_string(&ToolWire { + let json = python_json(&ToolWire { r#type: "function", function: tool, - }) - .unwrap_or_default(); + }); debug_assert!(!json.is_empty(), "a ToolSpec always serializes"); system.push('\n'); system.push_str(&json); @@ -804,7 +854,7 @@ impl ChatMl { if i > 0 { system.push_str(", "); } - let json = serde_json::to_string(tool).unwrap_or_default(); + let json = python_json(tool); debug_assert!(!json.is_empty(), "a ToolSpec always serializes"); system.push_str(&json); } @@ -883,8 +933,9 @@ mod tests { /// The tools section must match the Qwen2.5/Qwen3 chat template's wording /// and tag structure byte-for-byte (the model was trained on this text). - /// Inside a tool's `parameters` schema, keys serialize in serde_json's - /// canonical (sorted) order — key order isn't part of the trained text. + /// Inside a tool's `parameters` schema, keys serialize in INSERTION order + /// (serde_json `preserve_order`, a workspace feature) — matching how + /// Python/transformers renders the same schema from a dict. #[test] fn tools_render_matches_the_hermes_template_shape() { let raw = ChatMl::qwen2().render_with_tools( @@ -898,7 +949,7 @@ mod tests { You may call one or more functions to assist with the user query.\n\n\ You are provided with function signatures within XML tags:\n\ \n\ - {\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"description\":\"Get the current weather for a city.\",\"parameters\":{\"properties\":{\"city\":{\"type\":\"string\"}},\"required\":[\"city\"],\"type\":\"object\"}}}\n\ + {\"type\": \"function\", \"function\": {\"name\": \"get_weather\", \"description\": \"Get the current weather for a city.\", \"parameters\": {\"type\": \"object\", \"properties\": {\"city\": {\"type\": \"string\"}}, \"required\": [\"city\"]}}}\n\ \n\n\ For each function call, return a json object with function name and arguments within XML tags:\n\ \n{\"name\": , \"arguments\": }\n<|im_end|>\n\ @@ -927,10 +978,11 @@ mod tests { Turn::tool_response("{\"temp_c\": 21}"), Turn::tool_response("{\"temp_c\": 24}"), ]); - // The assistant history turn carries the block it emitted. + // The assistant history turn carries the block it emitted + // (Python json.dumps spacing — the shape the model itself emits). assert!(raw.contains( "<|im_start|>assistant\n\n\ - {\"name\":\"get_weather\",\"arguments\":{\"city\":\"Paris\"}}\n\ + {\"name\": \"get_weather\", \"arguments\": {\"city\": \"Paris\"}}\n\ <|im_end|>\n" )); // Both results ride in ONE user turn, each in its own block. @@ -1010,7 +1062,7 @@ mod tests { ], ); let expected = "<|startoftext|><|im_start|>system\nYou are a helpful assistant.\n\ - List of tools: [{\"name\":\"get_weather\",\"description\":\"Get the current weather for a city.\",\"parameters\":{\"properties\":{\"city\":{\"type\":\"string\"}},\"required\":[\"city\"],\"type\":\"object\"}}]<|im_end|>\n\ + List of tools: [{\"name\": \"get_weather\", \"description\": \"Get the current weather for a city.\", \"parameters\": {\"type\": \"object\", \"properties\": {\"city\": {\"type\": \"string\"}}, \"required\": [\"city\"]}}]<|im_end|>\n\ <|im_start|>user\nWeather in Paris?<|im_end|>\n\ <|im_start|>assistant\n"; assert_eq!(raw, expected); @@ -1030,8 +1082,8 @@ mod tests { let mut second = weather_tool(); second.name = "get_time".into(); let raw = ChatMl::lfm2().render_with_tools(&[weather_tool(), second], &[Turn::user("hi")]); - assert!(raw.contains("\"name\":\"get_weather\"")); - assert!(raw.contains("}, {\"name\":\"get_time\"")); + assert!(raw.contains("\"name\": \"get_weather\"")); + assert!(raw.contains("}, {\"name\": \"get_time\"")); assert_eq!(raw.matches("List of tools: [").count(), 1); } @@ -1115,12 +1167,13 @@ mod tests { }), }]; let turn = Turn::assistant_tool_calls_lfm(&calls); - // serde_json object keys iterate in sorted order. + // serde_json `preserve_order` (workspace feature): object keys iterate + // in INSERTION order — the same order Python/transformers renders. assert_eq!( turn.content, - "<|tool_call_start|>[f(i=-3, list=[1, \"two\"], map={\"k\": True}, \ - no=False, nothing=None, s=\"he said \\\"hi\\\"\\n\", x=1.5, \ - yes=True)]<|tool_call_end|>" + "<|tool_call_start|>[f(s=\"he said \\\"hi\\\"\\n\", i=-3, x=1.5, \ + yes=True, no=False, nothing=None, list=[1, \"two\"], \ + map={\"k\": True})]<|tool_call_end|>" ); } diff --git a/crates/mummu/src/models/lfm2.rs b/crates/mummu/src/models/lfm2.rs index a43c830..af3c10e 100644 --- a/crates/mummu/src/models/lfm2.rs +++ b/crates/mummu/src/models/lfm2.rs @@ -279,6 +279,12 @@ pub enum HybridKv { pub struct LoadedLfm2 { pub model: Lfm2, pub config: Lfm2Config, + /// The parsed sibling `tokenizer_config.json`, when one was present and + /// well-formed beside a safetensors checkpoint (the load-time gate has + /// already cross-checked its EOS against `config.json`). A consumer reads + /// config-driven EOS/BOS/PAD ids from it (`eos_id()`, `bos_id()`, …). `None` + /// for a GGUF load (self-contained; no sibling file) or a dir without one. + pub tokenizer_config: Option, } fn build(cfg: &Lfm2Config, device: &B::Device) -> Lfm2 { @@ -345,14 +351,24 @@ pub fn load_from_dir( // tokenizer_config.json EOS agreement with config.json, a chat-template that // speaks LFM2.5's bracket-notation tool-call convention, and added-token ids // that match the real tokenizer.json — a repackaging mismatch fails loudly. - crate::tokenizer::validate_checkpoint_dir( + let tokenizer_config = crate::tokenizer::validate_checkpoint_dir( dir, &config.eos_token_id.to_vec(), Some(crate::tok_config::ToolCallConvention::Lfm), )?; + debug_assert!( + tokenizer_config + .as_ref() + .and_then(crate::tok_config::TokenizerConfig::eos_id) + .is_none_or(|id| config.eos_token_id.contains(id)), + "validate_checkpoint_dir returned a config whose EOS disagrees with config.json" + ); let mut model = build::(&config, device); - let target_float = Tensor::::zeros([1], device).dtype(); + // Type-level float dtype (`B::FloatElem`) — a probe tensor would follow + // the per-DEVICE default policy, which another backend alias sharing the + // device (Gpu vs GpuF16) may have flipped in this process. + let target_float = ::dtype(); let mut store = SafetensorsStore::from_file(weights.clone()) .with_from_adapter(PyTorchToBurnAdapter.chain(CastFloatAdapter::new(target_float))) .allow_partial(true) @@ -368,7 +384,11 @@ pub fn load_from_dir( "$1.gamma", ); load_checked(&mut model, &mut store, &weights)?; - Ok(LoadedLfm2 { model, config }) + Ok(LoadedLfm2 { + model, + config, + tokenizer_config, + }) } /// GGUF (llama.cpp `lfm2` arch) tensor names → the HF checkpoint names the @@ -436,7 +456,10 @@ pub fn load_from_gguf( assert!(blob.len() > 8, "a parsed GGUF yields a non-empty blob"); let mut model = build::(&config, device); - let target_float = Tensor::::zeros([1], device).dtype(); + // Type-level float dtype (`B::FloatElem`) — a probe tensor would follow + // the per-DEVICE default policy, which another backend alias sharing the + // device (Gpu vs GpuF16) may have flipped in this process. + let target_float = ::dtype(); let mut store = SafetensorsStore::from_bytes(Some(blob)) .with_from_adapter(PyTorchToBurnAdapter.chain(CastFloatAdapter::new(target_float))) .allow_partial(true) @@ -452,7 +475,12 @@ pub fn load_from_gguf( "$1.gamma", ); load_checked(&mut model, &mut store, path)?; - Ok(LoadedLfm2 { model, config }) + // A GGUF is self-contained — no sibling tokenizer_config.json in this path. + Ok(LoadedLfm2 { + model, + config, + tokenizer_config: None, + }) } impl CausalLm for LoadedLfm2 { @@ -632,6 +660,7 @@ mod tests { let loaded = LoadedLfm2:: { model: build(&cfg, &device), config: cfg, + tokenizer_config: None, }; let prompt: Vec = vec![5, 11, 2, 30]; @@ -663,6 +692,7 @@ mod tests { let loaded = LoadedLfm2:: { model: build(&cfg, &device), config: cfg, + tokenizer_config: None, }; let out = loaded.greedy_generate(&[1, 2], 3, &device).unwrap(); assert!(out.len() <= 3); diff --git a/crates/mummu/src/models/minilm.rs b/crates/mummu/src/models/minilm.rs index 02be78f..922ded3 100644 --- a/crates/mummu/src/models/minilm.rs +++ b/crates/mummu/src/models/minilm.rs @@ -172,7 +172,10 @@ pub fn load_from_dir( })?; let mut model = build::(&config, device); - let target_float = Tensor::::zeros([1], device).dtype(); + // Type-level float dtype (`B::FloatElem`) — a probe tensor would follow + // the per-DEVICE default policy, which another backend alias sharing the + // device (Gpu vs GpuF16) may have flipped in this process. + let target_float = ::dtype(); match weights_file(dir)? { WeightsFile::Safetensors(weights) => { let mut store = SafetensorsStore::from_file(weights.clone()) diff --git a/crates/mummu/src/models/qwen2.rs b/crates/mummu/src/models/qwen2.rs index 550a8d4..8417630 100644 --- a/crates/mummu/src/models/qwen2.rs +++ b/crates/mummu/src/models/qwen2.rs @@ -201,6 +201,12 @@ pub struct Qwen2 { pub struct LoadedQwen2 { pub model: Qwen2, pub config: Qwen2Config, + /// The parsed sibling `tokenizer_config.json`, when one was present and + /// well-formed beside a safetensors checkpoint (the load-time gate has + /// already cross-checked its EOS against `config.json`). A consumer reads + /// config-driven EOS/BOS/PAD ids from it (`eos_id()`, `bos_id()`, …). `None` + /// for a GGUF load (self-contained; no sibling file) or a dir without one. + pub tokenizer_config: Option, } fn build(cfg: &Qwen2Config, device: &B::Device) -> Qwen2 { @@ -268,15 +274,25 @@ pub fn load_from_dir( // tokenizer_config.json EOS agreement with config.json, a chat-template that // speaks Qwen2's Hermes/ChatML tool-call convention, and added-token ids that // match the real tokenizer.json — a repackaging mismatch fails loudly at load. - crate::tokenizer::validate_checkpoint_dir( + let tokenizer_config = crate::tokenizer::validate_checkpoint_dir( dir, &config.eos_token_id.to_vec(), Some(crate::tok_config::ToolCallConvention::Hermes), )?; + debug_assert!( + tokenizer_config + .as_ref() + .and_then(crate::tok_config::TokenizerConfig::eos_id) + .is_none_or(|id| config.eos_token_id.contains(id)), + "validate_checkpoint_dir returned a config whose EOS disagrees with config.json" + ); let mut model = build::(&config, device); - // The backend's own float dtype (f32, or f16 on the GpuF16 alias). - let target_float = Tensor::::zeros([1], device).dtype(); + // The backend's own float dtype (f32, or f16 on the GpuF16 alias), taken + // from the TYPE (`B::FloatElem`), never from a probe tensor: unspecified- + // dtype tensor creation follows the per-DEVICE default policy, which + // another backend alias sharing the device may have flipped in-process. + let target_float = ::dtype(); let mut store = SafetensorsStore::from_file(weights.clone()) .with_from_adapter(PyTorchToBurnAdapter.chain(CastFloatAdapter::new(target_float))) .allow_partial(true) @@ -285,7 +301,11 @@ pub fn load_from_dir( .with_key_remapping(r"(post_attention_layernorm)\.weight$", "$1.gamma") .with_key_remapping(r"^norm\.weight$", "norm.gamma"); load_checked(&mut model, &mut store, &weights)?; - Ok(LoadedQwen2 { model, config }) + Ok(LoadedQwen2 { + model, + config, + tokenizer_config, + }) } /// GGUF (llama.cpp) tensor names → the HF checkpoint names the safetensors @@ -343,7 +363,10 @@ pub fn load_from_gguf( assert!(blob.len() > 8, "a parsed GGUF yields a non-empty blob"); let mut model = build::(&config, device); - let target_float = Tensor::::zeros([1], device).dtype(); + // Type-level float dtype (`B::FloatElem`) — a probe tensor would follow + // the per-DEVICE default policy, which another backend alias sharing the + // device (Gpu vs GpuF16) may have flipped in this process. + let target_float = ::dtype(); let mut store = SafetensorsStore::from_bytes(Some(blob)) .with_from_adapter(PyTorchToBurnAdapter.chain(CastFloatAdapter::new(target_float))) .allow_partial(true) @@ -352,7 +375,12 @@ pub fn load_from_gguf( .with_key_remapping(r"(post_attention_layernorm)\.weight$", "$1.gamma") .with_key_remapping(r"^norm\.weight$", "norm.gamma"); load_checked(&mut model, &mut store, path)?; - Ok(LoadedQwen2 { model, config }) + // A GGUF is self-contained — no sibling tokenizer_config.json in this path. + Ok(LoadedQwen2 { + model, + config, + tokenizer_config: None, + }) } impl CausalLm for LoadedQwen2 { @@ -494,6 +522,7 @@ mod tests { let loaded = LoadedQwen2:: { model: build(&cfg, &device), config: cfg, + tokenizer_config: None, }; let prompt: Vec = vec![3, 14, 15, 9, 26]; @@ -591,6 +620,7 @@ mod tests { let loaded = LoadedQwen2:: { model: build(&cfg, &device), config: cfg, + tokenizer_config: None, }; assert!(loaded.model.lm_head.is_some()); let mut cache = loaded.new_cache(); @@ -643,6 +673,7 @@ mod tests { let loaded = LoadedQwen2:: { model: build(&cfg, &device), config: cfg, + tokenizer_config: None, }; // A built (random-weight) model computes a live, finite, non-degenerate // distribution — the smoke passes and reports a valid top id. @@ -662,6 +693,7 @@ mod tests { let loaded = LoadedQwen2:: { model: build(&cfg, &device), config: cfg, + tokenizer_config: None, }; let out = loaded.greedy_generate(&[1, 2, 3], 4, &device).unwrap(); assert!(out.len() <= 4); diff --git a/crates/mummu/src/models/qwen3.rs b/crates/mummu/src/models/qwen3.rs index 16eaa05..00cfd1f 100644 --- a/crates/mummu/src/models/qwen3.rs +++ b/crates/mummu/src/models/qwen3.rs @@ -166,6 +166,12 @@ pub struct Qwen3 { pub struct LoadedQwen3 { pub model: Qwen3, pub config: Qwen3Config, + /// The parsed sibling `tokenizer_config.json`, when one was present and + /// well-formed beside a safetensors checkpoint (the load-time gate has + /// already cross-checked its EOS against `config.json`). A consumer reads + /// config-driven EOS/BOS/PAD ids from it (`eos_id()`, `bos_id()`, …). `None` + /// for a GGUF load (self-contained; no sibling file) or a dir without one. + pub tokenizer_config: Option, } fn build(cfg: &Qwen3Config, device: &B::Device) -> Qwen3 { @@ -242,21 +248,36 @@ pub fn load_from_dir( // renderer, and every added-token id it declares must match the real // tokenizer.json. A repackaging mismatch fails loudly here rather than // mis-stopping / mis-templating / mis-tokenizing later. - crate::tokenizer::validate_checkpoint_dir( + let tokenizer_config = crate::tokenizer::validate_checkpoint_dir( dir, &config.eos_token_id.to_vec(), Some(crate::tok_config::ToolCallConvention::Hermes), )?; + debug_assert!( + tokenizer_config + .as_ref() + .and_then(crate::tok_config::TokenizerConfig::eos_id) + .is_none_or(|id| config.eos_token_id.contains(id)), + "validate_checkpoint_dir returned a config whose EOS disagrees with config.json" + ); let mut model = build::(&config, device); - let target_float = Tensor::::zeros([1], device).dtype(); + // The backend's float dtype, taken from the TYPE (`B::FloatElem`), never + // from a probe tensor: unspecified-dtype tensor creation follows the + // per-DEVICE default policy, which another backend alias sharing the + // device (Gpu vs GpuF16) may have flipped in this process. + let target_float = ::dtype(); let mut store = install_remaps( SafetensorsStore::from_file(weights.clone()) .with_from_adapter(PyTorchToBurnAdapter.chain(CastFloatAdapter::new(target_float))) .allow_partial(true), ); load_checked(&mut model, &mut store, &weights)?; - Ok(LoadedQwen3 { model, config }) + Ok(LoadedQwen3 { + model, + config, + tokenizer_config, + }) } /// GGUF (llama.cpp `qwen3` arch) tensor names → the HF checkpoint names the @@ -311,14 +332,23 @@ pub fn load_from_gguf( assert!(blob.len() > 8, "a parsed GGUF yields a non-empty blob"); let mut model = build::(&config, device); - let target_float = Tensor::::zeros([1], device).dtype(); + // The backend's float dtype, taken from the TYPE (`B::FloatElem`), never + // from a probe tensor: unspecified-dtype tensor creation follows the + // per-DEVICE default policy, which another backend alias sharing the + // device (Gpu vs GpuF16) may have flipped in this process. + let target_float = ::dtype(); let mut store = install_remaps( SafetensorsStore::from_bytes(Some(blob)) .with_from_adapter(PyTorchToBurnAdapter.chain(CastFloatAdapter::new(target_float))) .allow_partial(true), ); load_checked(&mut model, &mut store, path)?; - Ok(LoadedQwen3 { model, config }) + // A GGUF is self-contained — no sibling tokenizer_config.json in this path. + Ok(LoadedQwen3 { + model, + config, + tokenizer_config: None, + }) } impl CausalLm for LoadedQwen3 { @@ -468,6 +498,7 @@ mod tests { let loaded = LoadedQwen3:: { model: build(&cfg, &device), config: cfg, + tokenizer_config: None, }; let prompt: Vec = vec![3, 14, 15, 9, 26]; @@ -502,6 +533,7 @@ mod tests { let loaded = LoadedQwen3:: { model: build(&cfg, &device), config: cfg, + tokenizer_config: None, }; assert!(loaded.model.lm_head.is_some()); let mut cache = loaded.new_cache(); @@ -611,6 +643,7 @@ mod tests { let loaded = LoadedQwen3:: { model: build(&cfg, &device), config: cfg, + tokenizer_config: None, }; let out = loaded.greedy_generate(&[1, 2, 3], 4, &device).unwrap(); assert!(out.len() <= 4); diff --git a/crates/mummu/tests/real_f16.rs b/crates/mummu/tests/real_f16.rs index 79d3e03..9e711d0 100644 --- a/crates/mummu/tests/real_f16.rs +++ b/crates/mummu/tests/real_f16.rs @@ -1,17 +1,25 @@ -//! f16 on-GPU validation (the P6 precision milestone): load Qwen2.5 on the -//! `GpuF16` backend and prove the three claims — no shader-compile crash, +//! f16 on-GPU validation (the P6 precision milestone): load real checkpoints +//! on the `GpuF16` backend and prove the claims — no shader-compile crash, //! materially lower VRAM than f32, coherent greedy output. Ignored by //! default; run with //! //! ```text -//! MUMMU_QWEN2_DIR=path/to/qwen2.5-1.5b cargo test -p mummu --release --test real_f16 -- --ignored --nocapture +//! MUMMU_QWEN2_DIR=path/to/qwen2.5-1.5b MUMMU_QWEN3_DIR=path/to/qwen3-0.6b \ +//! cargo test -p mummu --release --test real_f16 -- --ignored --nocapture //! ``` +//! +//! Every `GpuF16` leg lives in THIS binary, isolated from the f32 suites: +//! Burn resolves unspecified-dtype tensor creation against a per-DEVICE +//! default policy, so a `GpuF16` client and a `Gpu` client sharing the +//! device inside one process can flip each other's ambient float dtype +//! (observed 2026-07-23: an f32 GGUF leg read back F16 logits after an f16 +//! test ran first). Separate test binaries = separate processes = isolation. use std::path::PathBuf; use mummu::backend::{GpuF16, inventory}; use mummu::models::CausalLm; -use mummu::models::qwen2; +use mummu::models::{qwen2, qwen3}; use tokenizers::Tokenizer; fn qwen2_dir() -> Option { @@ -19,6 +27,12 @@ fn qwen2_dir() -> Option { dir.is_dir().then_some(dir) } +fn qwen3_dir() -> Option { + std::env::var_os("MUMMU_QWEN3_DIR") + .map(PathBuf::from) + .filter(|d| d.join("model.safetensors").is_file()) +} + #[test] #[ignore = "needs multi-GB local weights (MUMMU_QWEN2_DIR) + a SHADER_F16 GPU"] fn qwen2_decodes_coherently_in_f16_on_gpu() { @@ -66,3 +80,52 @@ fn qwen2_decodes_coherently_in_f16_on_gpu() { // Claim 2 (VRAM) is measured outside the process (nvidia-smi peak while // this test runs) — recorded in bench/BASELINE.md. } + +/// f16 leg for the Qwen3 dense arch — bf16 weights cast to f16 on load +/// (`CastFloatAdapter`), and its per-head q/k RMSNorm + decoupled head_dim +/// ride the SAME f32-softmax attention island Qwen2/LFM2 use, so the q·kᵀ +/// scores never overflow f16. Proves the dtype path (P3) and the f16 +/// precision milestone (P6) cover the new architecture, not just Qwen2. +/// (Moved here from `real_qwen3.rs` for the process isolation the module +/// docs describe.) +#[test] +#[ignore = "needs the Qwen3 safetensors dir (MUMMU_QWEN3_DIR) + a SHADER_F16 GPU"] +fn real_qwen3_decodes_coherently_in_f16() { + let dir = qwen3_dir().expect("set MUMMU_QWEN3_DIR to a Qwen3 safetensors dir"); + assert!( + inventory().any_shader_f16(), + "no adapter advertises SHADER_F16 — cannot validate f16 here" + ); + let device = burn::tensor::Device::::default(); + + let tok = Tokenizer::from_file(dir.join("tokenizer.json")).expect("tokenizer.json"); + let prompt_text = mummu::chat::ChatMl::qwen2().render(&[ + mummu::chat::Turn::system("You are a concise assistant. Do not think, answer directly."), + mummu::chat::Turn::user("What is 2+2? Answer in one short sentence."), + ]); + let prompt = tok + .encode(prompt_text, true) + .expect("prompt encodes") + .get_ids() + .to_vec(); + + // bf16 -> f16 on load; the build must not NaN through the qk-norm + softmax. + let model = qwen3::load_from_dir::(&dir, &device).expect("f16 weights load checked"); + let smoke = model + .sanity_check(&prompt, model.config.vocab_size, &device) + .expect("f16 forward is finite and non-degenerate (no overflow to NaN)"); + eprintln!( + "[real_f16/qwen3] sanity smoke: top_id {} · spread {:.3}", + smoke.top_id, smoke.spread + ); + + let ids = model + .greedy_generate(&prompt, 48, &device) + .expect("f16 decode"); + let text = tok.decode(&ids, true).expect("ids decode"); + eprintln!("[real_f16/qwen3] greedy: {text:?}"); + assert!( + text.contains('4'), + "expected the f16 answer to mention 4: {text:?}" + ); +} diff --git a/crates/mummu/tests/real_qwen3.rs b/crates/mummu/tests/real_qwen3.rs index 80f74aa..b707898 100644 --- a/crates/mummu/tests/real_qwen3.rs +++ b/crates/mummu/tests/real_qwen3.rs @@ -11,7 +11,7 @@ use std::path::PathBuf; -use mummu::backend::{Gpu, GpuF16, inventory, use_gpu}; +use mummu::backend::{Gpu, use_gpu}; use mummu::gguf::GgufFile; use mummu::models::CausalLm; use mummu::models::qwen3; @@ -89,6 +89,26 @@ fn real_qwen3_safetensors_loads_and_decodes_on_gpu() { model.config.tie_word_embeddings, ); + // The sibling tokenizer_config.json is surfaced on the loaded struct so a + // consumer can read config-driven EOS/BOS ids (P3). The cached Qwen3-0.6B + // ships one; its resolved EOS agrees with config.json's (the load-time gate + // guarantees it), and add_bos_token is false (ChatML). + let tc = model + .tokenizer_config + .as_ref() + .expect("Qwen3-0.6B ships a tokenizer_config.json, surfaced on the loaded struct"); + eprintln!( + "[real_qwen3] tokenizer_config: eos_id {:?} · bos_id {:?} · add_bos {}", + tc.eos_id(), + tc.bos_id(), + tc.add_bos_token, + ); + assert_eq!(tc.eos_id(), Some(151_645), "config-driven EOS = <|im_end|>"); + assert!( + model.config.eos_token_id.contains(151_645), + "surfaced EOS agrees with config.json" + ); + // The post-import sanity smoke passes on the real weights: one forward // yields finite, vocab-wide, non-degenerate logits (the liveness gate an // app runs right after install). @@ -107,53 +127,6 @@ fn real_qwen3_safetensors_loads_and_decodes_on_gpu() { assert!(text.contains('4'), "expected the answer 4 in: {text:?}"); } -/// f16 leg: the Qwen3 arch runs on the `GpuF16` backend too — bf16 weights -/// cast to f16 on load (`CastFloatAdapter`), and its per-head q/k RMSNorm + -/// decoupled head_dim ride the SAME f32-softmax attention island Qwen2/LFM2 -/// use, so the q·kᵀ scores never overflow f16. Proves the dtype path (P3) and -/// the f16 precision milestone (P6) cover the new architecture, not just Qwen2. -#[test] -#[ignore = "needs the Qwen3 safetensors dir (MUMMU_QWEN3_DIR) + a SHADER_F16 GPU"] -fn real_qwen3_decodes_coherently_in_f16() { - let dir = dir().expect("set MUMMU_QWEN3_DIR to a Qwen3 safetensors dir"); - assert!( - inventory().any_shader_f16(), - "no adapter advertises SHADER_F16 — cannot validate f16 here" - ); - let device = burn::tensor::Device::::default(); - - let tok = tokenizers::Tokenizer::from_file(dir.join("tokenizer.json")).expect("tokenizer.json"); - let prompt_text = mummu::chat::ChatMl::qwen2().render(&[ - mummu::chat::Turn::system("You are a concise assistant. Do not think, answer directly."), - mummu::chat::Turn::user("What is 2+2? Answer in one short sentence."), - ]); - let prompt = tok - .encode(prompt_text, true) - .expect("prompt encodes") - .get_ids() - .to_vec(); - - // bf16 -> f16 on load; the build must not NaN through the qk-norm + softmax. - let model = qwen3::load_from_dir::(&dir, &device).expect("f16 weights load checked"); - let smoke = model - .sanity_check(&prompt, model.config.vocab_size, &device) - .expect("f16 forward is finite and non-degenerate (no overflow to NaN)"); - eprintln!( - "[real_qwen3/f16] sanity smoke: top_id {} · spread {:.3}", - smoke.top_id, smoke.spread - ); - - let ids = model - .greedy_generate(&prompt, 48, &device) - .expect("f16 decode"); - let text = tok.decode(&ids, true).expect("ids decode"); - eprintln!("[real_qwen3/f16] greedy: {text:?}"); - assert!( - text.contains('4'), - "expected the f16 answer to mention 4: {text:?}" - ); -} - /// END-TO-END on the Q4_K_M GGUF alone (config + tokenizer + weights from the /// one file), cross-checked against the bf16 safetensors build: both decode a /// correct answer and agree on the top first-token id (small logit drift IS the @@ -197,6 +170,13 @@ fn real_qwen3_gguf_loads_and_agrees_with_safetensors() { ); let gguf_model = qwen3::load_from_gguf::(&path, &device).expect("gguf load is checked"); + // A GGUF is self-contained — the load path reads no sibling + // tokenizer_config.json, so the surfaced field is None (EOS still rides + // config.eos_token_id, derived from GGUF metadata). + assert!( + gguf_model.tokenizer_config.is_none(), + "a GGUF load surfaces no tokenizer_config" + ); let ids = gguf_model .greedy_generate(&prompt, 48, &device) .expect("decode"); diff --git a/crates/mummu/tests/template_gate.rs b/crates/mummu/tests/template_gate.rs new file mode 100644 index 0000000..acccb9c --- /dev/null +++ b/crates/mummu/tests/template_gate.rs @@ -0,0 +1,221 @@ +//! Template-vs-renderer BYTE gate (the `hf-chat-template` evaluation, P3). +//! +//! Mummu's prompt wrapping is hardcoded, byte-verified `chat` renderers; the +//! checkpoint ships the *authoritative* Jinja `chat_template` we import but do +//! not render. This gate renders the IMPORTED template through +//! `hf-chat-template` (byte-identical to `transformers.apply_chat_template`) +//! and compares against our `ChatMl::qwen2()` renderer on the same +//! conversation — turning "template-vs-renderer consistency" from a marker +//! check into a byte comparison. Ignored by default; run with: +//! +//! ```text +//! MUMMU_QWEN3_DIR=path/to/qwen3-0.6b \ +//! cargo test -p mummu --test template_gate -- --ignored --nocapture +//! ``` + +use std::path::{Path, PathBuf}; + +use hf_chat_template::{ChatTemplate, Message, RenderInput}; +use mummu::chat::{ChatMl, ToolSpec, Turn}; +use mummu::tok_config::TokenizerConfig; + +fn dir() -> Option { + std::env::var_os("MUMMU_QWEN3_DIR") + .map(PathBuf::from) + .filter(|d| d.join("tokenizer_config.json").is_file()) +} + +/// Load the checkpoint's imported chat template and compile it. +fn imported_template(dir: &Path) -> ChatTemplate { + let cfg = TokenizerConfig::from_dir(dir).expect("tokenizer_config.json parses"); + let template = cfg + .chat_template + .as_deref() + .expect("Qwen3 ships a chat template"); + assert!(!template.is_empty(), "template is non-empty"); + ChatTemplate::from_str(template).expect("the imported template compiles") +} + +/// Point out the first byte where two renders diverge (for a readable failure). +fn first_diff(a: &str, b: &str) -> Option { + a.bytes() + .zip(b.bytes()) + .position(|(x, y)| x != y) + .or_else(|| (a.len() != b.len()).then(|| a.len().min(b.len()))) +} + +fn diff_context(label: &str, ours: &str, reference: &str) -> String { + match first_diff(ours, reference) { + None => format!("{label}: byte-identical ({} B)", ours.len()), + Some(pos) => { + let lo = pos.saturating_sub(60); + format!( + "{label}: DIVERGES at byte {pos}\n ours …{:?}\n reference…{:?}", + &ours[lo..(pos + 60).min(ours.len())], + &reference[lo..(pos + 60).min(reference.len())], + ) + } + } +} + +/// The plain conversation leg: system + user + generation prompt must render +/// BYTE-IDENTICALLY through the checkpoint's own template and our renderer. +/// This is the exact shape the parity gates commit. +#[test] +#[ignore = "needs the local Qwen3 checkpoint dir (MUMMU_QWEN3_DIR)"] +fn qwen3_plain_render_byte_matches_the_imported_template() { + let dir = dir().expect("set MUMMU_QWEN3_DIR to a Qwen3 checkpoint dir"); + let template = imported_template(&dir); + + let reference = template + .render(&RenderInput { + messages: vec![ + Message::system("You are a helpful assistant."), + Message::user("List the first five prime numbers."), + ], + add_generation_prompt: true, + ..RenderInput::default() + }) + .expect("reference render succeeds"); + let ours = ChatMl::qwen2().render(&[ + Turn::system("You are a helpful assistant."), + Turn::user("List the first five prime numbers."), + ]); + + println!("{}", diff_context("plain", &ours, &reference)); + assert_eq!(ours, reference, "plain ChatML render must byte-match"); +} + +/// Multi-turn history (user → assistant → user): the template renders a +/// history assistant turn plainly (no think-block resurrection), same as ours. +#[test] +#[ignore = "needs the local Qwen3 checkpoint dir (MUMMU_QWEN3_DIR)"] +fn qwen3_multi_turn_render_byte_matches_the_imported_template() { + let dir = dir().expect("set MUMMU_QWEN3_DIR to a Qwen3 checkpoint dir"); + let template = imported_template(&dir); + + let reference = template + .render(&RenderInput { + messages: vec![ + Message::system("You are a helpful assistant."), + Message::user("Name a prime."), + Message::assistant("2 is prime."), + Message::user("Another?"), + ], + add_generation_prompt: true, + ..RenderInput::default() + }) + .expect("reference render succeeds"); + let ours = ChatMl::qwen2().render(&[ + Turn::system("You are a helpful assistant."), + Turn::user("Name a prime."), + Turn::assistant("2 is prime."), + Turn::user("Another?"), + ]); + + println!("{}", diff_context("multi-turn", &ours, &reference)); + assert_eq!(ours, reference, "multi-turn ChatML render must byte-match"); +} + +/// The tools leg: the full Hermes `# Tools` system block — tool JSON included +/// — must render byte-identically. This holds because `chat` serializes +/// prompt JSON with Python `json.dumps` separators (`python_json`), exactly +/// what the template's `tojson` produces. +#[test] +#[ignore = "needs the local Qwen3 checkpoint dir (MUMMU_QWEN3_DIR)"] +fn qwen3_tools_render_vs_imported_template() { + let dir = dir().expect("set MUMMU_QWEN3_DIR to a Qwen3 checkpoint dir"); + let template = imported_template(&dir); + + let tool_json = serde_json::json!({ + "type": "function", + "function": { + "name": "get_weather", + "description": "Get the current weather in a city.", + "parameters": { + "type": "object", + "properties": { + "city": {"type": "string", "description": "City name"} + }, + "required": ["city"] + } + } + }); + let reference = template + .render(&RenderInput { + messages: vec![ + Message::system("You are a helpful assistant."), + Message::user("What's the weather in Paris?"), + ], + tools: vec![tool_json], + add_generation_prompt: true, + ..RenderInput::default() + }) + .expect("reference render succeeds"); + + let spec = ToolSpec { + name: "get_weather".into(), + description: "Get the current weather in a city.".into(), + parameters: serde_json::json!({ + "type": "object", + "properties": { + "city": {"type": "string", "description": "City name"} + }, + "required": ["city"] + }), + }; + let ours = ChatMl::qwen2().render_with_tools( + &[spec], + &[ + Turn::system("You are a helpful assistant."), + Turn::user("What's the weather in Paris?"), + ], + ); + + println!("{}", diff_context("tools", &ours, &reference)); + assert_eq!(ours, reference, "Hermes tools render must byte-match"); +} + +/// Function-calling HISTORY: an assistant `` turn plus its tool +/// response must re-render byte-identically through both paths — the whole +/// multi-step-tool loop an agent replays every round. +#[test] +#[ignore = "needs the local Qwen3 checkpoint dir (MUMMU_QWEN3_DIR)"] +fn qwen3_tool_history_render_byte_matches_the_imported_template() { + let dir = dir().expect("set MUMMU_QWEN3_DIR to a Qwen3 checkpoint dir"); + let template = imported_template(&dir); + + let call_json = serde_json::json!({ + "name": "get_weather", + "arguments": {"city": "Paris"} + }); + let mut assistant_call = Message::assistant(""); + assistant_call.content = None; + assistant_call.tool_calls = vec![call_json]; + let reference = template + .render(&RenderInput { + messages: vec![ + Message::system("You are a helpful assistant."), + Message::user("What's the weather in Paris?"), + assistant_call, + Message::new("tool", "{\"temp_c\": 21}"), + ], + add_generation_prompt: true, + ..RenderInput::default() + }) + .expect("reference render succeeds"); + + let calls = [mummu::chat::ToolCall { + name: "get_weather".into(), + arguments: serde_json::json!({"city": "Paris"}), + }]; + let ours = ChatMl::qwen2().render(&[ + Turn::system("You are a helpful assistant."), + Turn::user("What's the weather in Paris?"), + Turn::assistant_tool_calls(&calls), + Turn::tool_response("{\"temp_c\": 21}"), + ]); + + println!("{}", diff_context("tool-history", &ours, &reference)); + assert_eq!(ours, reference, "FC history render must byte-match"); +}